diff --git a/.gitignore b/.gitignore
index b4ec8795057..5d7dbbefdc8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,11 +38,14 @@ dependency-reduced-pom.xml
 # osx stuff
 .DS_Store
 
+# default folders in which the create_bwc_index.py expects to find old es versions in
+/backwards
+/dev-tools/backwards
+
 # needed in case docs build is run...maybe we can configure doc build to generate files under build?
 html_docs
 
 # random old stuff that we should look at the necessity of...
 /tmp/
-backwards/
 eclipse-build
 
diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index dcd6c9981be..a1a01a8f231 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -25,12 +25,6 @@ run it using Gradle:
 gradle run
 -------------------------------------
 
-or to attach a remote debugger, run it as:
-
--------------------------------------
-gradle run --debug-jvm
--------------------------------------
-
 === Test case filtering.
 
 - `tests.class` is a class-filtering shell-like glob pattern,
@@ -430,15 +424,33 @@ cd $BATS_ARCHIVES
 sudo -E bats $BATS_TESTS/*.bats
 -------------------------------------------------
 
-Note: Starting vagrant VM outside of the elasticsearch folder requires to
-indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
-environment variable:
+You can also use Gradle to prepare the test environment and then starts a single VM:
 
 -------------------------------------------------
-gradle vagrantSetUp
-VAGRANT_CWD=/path/to/elasticsearch vagrant up centos-7 --provider virtualbox
+gradle vagrantFedora24#up
 -------------------------------------------------
 
+Or any of vagrantCentos6#up, vagrantDebian8#up, vagrantFedora24#up, vagrantOel6#up,
+vagrantOel7#up, vagrantOpensuse13#up, vagrantSles12#up, vagrantUbuntu1204#up,
+vagrantUbuntu1604#up.
+
+Once up, you can then connect to the VM using SSH from the elasticsearch directory:
+
+-------------------------------------------------
+vagrant ssh fedora-24
+-------------------------------------------------
+
+Or from another directory:
+
+-------------------------------------------------
+VAGRANT_CWD=/path/to/elasticsearch vagrant ssh fedora-24
+-------------------------------------------------
+
+Note: Starting vagrant VM outside of the elasticsearch folder requires to
+indicates the folder that contains the Vagrantfile using the VAGRANT_CWD
+environment variable.
+
+
 == Coverage analysis
 
 Tests can be run instrumented with jacoco to produce a coverage report in
@@ -462,7 +474,7 @@ Combined (Unit+Integration) coverage:
 mvn -Dtests.coverage verify jacoco:report
 ---------------------------------------------------------------------------
 
-== Debugging from an IDE
+== Launching and debugging from an IDE
 
 If you want to run elasticsearch from your IDE, the `gradle run` task
 supports a remote debugging option:
@@ -471,6 +483,17 @@ supports a remote debugging option:
 gradle run --debug-jvm
 ---------------------------------------------------------------------------
 
+== Debugging remotely from an IDE
+
+If you want to run Elasticsearch and be able to remotely attach the process
+for debugging purposes from your IDE, can start Elasticsearch using `ES_JAVA_OPTS`:
+
+---------------------------------------------------------------------------
+ES_JAVA_OPTS="-Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=4000,suspend=y" ./bin/elasticsearch
+---------------------------------------------------------------------------
+
+Read your IDE documentation for how to attach a debugger to a JVM process.
+
 == Building with extra plugins
 Additional plugins may be built alongside elasticsearch, where their
 dependency on elasticsearch will be substituted with the local elasticsearch
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 36732215d43..fe6d7b59eb3 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -55,7 +55,7 @@ dependencies {
     runtime 'org.apache.commons:commons-math3:3.2'
 }
 
-compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked,-processing"
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes
 // needs to be added separately otherwise Gradle will quote it and javac will fail
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
diff --git a/build.gradle b/build.gradle
index ba19a993bdc..fd97470ec6c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import java.nio.file.Path
 import org.eclipse.jgit.lib.Repository
 import org.eclipse.jgit.lib.RepositoryBuilder
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
@@ -29,8 +30,9 @@ subprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
+Path rootPath = rootDir.toPath()
 // setup pom license info, but only for artifacts that are part of elasticsearch
-configure(subprojects.findAll { it.path.startsWith(':x-plugins') == false }) {
+configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
 
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
@@ -204,6 +206,14 @@ allprojects {
       }
     }
   }
+
+  task cleanIdeaBuildDir(type: Delete) {
+    delete 'build-idea'
+  }
+  cleanIdeaBuildDir.setGroup("ide")
+  cleanIdeaBuildDir.setDescription("Deletes the IDEA build directory.")
+
+  tasks.cleanIdea.dependsOn(cleanIdeaBuildDir)
 }
 
 idea {
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 0e8c2dc1412..65c2c00dd13 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -23,14 +23,12 @@ apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
 
-// TODO: remove this when upgrading to a version that supports ProgressLogger
-// gradle 2.14 made internal apis unavailable to plugins, and gradle considered
-// ProgressLogger to be an internal api. Until this is made available again,
-// we can't upgrade without losing our nice progress logging
-// NOTE that this check duplicates that in BuildPlugin, but we need to check
-// early here before trying to compile the broken classes in buildSrc
-if (GradleVersion.current() != GradleVersion.version('2.13')) {
-  throw new GradleException('Gradle 2.13 is required to build elasticsearch')
+if (GradleVersion.current() < GradleVersion.version('2.13')) {
+  throw new GradleException('Gradle 2.13+ is required to build elasticsearch')
+}
+
+if (JavaVersion.current() < JavaVersion.VERSION_1_8) {
+  throw new GradleException('Java 1.8 is required to build elasticsearch gradle tools')
 }
 
 if (project == rootProject) {
@@ -96,9 +94,26 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'de.thetaphi:forbiddenapis:2.2'
   compile 'org.apache.rat:apache-rat:0.11'
-  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.0.1'
 }
 
+// Gradle version-specific options (allows build to run with Gradle 2.13 as well as 2.14+/3.+)
+if (GradleVersion.current() == GradleVersion.version("2.13")) {
+  // ProgressLogger(-Factory) classes are part of the public Gradle API
+  sourceSets.main.groovy.srcDir 'src/main/gradle-2.13-groovy'
+
+  dependencies {
+    compile 'ru.vyarus:gradle-animalsniffer-plugin:1.0.1' // last version compatible with Gradle 2.13
+  }
+} else {
+  // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
+  // Use logging dependency instead
+  sourceSets.main.groovy.srcDir 'src/main/gradle-2.14-groovy'
+
+  dependencies {
+    compileOnly "org.gradle:gradle-logging:${GradleVersion.current().getVersion()}"
+    compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
+  }
+}
 
 /*****************************************************************************
  *                         Bootstrap repositories                            *
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java b/buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
similarity index 67%
rename from core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java
rename to buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
index 27c1f074c40..5c02e255a1a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommandRegistry.java
+++ b/buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
@@ -16,16 +16,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-package org.elasticsearch.cluster.routing.allocation.command;
-
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
+package org.elasticsearch.gradle
 
 /**
- * Registry of allocation commands. This is it's own class just to make Guice happy.
+ * Wraps a ProgressLogger so that code in src/main/groovy does not need to
+ * define imports on Gradle 2.13/2.14+ ProgressLoggers
  */
-public class AllocationCommandRegistry extends ParseFieldRegistry<AllocationCommand.Parser<?>> {
-    public AllocationCommandRegistry() {
-        super("allocation_command");
+class ProgressLogger {
+    @Delegate org.gradle.logging.ProgressLogger progressLogger
+
+    ProgressLogger(org.gradle.logging.ProgressLogger progressLogger) {
+        this.progressLogger = progressLogger
     }
 }
diff --git a/buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy b/buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy
new file mode 100644
index 00000000000..290c4d581d6
--- /dev/null
+++ b/buildSrc/src/main/gradle-2.13-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle
+
+import org.gradle.logging.ProgressLoggerFactory
+
+import javax.inject.Inject
+
+/**
+ * Allows to inject a ProgressLoggerFactory to tasks in src/main/groovy
+ * without requiring the corresponding import of ProgressLoggerFactory,
+ * making it compatible with both Gradle 2.13 and 2.14+.
+ */
+trait ProgressLoggerFactoryInjection {
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException()
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java b/buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
similarity index 66%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
rename to buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
index 594f701c79a..2c9fab78b43 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java
+++ b/buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLogger.groovy
@@ -16,18 +16,16 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.gradle
 
 /**
- * Exception thrown whenever there is a problem parsing any of the REST test suite fragment
+ * Wraps a ProgressLogger so that code in src/main/groovy does not need to
+ * define imports on Gradle 2.13/2.14+ ProgressLoggers
  */
-public class ClientYamlTestParseException extends Exception {
+class ProgressLogger {
+    @Delegate org.gradle.internal.logging.progress.ProgressLogger progressLogger
 
-    ClientYamlTestParseException(String message) {
-        super(message);
-    }
-
-    ClientYamlTestParseException(String message, Throwable cause) {
-        super(message, cause);
+    ProgressLogger(org.gradle.internal.logging.progress.ProgressLogger progressLogger) {
+        this.progressLogger = progressLogger
     }
 }
diff --git a/buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy b/buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy
new file mode 100644
index 00000000000..8891d65611a
--- /dev/null
+++ b/buildSrc/src/main/gradle-2.14-groovy/org/elasticsearch/gradle/ProgressLoggerFactoryInjection.groovy
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle
+
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
+
+import javax.inject.Inject
+
+/**
+ * Allows to inject a ProgressLoggerFactory to tasks in src/main/groovy
+ * without requiring the corresponding import of ProgressLoggerFactory,
+ * making it compatible with both Gradle 2.13 and 2.14+.
+ */
+trait ProgressLoggerFactoryInjection {
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException()
+    }
+}
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
index b28e7210ea4..ecb50ce80b5 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
@@ -8,6 +8,7 @@ import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
+import org.elasticsearch.gradle.ProgressLoggerFactoryInjection
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
@@ -19,12 +20,9 @@ import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
-import org.gradle.logging.ProgressLoggerFactory
 import org.gradle.util.ConfigureUtil
 
-import javax.inject.Inject
-
-class RandomizedTestingTask extends DefaultTask {
+class RandomizedTestingTask extends DefaultTask implements ProgressLoggerFactoryInjection {
 
     // TODO: change to "executable" to match gradle test params?
     @Optional
@@ -81,6 +79,7 @@ class RandomizedTestingTask extends DefaultTask {
     String argLine = null
 
     Map<String, Object> systemProperties = new HashMap<>()
+    Map<String, Object> environmentVariables = new HashMap<>()
     PatternFilterable patternSet = new PatternSet()
 
     RandomizedTestingTask() {
@@ -89,11 +88,6 @@ class RandomizedTestingTask extends DefaultTask {
         listenersConfig.listeners.add(new TestReportLogger(logger: logger, config: testLoggingConfig))
     }
 
-    @Inject
-    ProgressLoggerFactory getProgressLoggerFactory() {
-        throw new UnsupportedOperationException();
-    }
-
     void jvmArgs(Iterable<String> arguments) {
         jvmArgs.addAll(arguments)
     }
@@ -106,6 +100,10 @@ class RandomizedTestingTask extends DefaultTask {
         systemProperties.put(property, value)
     }
 
+    void environment(String key, Object value) {
+        environmentVariables.put(key, value)
+    }
+
     void include(String... includes) {
         this.patternSet.include(includes);
     }
@@ -194,7 +192,8 @@ class RandomizedTestingTask extends DefaultTask {
             haltOnFailure: true, // we want to capture when a build failed, but will decide whether to rethrow later
             shuffleOnSlave: shuffleOnSlave,
             leaveTemporary: leaveTemporary,
-            ifNoTests: ifNoTests
+            ifNoTests: ifNoTests,
+            newenvironment: true
         ]
 
         DefaultLogger listener = null
@@ -250,6 +249,9 @@ class RandomizedTestingTask extends DefaultTask {
                 for (Map.Entry<String, Object> prop : systemProperties) {
                     sysproperty key: prop.getKey(), value: prop.getValue().toString()
                 }
+                for (Map.Entry<String, Object> envvar : environmentVariables) {
+                    env key: envvar.getKey(), value: envvar.getValue().toString()
+                }
                 makeListeners()
             }
         } catch (BuildException e) {
diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
index 14f5d476be3..a9786935c56 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
@@ -25,8 +25,7 @@ import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
 import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
 import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
+import org.elasticsearch.gradle.ProgressLogger
 
 import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDurationInSeconds
 import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.ERROR
@@ -52,8 +51,6 @@ import static java.lang.Math.max
  * quick.
  */
 class TestProgressLogger implements AggregatedEventListener {
-    /** Factory to build a progress logger when testing starts */
-    ProgressLoggerFactory factory
     ProgressLogger progressLogger
     int totalSuites
     int totalSlaves
@@ -79,12 +76,15 @@ class TestProgressLogger implements AggregatedEventListener {
     /* Note that we probably overuse volatile here but it isn't hurting us and
       lets us move things around without worying about breaking things. */
 
+    TestProgressLogger(Map args) {
+        progressLogger = new ProgressLogger(args.factory.newOperation(TestProgressLogger))
+        progressLogger.setDescription('Randomized test runner')
+    }
+
     @Subscribe
     void onStart(AggregatedStartEvent e) throws IOException {
         totalSuites = e.suiteCount
         totalSlaves = e.slaveCount
-        progressLogger = factory.newOperation(TestProgressLogger)
-        progressLogger.setDescription('Randomized test runner')
         progressLogger.started()
         progressLogger.progress(
             "Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms")
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 628e59de1a6..b49cb52ee7f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.gradle
 
+import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import nebula.plugin.extraconfigurations.ProvidedBasePlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.GradleException
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.JavaVersion
 import org.gradle.api.Plugin
 import org.gradle.api.Project
@@ -54,6 +56,11 @@ class BuildPlugin implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
+        if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
+              throw new InvalidUserDataException('elasticsearch.standalone-test, '
+                + 'elasticearch.standalone-rest-test, and elasticsearch.build '
+                + 'are mutually exclusive')
+        }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
         // these plugins add lots of info to our jars
@@ -195,18 +202,13 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Runs the given javascript using jjs from the jdk, and returns the output */
     private static String runJavascript(Project project, String javaHome, String script) {
-        File tmpScript = File.createTempFile('es-gradle-tmp', '.js')
-        tmpScript.setText(script, 'UTF-8')
         ByteArrayOutputStream output = new ByteArrayOutputStream()
-        ExecResult result = project.exec {
-            executable = new File(javaHome, 'bin/jjs')
-            args tmpScript.toString()
+        project.exec {
+            executable = new File(javaHome, 'bin/jrunscript')
+            args '-e', script
             standardOutput = output
             errorOutput = new ByteArrayOutputStream()
-            ignoreExitValue = true // we do not fail so we can first cleanup the tmp file
         }
-        java.nio.file.Files.delete(tmpScript.toPath())
-        result.assertNormalExitValue()
         return output.toString('UTF-8').trim()
     }
 
@@ -412,8 +414,10 @@ class BuildPlugin implements Plugin<Project> {
                     // hack until gradle supports java 9's new "--release" arg
                     assert minimumJava == JavaVersion.VERSION_1_8
                     options.compilerArgs << '--release' << '8'
-                    project.sourceCompatibility = null
-                    project.targetCompatibility = null
+                    doFirst{
+                        sourceCompatibility = null
+                        targetCompatibility = null
+                    }
                 }
             }
         }
@@ -509,11 +513,9 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            // System assertions (-esa) are disabled for now because of what looks like a
-            // JDK bug triggered by Groovy on JDK7. We should look at re-enabling system
-            // assertions when we upgrade to a new version of Groovy (currently 2.4.4) or
-            // require JDK8. See https://issues.apache.org/jira/browse/GROOVY-7528.
-            enableSystemAssertions false
+            boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
+            enableSystemAssertions assertionsEnabled
+            enableAssertions assertionsEnabled
 
             testLogging {
                 showNumFailuresAtEnd 25
@@ -554,11 +556,22 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Configures the test task */
     static Task configureTest(Project project) {
-        Task test = project.tasks.getByName('test')
+        RandomizedTestingTask test = project.tasks.getByName('test')
         test.configure(commonTestConfig(project))
         test.configure {
             include '**/*Tests.class'
         }
+
+        // Add a method to create additional unit tests for a project, which will share the same
+        // randomized testing setup, but by default run no tests.
+        project.extensions.add('additionalTest', { String name, Closure config ->
+            RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
+            additionalTest.classpath = test.classpath
+            additionalTest.testClassesDir = test.testClassesDir
+            additionalTest.configure(commonTestConfig(project))
+            additionalTest.configure(config)
+            test.dependsOn(additionalTest)
+        });
         return test
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
index a46a7bda374..66f9f0d4c4e 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -30,6 +30,7 @@ public class DocsTestPlugin extends RestTestPlugin {
 
     @Override
     public void apply(Project project) {
+        project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
         Map<String, String> defaultSubstitutions = [
             /* These match up with the asciidoc syntax for substitutions but
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index f451beeceb8..96b7ac42527 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -139,6 +139,7 @@ class PrecommitTasks {
             configProperties = [
                 suppressions: checkstyleSuppressions
             ]
+            toolVersion = 7.5
         }
         for (String taskName : ['checkstyleMain', 'checkstyleTest']) {
             Task task = project.tasks.findByName(taskName)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
index ca4957f7a6c..8d65f8c0d60 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
@@ -73,8 +73,8 @@ class ClusterConfiguration {
 
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
-            " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
-            " " + System.getProperty('tests.jvm.argline', '')
+        " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
+        " " + System.getProperty('tests.jvm.argline', '')
 
     /**
      * A closure to call which returns the unicast host to connect to for cluster formation.
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 74cae08298b..3fc622ef5aa 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -39,6 +39,7 @@ import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.Exec
 
 import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
 /**
  * A helper for creating tasks to build a cluster that is used by a task, and tear down the cluster when the task is finished.
@@ -91,6 +92,8 @@ class ClusterFormationTasks {
                 configureBwcPluginDependency("${task.name}_elasticsearchBwcPlugins", project, entry.getValue(),
                         project.configurations.elasticsearchBwcPlugins, config.bwcVersion)
             }
+            project.configurations.elasticsearchBwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
+            project.configurations.elasticsearchBwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
         }
         for (int i = 0; i < config.numNodes; i++) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
@@ -268,12 +271,12 @@ class ClusterFormationTasks {
     static Task configureWriteConfigTask(String name, Project project, Task setup, NodeInfo node, NodeInfo seedNode) {
         Map esConfig = [
                 'cluster.name'                 : node.clusterName,
+                'node.name'                    : "node-" + node.nodeNum,
                 'pidfile'                      : node.pidFile,
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test',
-                'repositories.url.allowed_urls': 'http://snapshot.test*'
+                'node.attr.testattr'           : 'test'
         ]
         // we set min master nodes to the total number of nodes in the cluster and
         // basically skip initial state recovery to allow the cluster to form using a realistic master election
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
index 1cca2c5aa49..1c0aec1bc00 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/MessyTestPlugin.groovy
@@ -48,7 +48,7 @@ class MessyTestPlugin extends StandaloneTestPlugin {
     }
 
     private static addPluginResources(Project project, Project pluginProject) {
-        String outputDir = "generated-resources/${pluginProject.name}"
+        String outputDir = "${project.buildDir}/generated-resources/${pluginProject.name}"
         String taskName = ClusterFormationTasks.pluginTaskName("copy", pluginProject.name, "Metadata")
         Copy copyPluginMetadata = project.tasks.create(taskName, Copy.class)
         copyPluginMetadata.into(outputDir)
@@ -57,7 +57,7 @@ class MessyTestPlugin extends StandaloneTestPlugin {
         project.sourceSets.test.output.dir(outputDir, builtBy: taskName)
 
         // add each generated dir to the test classpath in IDEs
-        //project.eclipse.classpath.sourceSets = [project.sourceSets.test]
         project.idea.module.singleEntryLibraries= ['TEST': [project.file(outputDir)]]
+        // Eclipse doesn't need this because it gets the entire module as a dependency
     }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index a9473cc28d2..73f32961fb3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -151,6 +151,9 @@ class NodeInfo {
         args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
+        if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) {
+            esJavaOpts += " -ea -esa"
+        }
         env.put('ES_JAVA_OPTS', esJavaOpts)
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
             if (property.key.startsWith('tests.es.')) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
index dc9aa769388..47a559efccb 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy
@@ -18,15 +18,29 @@
  */
 package org.elasticsearch.gradle.test
 
+import org.elasticsearch.gradle.BuildPlugin
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 
-/** A plugin to add rest integration tests. Used for qa projects.  */
+/**
+ * Adds support for starting an Elasticsearch cluster before running integration
+ * tests. Used in conjunction with {@link StandaloneRestTestPlugin} for qa
+ * projects and in conjunction with {@link BuildPlugin} for testing the rest
+ * client.
+ */
 public class RestTestPlugin implements Plugin<Project> {
+    List REQUIRED_PLUGINS = [
+        'elasticsearch.build',
+        'elasticsearch.standalone-rest-test']
 
     @Override
     public void apply(Project project) {
-        project.pluginManager.apply(StandaloneTestBasePlugin)
+        if (false == REQUIRED_PLUGINS.any {project.pluginManager.hasPlugin(it)}) {
+            throw new InvalidUserDataException('elasticsearch.rest-test '
+                + 'requires either elasticsearch.build or '
+                + 'elasticsearch.standalone-rest-test')
+        }
 
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.cluster.distribution = 'zip' // rest tests should run with the real zip
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
similarity index 77%
rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
index af2b20e4abf..c48dc890ab0 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
@@ -24,15 +24,26 @@ import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 
-/** Configures the build to have a rest integration test.  */
-public class StandaloneTestBasePlugin implements Plugin<Project> {
+/**
+ * Configures the build to compile tests against Elasticsearch's test framework
+ * and run REST tests. Use BuildPlugin if you want to build main code as well
+ * as tests.
+ */
+public class StandaloneRestTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
+        if (project.pluginManager.hasPlugin('elasticsearch.build')) {
+            throw new InvalidUserDataException('elasticsearch.standalone-test '
+                + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
+                + 'are mutually exclusive')
+        }
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(RandomizedTestingPlugin)
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
index fefd08fe4e5..de52d75c600 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
@@ -25,12 +25,15 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.JavaBasePlugin
 
-/** A plugin to add tests only. Used for QA tests that run arbitrary unit tests. */
+/**
+ * Configures the build to compile against Elasticsearch's test framework and
+ * run integration and unit tests. Use BuildPlugin if you want to build main
+ * code as well as tests. */
 public class StandaloneTestPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
-        project.pluginManager.apply(StandaloneTestBasePlugin)
+        project.pluginManager.apply(StandaloneRestTestPlugin)
 
         Map testOptions = [
             name: 'test',
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy
new file mode 100644
index 00000000000..7e370fd69e2
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test
+
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.ProjectDependency
+import org.gradle.api.tasks.Copy
+
+/**
+ * A plugin to run tests that depend on other plugins or modules.
+ *
+ * This plugin will add the plugin-metadata and properties files for each
+ * dependency to the test source set.
+ */
+class TestWithDependenciesPlugin implements Plugin<Project> {
+
+    @Override
+    void apply(Project project) {
+        if (project.isEclipse) {
+            /* The changes this plugin makes both break and aren't needed by
+             * Eclipse. This is because Eclipse flattens main and test
+             * dependencies into a single dependency. Because Eclipse is
+             * "special".... */
+            return
+        }
+
+        project.configurations.testCompile.dependencies.all { Dependency dep ->
+            // this closure is run every time a compile dependency is added
+            if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) {
+                project.gradle.projectsEvaluated {
+                    addPluginResources(project, dep.dependencyProject)
+                }
+            }
+        }
+    }
+
+    private static addPluginResources(Project project, Project pluginProject) {
+        String outputDir = "${project.buildDir}/generated-resources/${pluginProject.name}"
+        String taskName = ClusterFormationTasks.pluginTaskName("copy", pluginProject.name, "Metadata")
+        Copy copyPluginMetadata = project.tasks.create(taskName, Copy.class)
+        copyPluginMetadata.into(outputDir)
+        copyPluginMetadata.from(pluginProject.tasks.pluginProperties)
+        copyPluginMetadata.from(pluginProject.file('src/main/plugin-metadata'))
+        project.sourceSets.test.output.dir(outputDir, builtBy: taskName)
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
index 3f980c57a49..85fd433bc77 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
@@ -20,10 +20,9 @@ package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
 import groovy.transform.PackageScope
+import org.elasticsearch.gradle.ProgressLogger
 import org.gradle.api.GradleScriptException
 import org.gradle.api.logging.Logger
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
 
 import java.util.regex.Matcher
 
@@ -49,7 +48,7 @@ public class TapLoggerOutputStream extends LoggingOutputStream {
 
     TapLoggerOutputStream(Map args) {
         logger = args.logger
-        progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
+        progressLogger = new ProgressLogger(args.factory.newOperation(VagrantLoggerOutputStream))
         progressLogger.setDescription("TAP output for `${args.command}`")
     }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
index ecba08d7d4c..cd4d4bf87a5 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
@@ -19,17 +19,15 @@
 package org.elasticsearch.gradle.vagrant
 
 import org.apache.commons.io.output.TeeOutputStream
+import org.elasticsearch.gradle.ProgressLoggerFactoryInjection
 import org.elasticsearch.gradle.LoggedExec
 import org.gradle.api.tasks.Input
-import org.gradle.logging.ProgressLoggerFactory
-
-import javax.inject.Inject
 
 /**
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output
  * formatter and defaults to `vagrant` as first part of commandLine.
  */
-public class VagrantCommandTask extends LoggedExec {
+public class VagrantCommandTask extends LoggedExec implements ProgressLoggerFactoryInjection {
 
     @Input
     String boxName
@@ -57,9 +55,4 @@ public class VagrantCommandTask extends LoggedExec {
               stuff starts with ==> $box */
             squashedPrefix: "==> $boxName: ")
     }
-
-    @Inject
-    ProgressLoggerFactory getProgressLoggerFactory() {
-        throw new UnsupportedOperationException();
-    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
index 331a638b5ca..de6c5a36db9 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
@@ -19,9 +19,7 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
-import org.gradle.api.logging.Logger
-import org.gradle.logging.ProgressLogger
-import org.gradle.logging.ProgressLoggerFactory
+import org.elasticsearch.gradle.ProgressLogger
 
 /**
  * Adapts an OutputStream being written to by vagrant into a ProcessLogger. It
@@ -55,7 +53,7 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream {
     private String heading = ''
 
     VagrantLoggerOutputStream(Map args) {
-        progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
+        progressLogger = new ProgressLogger(args.factory.newOperation(VagrantLoggerOutputStream))
         progressLogger.setDescription("Vagrant output for `$args.command`")
         squashedPrefix = args.squashedPrefix
     }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
index a5bb054a8b6..0b7a105e8ab 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -404,10 +404,6 @@ class VagrantTestPlugin implements Plugin<Project> {
                 args 'halt', box
             }
             stop.dependsOn(halt)
-            if (project.extensions.esvagrant.boxes.contains(box) == false) {
-                // we only need a halt task if this box was not specified
-                continue;
-            }
 
             Task update = project.tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
                 boxName box
@@ -435,6 +431,11 @@ class VagrantTestPlugin implements Plugin<Project> {
                 dependsOn update
             }
 
+            if (project.extensions.esvagrant.boxes.contains(box) == false) {
+                // we d'ont need tests tasks if this box was not specified
+                continue;
+            }
+
             Task smoke = project.tasks.create("vagrant${boxTask}#smoketest", Exec) {
                 environment vagrantEnvVars
                 dependsOn up
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties
new file mode 100644
index 00000000000..2daf4dc27c0
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.test.StandaloneRestTestPlugin
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties
new file mode 100644
index 00000000000..bcb374a85c6
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.test.TestWithDependenciesPlugin
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index eba6dbfc819..a800411cc63 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -10,9 +10,6 @@
   <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessLexer\.java" checks="." />
   <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessParser(|BaseVisitor|Visitor)\.java" checks="." />
 
-  <!-- ThrowableProxy is a forked copy from Log4j to hack around a bug; this can be removed when the hack is removed -->
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]logging[/\\]log4j[/\\]core[/\\]impl[/\\]ThrowableProxy.java" checks="RegexpSinglelineJava" />
-
   <!-- Hopefully temporary suppression of LineLength on files that don't pass it. We should remove these when we the
     files start to pass. -->
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]BlendedTermQuery.java" checks="LineLength" />
@@ -29,7 +26,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]delete[/\\]TransportDeleteRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]get[/\\]GetRepositoriesRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]get[/\\]TransportGetRepositoriesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]PutRepositoryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]PutRepositoryRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]TransportPutRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]verify[/\\]TransportVerifyRepositoryAction.java" checks="LineLength" />
@@ -56,7 +52,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]TransportClusterStatsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]PendingClusterTasksResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]tasks[/\\]TransportPendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]validate[/\\]template[/\\]RenderSearchTemplateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]validate[/\\]template[/\\]RenderSearchTemplateRequestBuilder.java" checks="LineLength" />
@@ -135,15 +130,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportBulkAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]TransportShardBulkAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]delete[/\\]DeleteRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]delete[/\\]TransportDeleteAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]explain[/\\]TransportExplainAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]GetRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]TransportShardMultiGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]IndexRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]IndexRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]index[/\\]TransportIndexAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]indexedscripts[/\\]delete[/\\]DeleteIndexedScriptRequestBuilder.java" checks="LineLength" />
@@ -165,19 +157,16 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseExecutionException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]ShardSearchFailure.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportClearScrollAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportMultiSearchAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]suggest[/\\]SuggestResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ActionFilter.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]AutoCreateIndex.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]DelegatingActionListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptions.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ToXContentToBytes.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastOperationRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]TransportBroadcastAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]node[/\\]TransportBroadcastByNodeAction.java" checks="LineLength" />
@@ -195,7 +184,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]instance[/\\]InstanceShardOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]instance[/\\]TransportInstanceSingleOperationAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]SingleShardOperationRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]SingleShardRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]shard[/\\]TransportSingleShardAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]MultiTermVectorsRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]MultiTermVectorsRequestBuilder.java" checks="LineLength" />
@@ -212,7 +200,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]update[/\\]UpdateRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Bootstrap.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNAKernel32Library.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JNANatives.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JVMCheck.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JarHell.java" checks="LineLength" />
@@ -234,7 +221,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]AutoExpandReplicas.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]IndexNameExpressionResolver.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MappingMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataCreateIndexService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataIndexStateService.java" checks="LineLength" />
@@ -260,10 +246,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]decider[/\\]AllocationDeciders.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]service[/\\]InternalClusterService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Base64.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Booleans.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Numbers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]fs[/\\]FsBlobStore.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]url[/\\]URLBlobStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]BytesArray.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]PagedBytesReference.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cache[/\\]Cache.java" checks="LineLength" />
@@ -315,7 +299,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]AlreadyExpiredException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]CompositeIndexEventListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexSettings.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicyConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisRegistry.java" checks="LineLength" />
@@ -338,7 +321,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]FloatValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]LongValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]GlobalOrdinalsBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]InternalGlobalOrdinalsIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]MultiOrdinals.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]OrdinalsBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]SinglePackedOrdinals.java" checks="LineLength" />
@@ -349,16 +331,13 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]ParentChildIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]SortedNumericDVIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]SortedSetDVOrdinalsIndexFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]get[/\\]GetResult.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]get[/\\]ShardGetService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMappers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeLookup.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MappedFieldType.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MapperService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]Mapping.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]MetadataFieldMapper.java" checks="LineLength" />
@@ -392,7 +371,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryBuilders.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryValidationException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]InnerHitsQueryParserHelper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]QueryParsers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MatchQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MultiMatchQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]geo[/\\]IndexedGeoBoundingBoxQuery.java" checks="LineLength" />
@@ -406,8 +384,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]StoreRecovery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]TranslogRecoveryPerformer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardSnapshots.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStoreConfig.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]LegacyVerification.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]Store.java" checks="LineLength" />
@@ -432,14 +408,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]PeerRecoverySourceService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryState.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]StartRecoveryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]TransportNodesListShardStoreMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]ttl[/\\]IndicesTTLService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]RepositoriesService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]Repository.java" checks="LineLength" />
@@ -450,10 +424,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]ChecksumBlobStoreFormat.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestThreadPoolAction.java" checks="LineLength" />
@@ -475,18 +446,12 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]FiltersParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]InternalFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]geogrid[/\\]GeoHashGridAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]GlobalAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]InternalGlobal.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]histogram[/\\]HistogramAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]InternalMissing.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]MissingAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]nested[/\\]InternalReverseNested.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]nested[/\\]ReverseNestedAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]RangeAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedBytesHashSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedMapSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedNumericSamplerAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedOrdinalsSamplerAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]InternalSampler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]SamplerAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]GlobalOrdinalsSignificantTermsAggregator.java" checks="LineLength" />
@@ -498,7 +463,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]ScriptHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]SignificanceHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]AbstractTermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]GlobalOrdinalsStringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]InternalOrder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]LongTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]StringTermsAggregator.java" checks="LineLength" />
@@ -515,7 +479,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]ScriptedMetricAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]ExtendedStatsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]tophits[/\\]TopHitsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]bucketscript[/\\]BucketScriptPipelineAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]AggregationPath.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]ValuesSourceParser.java" checks="LineLength" />
@@ -548,7 +511,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]threadpool[/\\]ThreadPool.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]queries[/\\]BlendedTermQueryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]search[/\\]postingshighlight[/\\]CustomPostingsHighlighterTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ESExceptionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]NamingConventionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]VersionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]RejectionActionIT.java" checks="LineLength" />
@@ -576,8 +538,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineResponseTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]WriteableIngestDocumentTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchRequestBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptionsTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptionsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]TransportActionFilterChainTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]WaitActiveShardCountIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]node[/\\]TransportBroadcastByNodeActionTests.java" checks="LineLength" />
@@ -591,9 +551,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]termvectors[/\\]TermVectorsUnitTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]aliases[/\\]IndexAliasesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JarHellTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bwcompat[/\\]BasicAnalysisBackwardCompatibilityIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bwcompat[/\\]BasicBackwardsCompatibilityIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bwcompat[/\\]GetIndexBackwardsCompatibilityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bwcompat[/\\]RestoreBackwardsCompatIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]AbstractClientHeadersTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterHealthIT.java" checks="LineLength" />
@@ -669,7 +626,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]settings[/\\]ClusterSettingsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]shards[/\\]ClusterSearchShardsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]structure[/\\]RoutingIteratorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]BooleansTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreContainerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]FsBlobStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]breaker[/\\]MemoryCircuitBreakerTests.java" checks="LineLength" />
@@ -704,8 +660,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]PrimaryShardAllocatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]PriorityComparatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]QuorumGatewayIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoverAfterNodesIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoveryBackwardsCompatibilityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]RecoveryFromGatewayIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReplicaShardAllocatorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReusePeerRecoverySharedTest.java" checks="LineLength" />
@@ -789,7 +743,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]HasChildQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MoreLikeThisQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MultiMatchQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RandomQueryBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanMultiTermQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanNotQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]QueryInnerHitsTests.java" checks="LineLength" />
@@ -801,7 +754,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]LongNestedSortingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]NestedSortingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexShardTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]IndexingOperationListenerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardPathTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]similarity[/\\]SimilarityTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]FileInfoTests.java" checks="LineLength" />
@@ -819,7 +771,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesOptionsIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]AnalyzeActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]HunspellServiceIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]exists[/\\]indices[/\\]IndicesExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]exists[/\\]types[/\\]TypesExistsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]FlushIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]SyncedFlushSingleNodeTests.java" checks="LineLength" />
@@ -924,7 +875,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]DedicatedClusterSnapshotRestoreIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RepositoriesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SharedClusterSnapshotRestoreIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotBackwardsCompatibilityIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotUtilsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESBlobStoreRepositoryIntegTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]geo[/\\]RandomShapeGenerator.java" checks="LineLength" />
@@ -935,9 +885,8 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]validate[/\\]SimpleValidateQueryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]versioning[/\\]SimpleVersioningIT.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngineService.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionSearchScript.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]IndexedExpressionTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" />
@@ -994,13 +943,8 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]MockBigArrays.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]script[/\\]NativeSignificanceScoreScriptWithParams.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]BackgroundIndexer.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CompositeTestCluster.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CorruptionUtils.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESBackcompatTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESIntegTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESSingleNodeTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ESTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]ExternalNode.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]IndexSettingsModule.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]InternalTestCluster.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]MockIndexEventListener.java" checks="LineLength" />
@@ -1012,7 +956,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]AssertingSearcher.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 44835f7227c..ddbf87c4a71 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,10 +1,11 @@
+# When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 elasticsearch     = 6.0.0-alpha1
-lucene            = 6.4.0-snapshot-ec38570
+lucene            = 6.4.0
 
 # optional dependencies
 spatial4j         = 0.6
 jts               = 1.13
-jackson           = 2.8.1
+jackson           = 2.8.6
 snakeyaml         = 1.15
 # When updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.7
@@ -15,10 +16,16 @@ jna               = 4.2.2
 randomizedrunner  = 2.4.0
 junit             = 4.11
 httpclient        = 4.5.2
+# When updating httpcore, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 httpcore          = 4.4.5
+# When updating httpasyncclient, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+httpasyncclient   = 4.1.2
 commonslogging    = 1.1.3
 commonscodec      = 1.10
 hamcrest          = 1.3
 securemock        = 1.2
+# When updating mocksocket, please also update core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+mocksocket        = 1.1
+
 # benchmark dependencies
 jmh               = 1.17.3
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
index 214a75d12cc..e9cde26e6c8 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/ops/bulk/BulkBenchmarkTask.java
@@ -95,7 +95,7 @@ public class BulkBenchmarkTask implements BenchmarkTask {
         private final BlockingQueue<List<String>> bulkQueue;
         private final int bulkSize;
 
-        public LoadGenerator(Path bulkDataFile, BlockingQueue<List<String>> bulkQueue, int bulkSize) {
+        LoadGenerator(Path bulkDataFile, BlockingQueue<List<String>> bulkQueue, int bulkSize) {
             this.bulkDataFile = bulkDataFile;
             this.bulkQueue = bulkQueue;
             this.bulkSize = bulkSize;
@@ -143,7 +143,7 @@ public class BulkBenchmarkTask implements BenchmarkTask {
         private final BulkRequestExecutor bulkRequestExecutor;
         private final SampleRecorder sampleRecorder;
 
-        public BulkIndexer(BlockingQueue<List<String>> bulkData, int warmupIterations, int measurementIterations,
+        BulkIndexer(BlockingQueue<List<String>> bulkData, int warmupIterations, int measurementIterations,
                            SampleRecorder sampleRecorder, BulkRequestExecutor bulkRequestExecutor) {
             this.bulkData = bulkData;
             this.warmupIterations = warmupIterations;
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
index b342d93fba5..9210526e7c8 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/rest/RestClientBenchmark.java
@@ -73,7 +73,7 @@ public final class RestClientBenchmark extends AbstractBenchmark<RestClient> {
         private final RestClient client;
         private final String actionMetaData;
 
-        public RestBulkRequestExecutor(RestClient client, String index, String type) {
+        RestBulkRequestExecutor(RestClient client, String index, String type) {
             this.client = client;
             this.actionMetaData = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
         }
diff --git a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
index 6d6e5ade827..d2aee2251a6 100644
--- a/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
+++ b/client/benchmark/src/main/java/org/elasticsearch/client/benchmark/transport/TransportClientBenchmark.java
@@ -28,6 +28,7 @@ import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugin.noop.NoopPlugin;
 import org.elasticsearch.plugin.noop.action.bulk.NoopBulkAction;
@@ -70,7 +71,7 @@ public final class TransportClientBenchmark extends AbstractBenchmark<TransportC
         private final String indexName;
         private final String typeName;
 
-        public TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
+        TransportBulkRequestExecutor(TransportClient client, String indexName, String typeName) {
             this.client = client;
             this.indexName = indexName;
             this.typeName = typeName;
@@ -80,7 +81,7 @@ public final class TransportClientBenchmark extends AbstractBenchmark<TransportC
         public boolean bulkIndex(List<String> bulkData) {
             NoopBulkRequestBuilder builder = NoopBulkAction.INSTANCE.newRequestBuilder(client);
             for (String bulkItem : bulkData) {
-                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8)));
+                builder.add(new IndexRequest(indexName, typeName).source(bulkItem.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
             }
             BulkResponse bulkResponse;
             try {
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
index ac45f20dc25..e8ed27715c1 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/NoopPlugin.java
@@ -23,15 +23,23 @@ import org.elasticsearch.plugin.noop.action.bulk.RestNoopBulkAction;
 import org.elasticsearch.plugin.noop.action.bulk.TransportNoopBulkAction;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugin.noop.action.search.NoopSearchAction;
 import org.elasticsearch.plugin.noop.action.search.RestNoopSearchAction;
 import org.elasticsearch.plugin.noop.action.search.TransportNoopSearchAction;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 public class NoopPlugin extends Plugin implements ActionPlugin {
     @Override
@@ -43,7 +51,11 @@ public class NoopPlugin extends Plugin implements ActionPlugin {
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestNoopBulkAction.class, RestNoopSearchAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestNoopBulkAction(settings, restController),
+                new RestNoopSearchAction(settings, restController));
     }
 }
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
index ceaf9f8cc9d..1034e722e87 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/NoopBulkRequestBuilder.java
@@ -33,6 +33,7 @@ import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class NoopBulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkResponse, NoopBulkRequestBuilder>
         implements WriteRequestBuilder<NoopBulkRequestBuilder> {
@@ -95,17 +96,17 @@ public class NoopBulkRequestBuilder extends ActionRequestBuilder<BulkRequest, Bu
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length) throws Exception {
-        request.add(data, from, length, null, null);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception {
+        request.add(data, from, length, null, null, xContentType);
         return this;
     }
 
     /**
      * Adds a framed data in binary format
      */
-    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType)
-        throws Exception {
-        request.add(data, from, length, defaultIndex, defaultType);
+    public NoopBulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                                      XContentType xContentType) throws Exception {
+        request.add(data, from, length, defaultIndex, defaultType, xContentType);
         return this;
     }
 
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
index 06082ed7d29..d8a5e4fa0ea 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java
@@ -18,8 +18,8 @@
  */
 package org.elasticsearch.plugin.noop.action.bulk;
 
-import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
@@ -28,7 +28,6 @@ import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
@@ -47,7 +46,6 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestNoopBulkAction extends BaseRestHandler {
-    @Inject
     public RestNoopBulkAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -75,7 +73,8 @@ public class RestNoopBulkAction extends BaseRestHandler {
         }
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
-        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true);
+        bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, null, defaultPipeline, null, true,
+            request.getXContentType());
 
         // short circuit the call to the transport layer
         return channel -> {
@@ -91,7 +90,7 @@ public class RestNoopBulkAction extends BaseRestHandler {
         private final RestRequest request;
 
 
-        public BulkRestBuilderListener(RestChannel channel, RestRequest request) {
+        BulkRestBuilderListener(RestChannel channel, RestRequest request) {
             super(channel);
             this.request = request;
         }
@@ -103,9 +102,7 @@ public class RestNoopBulkAction extends BaseRestHandler {
             builder.field(Fields.ERRORS, false);
             builder.startArray(Fields.ITEMS);
             for (int idx = 0; idx < bulkRequest.numberOfActions(); idx++) {
-                builder.startObject();
                 ITEM_RESPONSE.toXContent(builder, request);
-                builder.endObject();
             }
             builder.endArray();
             builder.endObject();
diff --git a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
index 9bcde4ca399..48a453c3725 100644
--- a/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
+++ b/client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/RestNoopSearchAction.java
@@ -20,7 +20,6 @@ package org.elasticsearch.plugin.noop.action.search;
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -33,8 +32,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestNoopSearchAction extends BaseRestHandler {
-
-    @Inject
     public RestNoopSearchAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_noop_search", this);
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
new file mode 100644
index 00000000000..162e8608d44
--- /dev/null
+++ b/client/rest-high-level/build.gradle
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.rest-test'
+
+group = 'org.elasticsearch.client'
+
+dependencies {
+  compile "org.elasticsearch:elasticsearch:${version}"
+  compile "org.elasticsearch.client:rest:${version}"
+
+  testCompile "org.elasticsearch.client:test:${version}"
+  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testCompile "junit:junit:${versions.junit}"
+  testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+}
+
+dependencyLicenses {
+  // Don't check licenses for dependency that are part of the elasticsearch project
+  // But any other dependency should have its license/notice/sha1
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false
+  }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
similarity index 50%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
rename to client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
index 698a9dfc364..58ecc5f9c2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
+++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
@@ -17,33 +17,37 @@
  * under the License.
  */
 
-package org.elasticsearch.painless.antlr;
+package org.elasticsearch.client;
 
-import org.antlr.v4.runtime.Token;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+
+import java.io.IOException;
+import java.util.Objects;
 
 /**
- * Utility to figure out if a {@code /} is division or the start of a regex literal.
+ * High level REST client that wraps an instance of the low level {@link RestClient} and allows to build requests and read responses.
+ * The provided {@link RestClient} is externally built and closed.
  */
-public class SlashStrategy {
-    public static boolean slashIsRegex(PainlessLexer lexer) {
-        EnhancedPainlessLexer realLexer = (EnhancedPainlessLexer) lexer;
-        Token lastToken = realLexer.getPreviousToken();
-        if (lastToken == null) {
+public final class RestHighLevelClient {
+
+    private static final Log logger = LogFactory.getLog(RestHighLevelClient.class);
+
+    private final RestClient client;
+
+    public RestHighLevelClient(RestClient client) {
+        this.client = Objects.requireNonNull(client);
+    }
+
+    public boolean ping(Header... headers) {
+        try {
+            client.performRequest("HEAD", "/", headers);
             return true;
-        }
-        switch (lastToken.getType()) {
-        case PainlessLexer.RBRACE:
-        case PainlessLexer.RP:
-        case PainlessLexer.OCTAL:
-        case PainlessLexer.HEX:
-        case PainlessLexer.INTEGER:
-        case PainlessLexer.DECIMAL:
-        case PainlessLexer.ID:
-        case PainlessLexer.DOTINTEGER:
-        case PainlessLexer.DOTID:
+        } catch(IOException exception) {
             return false;
-        default:
-            return true;
         }
     }
+
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
similarity index 53%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
index f456c32d092..bc12b1433d7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java
@@ -16,23 +16,33 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
 
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
+package org.elasticsearch.client;
+
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
 
 import java.io.IOException;
 
-/**
- * Parser for match assert sections
- */
-public class MatchParser implements ClientYamlTestFragmentParser<MatchAssertion> {
+public abstract class ESRestHighLevelClientTestCase extends ESRestTestCase {
 
-    @Override
-    public MatchAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    private static RestHighLevelClient restHighLevelClient;
+
+    @Before
+    public void initHighLevelClient() throws IOException {
+        super.initClient();
+        if (restHighLevelClient == null) {
+            restHighLevelClient = new RestHighLevelClient(client());
+        }
+    }
+
+    @AfterClass
+    public static void cleanupClient() throws IOException {
+        restHighLevelClient = null;
+    }
+
+    protected static RestHighLevelClient highLevelClient() {
+        return restHighLevelClient;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MainActionIT.java
similarity index 71%
rename from core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java
rename to client/rest-high-level/src/test/java/org/elasticsearch/client/MainActionIT.java
index a7e61143893..717ab7a44f3 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServerAdapter.java
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MainActionIT.java
@@ -17,14 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.http;
+package org.elasticsearch.client;
 
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestRequest;
-
-public interface HttpServerAdapter {
-
-    void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext context);
+public class MainActionIT extends ESRestHighLevelClientTestCase {
 
+    public void testPing() {
+        assertTrue(highLevelClient().ping());
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
new file mode 100644
index 00000000000..7d513e48998
--- /dev/null
+++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+import org.mockito.ArgumentMatcher;
+import org.mockito.internal.matchers.ArrayEquals;
+import org.mockito.internal.matchers.VarargMatcher;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RestHighLevelClientTests extends ESTestCase {
+
+    private RestClient restClient;
+    private RestHighLevelClient restHighLevelClient;
+
+    @Before
+    public void initClient() throws IOException {
+        restClient = mock(RestClient.class);
+        restHighLevelClient = new RestHighLevelClient(restClient);
+    }
+
+    public void testPing() throws IOException {
+        assertTrue(restHighLevelClient.ping());
+        verify(restClient).performRequest(eq("HEAD"), eq("/"), argThat(new HeadersVarargMatcher()));
+    }
+
+    public void testPingFailure() throws IOException {
+        when(restClient.performRequest(any(), any())).thenThrow(new IllegalStateException());
+        expectThrows(IllegalStateException.class, () -> restHighLevelClient.ping());
+    }
+
+    public void testPingFailed() throws IOException {
+        when(restClient.performRequest(any(), any())).thenThrow(new SocketTimeoutException());
+        assertFalse(restHighLevelClient.ping());
+    }
+
+    public void testPingWithHeaders() throws IOException {
+        Header[] headers = RestClientTestUtil.randomHeaders(random(), "Header");
+        assertTrue(restHighLevelClient.ping(headers));
+        verify(restClient).performRequest(eq("HEAD"), eq("/"), argThat(new HeadersVarargMatcher(headers)));
+    }
+
+    private class HeadersVarargMatcher extends ArgumentMatcher<Header[]> implements VarargMatcher {
+        private Header[] expectedHeaders;
+
+        HeadersVarargMatcher(Header... expectedHeaders) {
+            this.expectedHeaders = expectedHeaders;
+        }
+
+        @Override
+        public boolean matches(Object varargArgument) {
+            if (varargArgument instanceof Header[]) {
+                Header[] actualHeaders = (Header[]) varargArgument;
+                return new ArrayEquals(expectedHeaders).matches(actualHeaders);
+            }
+            return false;
+        }
+    }
+}
diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index 1c92013da97..d5d9c9cfbb5 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -33,7 +33,7 @@ group = 'org.elasticsearch.client'
 dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-  compile "org.apache.httpcomponents:httpasyncclient:4.1.2"
+  compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
   compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -43,6 +43,7 @@ dependencies {
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
diff --git a/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java b/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java
index a5e5b39bed5..528fb9a7fc8 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/HttpAsyncResponseConsumerFactory.java
@@ -53,7 +53,7 @@ interface HttpAsyncResponseConsumerFactory {
 
         private final int bufferLimit;
 
-        public HeapBufferedResponseConsumerFactory(int bufferLimitBytes) {
+        HeapBufferedResponseConsumerFactory(int bufferLimitBytes) {
             this.bufferLimit = bufferLimitBytes;
         }
 
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
index 89c3309dbbd..f808c36d60a 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -25,6 +25,7 @@ import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpResponse;
+import org.apache.http.client.AuthCache;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
@@ -34,8 +35,11 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.concurrent.FutureCallback;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.nio.client.methods.HttpAsyncMethods;
 import org.apache.http.nio.protocol.HttpAsyncRequestProducer;
@@ -49,6 +53,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -91,7 +96,7 @@ public class RestClient implements Closeable {
     private final long maxRetryTimeoutMillis;
     private final String pathPrefix;
     private final AtomicInteger lastHostIndex = new AtomicInteger(0);
-    private volatile Set<HttpHost> hosts;
+    private volatile HostTuple<Set<HttpHost>> hostTuple;
     private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
     private final FailureListener failureListener;
 
@@ -121,11 +126,13 @@ public class RestClient implements Closeable {
             throw new IllegalArgumentException("hosts must not be null nor empty");
         }
         Set<HttpHost> httpHosts = new HashSet<>();
+        AuthCache authCache = new BasicAuthCache();
         for (HttpHost host : hosts) {
             Objects.requireNonNull(host, "host cannot be null");
             httpHosts.add(host);
+            authCache.put(host, new BasicScheme());
         }
-        this.hosts = Collections.unmodifiableSet(httpHosts);
+        this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(httpHosts), authCache);
         this.blacklist.clear();
     }
 
@@ -282,29 +289,61 @@ public class RestClient implements Closeable {
     public void performRequestAsync(String method, String endpoint, Map<String, String> params,
                                     HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
                                     ResponseListener responseListener, Header... headers) {
-        URI uri = buildUri(pathPrefix, endpoint, params);
+        Objects.requireNonNull(params, "params must not be null");
+        Map<String, String> requestParams = new HashMap<>(params);
+        //ignore is a special parameter supported by the clients, shouldn't be sent to es
+        String ignoreString = requestParams.remove("ignore");
+        Set<Integer> ignoreErrorCodes;
+        if (ignoreString == null) {
+            if (HttpHead.METHOD_NAME.equals(method)) {
+                //404 never causes error if returned for a HEAD request
+                ignoreErrorCodes = Collections.singleton(404);
+            } else {
+                ignoreErrorCodes = Collections.emptySet();
+            }
+        } else {
+            String[] ignoresArray = ignoreString.split(",");
+            ignoreErrorCodes = new HashSet<>();
+            if (HttpHead.METHOD_NAME.equals(method)) {
+                //404 never causes error if returned for a HEAD request
+                ignoreErrorCodes.add(404);
+            }
+            for (String ignoreCode : ignoresArray) {
+                try {
+                    ignoreErrorCodes.add(Integer.valueOf(ignoreCode));
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead", e);
+                }
+            }
+        }
+        URI uri = buildUri(pathPrefix, endpoint, requestParams);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
         setHeaders(request, headers);
         FailureTrackingResponseListener failureTrackingResponseListener = new FailureTrackingResponseListener(responseListener);
         long startTime = System.nanoTime();
-        performRequestAsync(startTime, nextHost().iterator(), request, httpAsyncResponseConsumerFactory, failureTrackingResponseListener);
+        performRequestAsync(startTime, nextHost(), request, ignoreErrorCodes, httpAsyncResponseConsumerFactory,
+                            failureTrackingResponseListener);
     }
 
-    private void performRequestAsync(final long startTime, final Iterator<HttpHost> hosts, final HttpRequestBase request,
+    private void performRequestAsync(final long startTime, final HostTuple<Iterator<HttpHost>> hostTuple, final HttpRequestBase request,
+                                     final Set<Integer> ignoreErrorCodes,
                                      final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory,
                                      final FailureTrackingResponseListener listener) {
-        final HttpHost host = hosts.next();
+        final HttpHost host = hostTuple.hosts.next();
         //we stream the request body if the entity allows for it
-        HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request);
-        HttpAsyncResponseConsumer<HttpResponse> asyncResponseConsumer = httpAsyncResponseConsumerFactory.createHttpAsyncResponseConsumer();
-        client.execute(requestProducer, asyncResponseConsumer, new FutureCallback<HttpResponse>() {
+        final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request);
+        final HttpAsyncResponseConsumer<HttpResponse> asyncResponseConsumer =
+            httpAsyncResponseConsumerFactory.createHttpAsyncResponseConsumer();
+        final HttpClientContext context = HttpClientContext.create();
+        context.setAuthCache(hostTuple.authCache);
+        client.execute(requestProducer, asyncResponseConsumer, context, new FutureCallback<HttpResponse>() {
             @Override
             public void completed(HttpResponse httpResponse) {
                 try {
                     RequestLogger.logResponse(logger, request, host, httpResponse);
                     int statusCode = httpResponse.getStatusLine().getStatusCode();
                     Response response = new Response(request.getRequestLine(), host, httpResponse);
-                    if (isSuccessfulResponse(request.getMethod(), statusCode)) {
+                    if (isSuccessfulResponse(statusCode) || ignoreErrorCodes.contains(response.getStatusLine().getStatusCode())) {
                         onResponse(host);
                         listener.onSuccess(response);
                     } else {
@@ -312,7 +351,7 @@ public class RestClient implements Closeable {
                         if (isRetryStatus(statusCode)) {
                             //mark host dead and retry against next one
                             onFailure(host);
-                            retryIfPossible(responseException, hosts, request);
+                            retryIfPossible(responseException);
                         } else {
                             //mark host alive and don't retry, as the error should be a request problem
                             onResponse(host);
@@ -329,14 +368,14 @@ public class RestClient implements Closeable {
                 try {
                     RequestLogger.logFailedRequest(logger, request, host, failure);
                     onFailure(host);
-                    retryIfPossible(failure, hosts, request);
+                    retryIfPossible(failure);
                 } catch(Exception e) {
                     listener.onDefinitiveFailure(e);
                 }
             }
 
-            private void retryIfPossible(Exception exception, Iterator<HttpHost> hosts, HttpRequestBase request) {
-                if (hosts.hasNext()) {
+            private void retryIfPossible(Exception exception) {
+                if (hostTuple.hosts.hasNext()) {
                     //in case we are retrying, check whether maxRetryTimeout has been reached
                     long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
                     long timeout = maxRetryTimeoutMillis - timeElapsedMillis;
@@ -347,7 +386,7 @@ public class RestClient implements Closeable {
                     } else {
                         listener.trackFailure(exception);
                         request.reset();
-                        performRequestAsync(startTime, hosts, request, httpAsyncResponseConsumerFactory, listener);
+                        performRequestAsync(startTime, hostTuple, request, ignoreErrorCodes, httpAsyncResponseConsumerFactory, listener);
                     }
                 } else {
                     listener.onDefinitiveFailure(exception);
@@ -385,17 +424,18 @@ public class RestClient implements Closeable {
      * The iterator returned will never be empty. In case there are no healthy hosts available, or dead ones to be be retried,
      * one dead host gets returned so that it can be retried.
      */
-    private Iterable<HttpHost> nextHost() {
+    private HostTuple<Iterator<HttpHost>> nextHost() {
+        final HostTuple<Set<HttpHost>> hostTuple = this.hostTuple;
         Collection<HttpHost> nextHosts = Collections.emptySet();
         do {
-            Set<HttpHost> filteredHosts = new HashSet<>(hosts);
+            Set<HttpHost> filteredHosts = new HashSet<>(hostTuple.hosts);
             for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
                 if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
                     filteredHosts.remove(entry.getKey());
                 }
             }
             if (filteredHosts.isEmpty()) {
-                //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
+                //last resort: if there are no good host to use, return a single dead one, the one that's closest to being retried
                 List<Map.Entry<HttpHost, DeadHostState>> sortedHosts = new ArrayList<>(blacklist.entrySet());
                 if (sortedHosts.size() > 0) {
                     Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
@@ -414,7 +454,7 @@ public class RestClient implements Closeable {
                 nextHosts = rotatedHosts;
             }
         } while(nextHosts.isEmpty());
-        return nextHosts;
+        return new HostTuple<>(nextHosts.iterator(), hostTuple.authCache);
     }
 
     /**
@@ -452,8 +492,8 @@ public class RestClient implements Closeable {
         client.close();
     }
 
-    private static boolean isSuccessfulResponse(String method, int statusCode) {
-        return statusCode < 300 || (HttpHead.METHOD_NAME.equals(method) && statusCode == 404);
+    private static boolean isSuccessfulResponse(int statusCode) {
+        return statusCode < 300;
     }
 
     private static boolean isRetryStatus(int statusCode) {
@@ -510,7 +550,6 @@ public class RestClient implements Closeable {
     }
 
     private static URI buildUri(String pathPrefix, String path, Map<String, String> params) {
-        Objects.requireNonNull(params, "params must not be null");
         Objects.requireNonNull(path, "path must not be null");
         try {
             String fullPath;
@@ -657,4 +696,18 @@ public class RestClient implements Closeable {
 
         }
     }
+
+    /**
+     * {@code HostTuple} enables the {@linkplain HttpHost}s and {@linkplain AuthCache} to be set together in a thread
+     * safe, volatile way.
+     */
+    private static class HostTuple<T> {
+        public final T hosts;
+        public final AuthCache authCache;
+
+        HostTuple(final T hosts, final AuthCache authCache) {
+            this.hosts = hosts;
+            this.authCache = authCache;
+        }
+    }
 }
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
index d881bd70a44..4466a61d9df 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java
@@ -28,6 +28,8 @@ import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.Objects;
 
 /**
@@ -177,7 +179,12 @@ public final class RestClientBuilder {
         if (failureListener == null) {
             failureListener = new RestClient.FailureListener();
         }
-        CloseableHttpAsyncClient httpClient = createHttpClient();
+        CloseableHttpAsyncClient httpClient = AccessController.doPrivileged(new PrivilegedAction<CloseableHttpAsyncClient>() {
+            @Override
+            public CloseableHttpAsyncClient run() {
+                return createHttpClient();
+            }
+        });
         RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, pathPrefix, failureListener);
         httpClient.start();
         return restClient;
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java
index f997f798712..da5a960c2e8 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java
@@ -24,6 +24,7 @@ import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
 import org.apache.http.HttpHost;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -80,7 +81,7 @@ public class RestClientMultipleHostsIntegTests extends RestClientTestCase {
     }
 
     private static HttpServer createHttpServer() throws Exception {
-        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         //returns a different status code depending on the path
         for (int statusCode : getAllStatusCodes()) {
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index 90ee4431009..6f87a244ff5 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -26,8 +26,10 @@ import org.apache.http.HttpResponse;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
@@ -73,13 +75,15 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
     public void createRestClient() throws IOException {
         CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+               any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
             @Override
             public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
                 HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
                 HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
                 HttpHost httpHost = requestProducer.getTarget();
-                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2];
+                assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class));
+                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                 //return the desired status code or exception depending on the path
                 if (request.getURI().getPath().equals("/soe")) {
                     futureCallback.failed(new SocketTimeoutException(httpHost.toString()));
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java
index 4440c1e8f97..e75de2f609c 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java
@@ -26,9 +26,14 @@ import com.sun.net.httpserver.HttpServer;
 import org.apache.http.Consts;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -39,7 +44,6 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -48,8 +52,10 @@ import java.util.Set;
 import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
 import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
 import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -67,27 +73,14 @@ public class RestClientSingleHostIntegTests extends RestClientTestCase {
 
     @BeforeClass
     public static void startHttpServer() throws Exception {
-        String pathPrefixWithoutLeadingSlash;
-        if (randomBoolean()) {
-            pathPrefixWithoutLeadingSlash = "testPathPrefix/" + randomAsciiOfLengthBetween(1, 5);
-            pathPrefix = "/" + pathPrefixWithoutLeadingSlash;
-        } else {
-            pathPrefix = pathPrefixWithoutLeadingSlash = "";
-        }
-
+        pathPrefix = randomBoolean() ? "/testPathPrefix/" + randomAsciiOfLengthBetween(1, 5) : "";
         httpServer = createHttpServer();
-        int numHeaders = randomIntBetween(0, 5);
-        defaultHeaders = generateHeaders("Header-default", "Header-array", numHeaders);
-        RestClientBuilder restClientBuilder = RestClient.builder(
-                new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort())).setDefaultHeaders(defaultHeaders);
-        if (pathPrefix.length() > 0) {
-            restClientBuilder.setPathPrefix((randomBoolean() ? "/" : "") + pathPrefixWithoutLeadingSlash);
-        }
-        restClient = restClientBuilder.build();
+        defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default");
+        restClient = createRestClient(false, true);
     }
 
     private static HttpServer createHttpServer() throws Exception {
-        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         //returns a different status code depending on the path
         for (int statusCode : getAllStatusCodes()) {
@@ -131,6 +124,35 @@ public class RestClientSingleHostIntegTests extends RestClientTestCase {
         }
     }
 
+    private static RestClient createRestClient(final boolean useAuth, final boolean usePreemptiveAuth) {
+        // provide the username/password for every request
+        final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("user", "pass"));
+
+        final RestClientBuilder restClientBuilder = RestClient.builder(
+            new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort())).setDefaultHeaders(defaultHeaders);
+        if (pathPrefix.length() > 0) {
+            // sometimes cut off the leading slash
+            restClientBuilder.setPathPrefix(randomBoolean() ? pathPrefix.substring(1) : pathPrefix);
+        }
+
+        if (useAuth) {
+            restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+                @Override
+                public HttpAsyncClientBuilder customizeHttpClient(final HttpAsyncClientBuilder httpClientBuilder) {
+                    if (usePreemptiveAuth == false) {
+                        // disable preemptive auth by ignoring any authcache
+                        httpClientBuilder.disableAuthCaching();
+                    }
+
+                    return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                }
+            });
+        }
+
+        return restClientBuilder.build();
+    }
+
     @AfterClass
     public static void stopHttpServers() throws IOException {
         restClient.close();
@@ -150,42 +172,25 @@ public class RestClientSingleHostIntegTests extends RestClientTestCase {
             if (method.equals("HEAD") == false) {
                 standardHeaders.add("Content-length");
             }
-
-            final int numHeaders = randomIntBetween(1, 5);
-            final Header[] headers = generateHeaders("Header", "Header-array", numHeaders);
-            final Map<String, List<String>> expectedHeaders = new HashMap<>();
-
-            addHeaders(expectedHeaders, defaultHeaders, headers);
-
+            final Header[] requestHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header");
             final int statusCode = randomStatusCode(getRandom());
             Response esResponse;
             try {
-                esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), headers);
+                esResponse = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), requestHeaders);
             } catch(ResponseException e) {
                 esResponse = e.getResponse();
             }
 
             assertEquals(method, esResponse.getRequestLine().getMethod());
             assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
-            assertEquals((pathPrefix.length() > 0 ? pathPrefix : "") + "/" + statusCode, esResponse.getRequestLine().getUri());
-
+            assertEquals(pathPrefix + "/" + statusCode, esResponse.getRequestLine().getUri());
+            assertHeaders(defaultHeaders, requestHeaders, esResponse.getHeaders(), standardHeaders);
             for (final Header responseHeader : esResponse.getHeaders()) {
-                final String name = responseHeader.getName();
-                final String value = responseHeader.getValue();
-                if (name.startsWith("Header")) {
-                    final List<String> values = expectedHeaders.get(name);
-                    assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
-                    assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
-
-                    // we've collected them all
-                    if (values.isEmpty()) {
-                        expectedHeaders.remove(name);
-                    }
-                } else {
+                String name = responseHeader.getName();
+                if (name.startsWith("Header") == false) {
                     assertTrue("unknown header was returned " + name, standardHeaders.remove(name));
                 }
             }
-            assertTrue("some headers that were sent weren't returned: " + expectedHeaders, expectedHeaders.isEmpty());
             assertTrue("some expected standard headers weren't returned: " + standardHeaders, standardHeaders.isEmpty());
         }
     }
@@ -208,7 +213,41 @@ public class RestClientSingleHostIntegTests extends RestClientTestCase {
         bodyTest("GET");
     }
 
-    private void bodyTest(String method) throws IOException {
+    /**
+     * Verify that credentials are sent on the first request with preemptive auth enabled (default when provided with credentials).
+     */
+    public void testPreemptiveAuthEnabled() throws IOException  {
+        final String[] methods = { "POST", "PUT", "GET", "DELETE" };
+
+        try (RestClient restClient = createRestClient(true, true)) {
+            for (final String method : methods) {
+                final Response response = bodyTest(restClient, method);
+
+                assertThat(response.getHeader("Authorization"), startsWith("Basic"));
+            }
+        }
+    }
+
+    /**
+     * Verify that credentials are <em>not</em> sent on the first request with preemptive auth disabled.
+     */
+    public void testPreemptiveAuthDisabled() throws IOException  {
+        final String[] methods = { "POST", "PUT", "GET", "DELETE" };
+
+        try (RestClient restClient = createRestClient(true, false)) {
+            for (final String method : methods) {
+                final Response response = bodyTest(restClient, method);
+
+                assertThat(response.getHeader("Authorization"), nullValue());
+            }
+        }
+    }
+
+    private Response bodyTest(final String method) throws IOException {
+        return bodyTest(restClient, method);
+    }
+
+    private Response bodyTest(final RestClient restClient, final String method) throws IOException {
         String requestBody = "{ \"field\": \"value\" }";
         StringEntity entity = new StringEntity(requestBody);
         int statusCode = randomStatusCode(getRandom());
@@ -220,7 +259,9 @@ public class RestClientSingleHostIntegTests extends RestClientTestCase {
         }
         assertEquals(method, esResponse.getRequestLine().getMethod());
         assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
-        assertEquals((pathPrefix.length() > 0 ? pathPrefix : "") + "/" + statusCode, esResponse.getRequestLine().getUri());
+        assertEquals(pathPrefix + "/" + statusCode, esResponse.getRequestLine().getUri());
         assertEquals(requestBody, EntityUtils.toString(esResponse.getEntity()));
+
+        return esResponse;
     }
 }
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index ce0d6d0936e..69048988ee9 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -34,10 +34,12 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
@@ -56,7 +58,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Future;
@@ -70,7 +71,6 @@ import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -98,11 +98,13 @@ public class RestClientSingleHostTests extends RestClientTestCase {
     public void createRestClient() throws IOException {
         httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+                any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
                     @Override
                     public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
                         HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
-                        FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                        HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2];
+                        assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class));
+                        FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                         HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest();
                         //return the desired status code or exception depending on the path
                         if (request.getURI().getPath().equals("/soe")) {
@@ -131,9 +133,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
                     }
                 });
 
-
-        int numHeaders = randomIntBetween(0, 3);
-        defaultHeaders = generateHeaders("Header-default", "Header-array", numHeaders);
+        defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default");
         httpHost = new HttpHost("localhost", 9200);
         failureListener = new HostsTrackingFailureListener();
         restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, null, failureListener);
@@ -160,7 +160,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         for (String httpMethod : getHttpMethods()) {
             HttpUriRequest expectedRequest = performRandomRequest(httpMethod);
             verify(httpClient, times(++times)).<HttpResponse>execute(requestArgumentCaptor.capture(),
-                    any(HttpAsyncResponseConsumer.class), any(FutureCallback.class));
+                    any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class));
             HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest();
             assertEquals(expectedRequest.getURI(), actualRequest.getURI());
             assertEquals(expectedRequest.getClass(), actualRequest.getClass());
@@ -220,23 +220,45 @@ public class RestClientSingleHostTests extends RestClientTestCase {
      */
     public void testErrorStatusCodes() throws IOException {
         for (String method : getHttpMethods()) {
+            Set<Integer> expectedIgnores = new HashSet<>();
+            String ignoreParam = "";
+            if (HttpHead.METHOD_NAME.equals(method)) {
+                expectedIgnores.add(404);
+            }
+            if (randomBoolean()) {
+                int numIgnores = randomIntBetween(1, 3);
+                for (int i = 0; i < numIgnores; i++) {
+                    Integer code = randomFrom(getAllErrorStatusCodes());
+                    expectedIgnores.add(code);
+                    ignoreParam += code;
+                    if (i < numIgnores - 1) {
+                        ignoreParam += ",";
+                    }
+                }
+            }
             //error status codes should cause an exception to be thrown
             for (int errorStatusCode : getAllErrorStatusCodes()) {
                 try {
-                    Response response = performRequest(method, "/" + errorStatusCode);
-                    if (method.equals("HEAD") && errorStatusCode == 404) {
-                        //no exception gets thrown although we got a 404
-                        assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    Map<String, String> params;
+                    if (ignoreParam.isEmpty()) {
+                        params = Collections.emptyMap();
+                    } else {
+                        params = Collections.singletonMap("ignore", ignoreParam);
+                    }
+                    Response response = performRequest(method, "/" + errorStatusCode, params);
+                    if (expectedIgnores.contains(errorStatusCode)) {
+                        //no exception gets thrown although we got an error status code, as it was configured to be ignored
+                        assertEquals(errorStatusCode, response.getStatusLine().getStatusCode());
                     } else {
                         fail("request should have failed");
                     }
                 } catch(ResponseException e) {
-                    if (method.equals("HEAD") && errorStatusCode == 404) {
+                    if (expectedIgnores.contains(errorStatusCode)) {
                         throw e;
                     }
-                    assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    assertEquals(errorStatusCode, e.getResponse().getStatusLine().getStatusCode());
                 }
-                if (errorStatusCode <= 500) {
+                if (errorStatusCode <= 500 || expectedIgnores.contains(errorStatusCode)) {
                     failureListener.assertNotCalled();
                 } else {
                     failureListener.assertCalled(httpHost);
@@ -339,44 +361,26 @@ public class RestClientSingleHostTests extends RestClientTestCase {
      */
     public void testHeaders() throws IOException {
         for (String method : getHttpMethods()) {
-            final int numHeaders = randomIntBetween(1, 5);
-            final Header[] headers = generateHeaders("Header", null, numHeaders);
-            final Map<String, List<String>> expectedHeaders = new HashMap<>();
-
-            addHeaders(expectedHeaders, defaultHeaders, headers);
-
+            final Header[] requestHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header");
             final int statusCode = randomStatusCode(getRandom());
             Response esResponse;
             try {
-                esResponse = restClient.performRequest(method, "/" + statusCode, headers);
+                esResponse = restClient.performRequest(method, "/" + statusCode, requestHeaders);
             } catch(ResponseException e) {
                 esResponse = e.getResponse();
             }
             assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
-            for (Header responseHeader : esResponse.getHeaders()) {
-                final String name = responseHeader.getName();
-                final String value = responseHeader.getValue();
-                final List<String> values = expectedHeaders.get(name);
-                assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
-                assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
-
-                // we've collected them all
-                if (values.isEmpty()) {
-                    expectedHeaders.remove(name);
-                }
-            }
-            assertTrue("some headers that were sent weren't returned " + expectedHeaders, expectedHeaders.isEmpty());
+            assertHeaders(defaultHeaders, requestHeaders, esResponse.getHeaders(), Collections.<String>emptySet());
         }
     }
 
     private HttpUriRequest performRandomRequest(String method) throws Exception {
         String uriAsString = "/" + randomStatusCode(getRandom());
         URIBuilder uriBuilder = new URIBuilder(uriAsString);
-        Map<String, String> params = Collections.emptyMap();
+        final Map<String, String> params = new HashMap<>();
         boolean hasParams = randomBoolean();
         if (hasParams) {
             int numParams = randomIntBetween(1, 3);
-            params = new HashMap<>(numParams);
             for (int i = 0; i < numParams; i++) {
                 String paramKey = "param-" + i;
                 String paramValue = randomAsciiOfLengthBetween(3, 10);
@@ -384,6 +388,14 @@ public class RestClientSingleHostTests extends RestClientTestCase {
                 uriBuilder.addParameter(paramKey, paramValue);
             }
         }
+        if (randomBoolean()) {
+            //randomly add some ignore parameter, which doesn't get sent as part of the request
+            String ignore = Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
+            if (randomBoolean()) {
+                ignore += "," + Integer.toString(randomFrom(RestClientTestUtil.getAllErrorStatusCodes()));
+            }
+            params.put("ignore", ignore);
+        }
         URI uri = uriBuilder.build();
 
         HttpUriRequest request;
@@ -424,10 +436,9 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         }
 
         Header[] headers = new Header[0];
-        final int numHeaders = randomIntBetween(1, 5);
-        final Set<String> uniqueNames = new HashSet<>(numHeaders);
+        final Set<String> uniqueNames = new HashSet<>();
         if (randomBoolean()) {
-            headers = generateHeaders("Header", "Header-array", numHeaders);
+            headers = RestClientTestUtil.randomHeaders(getRandom(), "Header");
             for (Header header : headers) {
                 request.addHeader(header);
                 uniqueNames.add(header.getName());
@@ -455,16 +466,25 @@ public class RestClientSingleHostTests extends RestClientTestCase {
     }
 
     private Response performRequest(String method, String endpoint, Header... headers) throws IOException {
-        switch(randomIntBetween(0, 2)) {
+        return performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
+    }
+
+    private Response performRequest(String method, String endpoint, Map<String, String> params, Header... headers) throws IOException {
+        int methodSelector;
+        if (params.isEmpty()) {
+            methodSelector = randomIntBetween(0, 2);
+        } else {
+            methodSelector = randomIntBetween(1, 2);
+        }
+        switch(methodSelector) {
             case 0:
                 return restClient.performRequest(method, endpoint, headers);
             case 1:
-                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
+                return restClient.performRequest(method, endpoint, params, headers);
             case 2:
-                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), (HttpEntity)null, headers);
+                return restClient.performRequest(method, endpoint, params, (HttpEntity)null, headers);
             default:
                 throw new UnsupportedOperationException();
         }
     }
-
 }
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index f35110e4f9e..5542792835b 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -43,6 +43,7 @@ dependencies {
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
diff --git a/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1
deleted file mode 100644
index b92131d6fab..00000000000
--- a/client/sniffer/licenses/jackson-core-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd13b1c033741d48291315c6370f7d475a42dccf
\ No newline at end of file
diff --git a/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1 b/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1
new file mode 100644
index 00000000000..af7677d13c2
--- /dev/null
+++ b/client/sniffer/licenses/jackson-core-2.8.6.jar.sha1
@@ -0,0 +1 @@
+2ef7b1cc34de149600f5e75bc2d5bf40de894e60
\ No newline at end of file
diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
index aeb0620134b..5221b205dd4 100644
--- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
+++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientTestCase;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.junit.After;
 import org.junit.Before;
 
@@ -141,7 +142,7 @@ public class ElasticsearchHostsSnifferTests extends RestClientTestCase {
     }
 
     private static HttpServer createHttpServer(final SniffResponse sniffResponse, final int sniffTimeoutMillis) throws IOException {
-        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
+        HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.createContext("/_nodes/http", new ResponseHandler(sniffTimeoutMillis, sniffResponse));
         return httpServer;
     }
diff --git a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
index 4296932a002..6a2a45ef281 100644
--- a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
+++ b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
@@ -30,16 +30,19 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
 import org.apache.http.Header;
-import org.apache.http.message.BasicHeader;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 @TestMethodProviders({
         JUnit3MethodProvider.class
 })
@@ -53,70 +56,56 @@ import java.util.Set;
 public abstract class RestClientTestCase extends RandomizedTest {
 
     /**
-     * Create the specified number of {@link Header}s.
-     * <p>
-     * Generated header names will be the {@code baseName} plus its index or, rarely, the {@code arrayName} if it's supplied.
+     * Assert that the actual headers are the expected ones given the original default and request headers. Some headers can be ignored,
+     * for instance in case the http client is adding its own automatically.
      *
-     * @param baseName The base name to use for all headers.
-     * @param arrayName The optional ({@code null}able) array name to use randomly.
-     * @param headers The number of headers to create.
-     * @return Never {@code null}.
+     * @param defaultHeaders the default headers set to the REST client instance
+     * @param requestHeaders the request headers sent with a particular request
+     * @param actualHeaders the actual headers as a result of the provided default and request headers
+     * @param ignoreHeaders header keys to be ignored as they are not part of default nor request headers, yet they
+     *                      will be part of the actual ones
      */
-    protected static Header[] generateHeaders(final String baseName, final String arrayName, final int headers) {
-        final Header[] generated = new Header[headers];
-        for (int i = 0; i < headers; i++) {
-            String headerName = baseName + i;
-            if (arrayName != null && rarely()) {
-                headerName = arrayName;
-            }
-
-            generated[i] = new BasicHeader(headerName, randomAsciiOfLengthBetween(3, 10));
+    protected static void assertHeaders(final Header[] defaultHeaders, final Header[] requestHeaders,
+                                        final Header[] actualHeaders, final Set<String> ignoreHeaders) {
+        final Map<String, List<String>> expectedHeaders = new HashMap<>();
+        final Set<String> requestHeaderKeys = new HashSet<>();
+        for (final Header header : requestHeaders) {
+            final String name = header.getName();
+            addValueToListEntry(expectedHeaders, name, header.getValue());
+            requestHeaderKeys.add(name);
         }
-        return generated;
+        for (final Header defaultHeader : defaultHeaders) {
+            final String name = defaultHeader.getName();
+            if (requestHeaderKeys.contains(name) == false) {
+                addValueToListEntry(expectedHeaders, name, defaultHeader.getValue());
+            }
+        }
+        Set<String> actualIgnoredHeaders = new HashSet<>();
+        for (Header responseHeader : actualHeaders) {
+            final String name = responseHeader.getName();
+            if (ignoreHeaders.contains(name)) {
+                expectedHeaders.remove(name);
+                actualIgnoredHeaders.add(name);
+                continue;
+            }
+            final String value = responseHeader.getValue();
+            final List<String> values = expectedHeaders.get(name);
+            assertNotNull("found response header [" + name + "] that wasn't originally sent: " + value, values);
+            assertTrue("found incorrect response header [" + name + "]: " + value, values.remove(value));
+            if (values.isEmpty()) {
+                expectedHeaders.remove(name);
+            }
+        }
+        assertEquals("some headers meant to be ignored were not part of the actual headers", ignoreHeaders, actualIgnoredHeaders);
+        assertTrue("some headers that were sent weren't returned " + expectedHeaders, expectedHeaders.isEmpty());
     }
 
-    /**
-     * Create a new {@link List} within the {@code map} if none exists for {@code name} or append to the existing list.
-     *
-     * @param map The map to manipulate.
-     * @param name The name to create/append the list for.
-     * @param value The value to add.
-     */
-    private static void createOrAppendList(final Map<String, List<String>> map, final String name, final String value) {
+    private static void addValueToListEntry(final Map<String, List<String>> map, final String name, final String value) {
         List<String> values = map.get(name);
-
         if (values == null) {
             values = new ArrayList<>();
             map.put(name, values);
         }
-
         values.add(value);
     }
-
-    /**
-     * Add the {@code headers} to the {@code map} so that related tests can more easily assert that they exist.
-     * <p>
-     * If both the {@code defaultHeaders} and {@code headers} contain the same {@link Header}, based on its
-     * {@linkplain Header#getName() name}, then this will only use the {@code Header}(s) from {@code headers}.
-     *
-     * @param map The map to build with name/value(s) pairs.
-     * @param defaultHeaders The headers to add to the map representing default headers.
-     * @param headers The headers to add to the map representing request-level headers.
-     * @see #createOrAppendList(Map, String, String)
-     */
-    protected static void addHeaders(final Map<String, List<String>> map, final Header[] defaultHeaders, final Header[] headers) {
-        final Set<String> uniqueHeaders = new HashSet<>();
-        for (final Header header : headers) {
-            final String name = header.getName();
-            createOrAppendList(map, name, header.getValue());
-            uniqueHeaders.add(name);
-        }
-        for (final Header defaultHeader : defaultHeaders) {
-            final String name = defaultHeader.getName();
-            if (uniqueHeaders.contains(name) == false) {
-                createOrAppendList(map, name, defaultHeader.getValue());
-            }
-        }
-    }
-
 }
diff --git a/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
index 4d4aa00f492..a0a6641abbc 100644
--- a/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
+++ b/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
@@ -19,7 +19,11 @@
 
 package org.elasticsearch.client;
 
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.http.Header;
+import org.apache.http.message.BasicHeader;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,7 +59,7 @@ final class RestClientTestUtil {
     }
 
     static int randomStatusCode(Random random) {
-        return RandomPicks.randomFrom(random, ALL_ERROR_STATUS_CODES);
+        return RandomPicks.randomFrom(random, ALL_STATUS_CODES);
     }
 
     static int randomOkStatusCode(Random random) {
@@ -81,4 +85,23 @@ final class RestClientTestUtil {
     static List<Integer> getAllStatusCodes() {
         return ALL_STATUS_CODES;
     }
+
+    /**
+     * Create a random number of {@link Header}s.
+     * Generated header names will either be the {@code baseName} plus its index, or exactly the provided {@code baseName} so that the
+     * we test also support for multiple headers with same key and different values.
+     */
+    static Header[] randomHeaders(Random random, final String baseName) {
+        int numHeaders = RandomNumbers.randomIntBetween(random, 0, 5);
+        final Header[] headers = new Header[numHeaders];
+        for (int i = 0; i < numHeaders; i++) {
+            String headerName = baseName;
+            //randomly exercise the code path that supports multiple headers with same key
+            if (random.nextBoolean()) {
+                headerName = headerName + i;
+            }
+            headers[i] = new BasicHeader(headerName, RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
+        }
+        return headers;
+    }
 }
diff --git a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
index ca90723ae82..3233470a253 100644
--- a/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
+++ b/client/transport/src/main/java/org/elasticsearch/transport/client/PreBuiltTransportClient.java
@@ -22,6 +22,7 @@ package org.elasticsearch.transport.client;
 import io.netty.util.ThreadDeathWatcher;
 import io.netty.util.concurrent.GlobalEventExecutor;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.reindex.ReindexPlugin;
@@ -36,41 +37,70 @@ import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 /**
- * A builder to create an instance of {@link TransportClient}
- * This class pre-installs the
+ * A builder to create an instance of {@link TransportClient}. This class pre-installs the
  * {@link Netty4Plugin},
  * {@link ReindexPlugin},
  * {@link PercolatorPlugin},
  * and {@link MustachePlugin}
- * for the client. These plugins are all elasticsearch core modules required.
+ * plugins for the client. These plugins are all the required modules for Elasticsearch.
  */
 @SuppressWarnings({"unchecked","varargs"})
 public class PreBuiltTransportClient extends TransportClient {
 
-    private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
-            Collections.unmodifiableList(
-                    Arrays.asList(
-                            Netty4Plugin.class,
-                            ReindexPlugin.class,
-                            PercolatorPlugin.class,
-                            MustachePlugin.class));
+    static {
+        // initialize Netty system properties before triggering any Netty class loads
+        initializeNetty();
+    }
 
+    /**
+     * Netty wants to do some unsafe things like use unsafe and replace a private field. This method disables these things by default, but
+     * can be overridden by setting the corresponding system properties.
+     */
+    @SuppressForbidden(reason = "set system properties to configure Netty")
+    private static void initializeNetty() {
+        final String noUnsafeKey = "io.netty.noUnsafe";
+        final String noUnsafe = System.getProperty(noUnsafeKey);
+        if (noUnsafe == null) {
+            // disable Netty from using unsafe
+            // while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
+            // the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
+            System.setProperty(noUnsafeKey, Boolean.toString(true));
+        }
+
+        final String noKeySetOptimizationKey = "io.netty.noKeySetOptimization";
+        final String noKeySetOptimization = System.getProperty(noKeySetOptimizationKey);
+        if (noKeySetOptimization == null) {
+            // disable Netty from replacing the selector key set
+            // while permissions are needed to set this, if a security exception is thrown the permission needed can either be granted or
+            // the system property can be set directly before starting the JVM; therefore, we do not catch a security exception here
+            System.setProperty(noKeySetOptimizationKey, Boolean.toString(true));
+        }
+    }
+
+    private static final Collection<Class<? extends Plugin>> PRE_INSTALLED_PLUGINS =
+        Collections.unmodifiableList(
+            Arrays.asList(
+                Netty4Plugin.class,
+                ReindexPlugin.class,
+                PercolatorPlugin.class,
+                MustachePlugin.class));
 
     /**
      * Creates a new transport client with pre-installed plugins.
+     *
      * @param settings the settings passed to this transport client
-     * @param plugins an optional array of additional plugins to run with this client
+     * @param plugins  an optional array of additional plugins to run with this client
      */
     @SafeVarargs
     public PreBuiltTransportClient(Settings settings, Class<? extends Plugin>... plugins) {
         this(settings, Arrays.asList(plugins));
     }
 
-
     /**
      * Creates a new transport client with pre-installed plugins.
+     *
      * @param settings the settings passed to this transport client
-     * @param plugins a collection of additional plugins to run with this client
+     * @param plugins  a collection of additional plugins to run with this client
      */
     public PreBuiltTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
         this(settings, plugins, null);
@@ -78,12 +108,15 @@ public class PreBuiltTransportClient extends TransportClient {
 
     /**
      * Creates a new transport client with pre-installed plugins.
-     * @param settings the settings passed to this transport client
-     * @param plugins a collection of additional plugins to run with this client
-     * @param hostFailureListener a failure listener that is invoked if a node is disconnected. This can be <code>null</code>
+     *
+     * @param settings            the settings passed to this transport client
+     * @param plugins             a collection of additional plugins to run with this client
+     * @param hostFailureListener a failure listener that is invoked if a node is disconnected; this can be <code>null</code>
      */
-    public PreBuiltTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins,
-                                   HostFailureListener hostFailureListener) {
+    public PreBuiltTransportClient(
+        Settings settings,
+        Collection<Class<? extends Plugin>> plugins,
+        HostFailureListener hostFailureListener) {
         super(settings, Settings.EMPTY, addPlugins(plugins, PRE_INSTALLED_PLUGINS), hostFailureListener);
     }
 
diff --git a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
index a1d95b68af7..161b1d7e89c 100644
--- a/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
+++ b/client/transport/src/test/java/org/elasticsearch/transport/client/PreBuiltTransportClientTests.java
@@ -41,8 +41,6 @@ public class PreBuiltTransportClientTests extends RandomizedTest {
 
     @Test
     public void testPluginInstalled() {
-        // TODO: remove when Netty 4.1.5 is upgraded to Netty 4.1.6 including https://github.com/netty/netty/pull/5778
-        assumeFalse(Constants.JRE_IS_MINIMUM_JAVA9);
         try (TransportClient client = new PreBuiltTransportClient(Settings.EMPTY)) {
             Settings settings = client.settings();
             assertEquals(Netty4Plugin.NETTY_TRANSPORT_NAME, NetworkModule.HTTP_DEFAULT_TYPE_SETTING.get(settings));
diff --git a/core/build.gradle b/core/build.gradle
index 7a580335571..6e0b94dd6f9 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -94,6 +94,8 @@ dependencies {
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
     }
   }
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
 }
 
 if (isEclipse) {
diff --git a/core/licenses/jackson-core-2.8.1.jar.sha1 b/core/licenses/jackson-core-2.8.1.jar.sha1
deleted file mode 100644
index b92131d6fab..00000000000
--- a/core/licenses/jackson-core-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd13b1c033741d48291315c6370f7d475a42dccf
\ No newline at end of file
diff --git a/core/licenses/jackson-core-2.8.6.jar.sha1 b/core/licenses/jackson-core-2.8.6.jar.sha1
new file mode 100644
index 00000000000..af7677d13c2
--- /dev/null
+++ b/core/licenses/jackson-core-2.8.6.jar.sha1
@@ -0,0 +1 @@
+2ef7b1cc34de149600f5e75bc2d5bf40de894e60
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1 b/core/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1
deleted file mode 100644
index 7f1609bfd85..00000000000
--- a/core/licenses/jackson-dataformat-cbor-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a6fb7e75c9972559a78cf5cfc5a48a41a13ea40
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1
new file mode 100644
index 00000000000..6a2e9802353
--- /dev/null
+++ b/core/licenses/jackson-dataformat-cbor-2.8.6.jar.sha1
@@ -0,0 +1 @@
+b88721371cfa2d7242bb5e52fe70861aa061c050
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-smile-2.8.1.jar.sha1 b/core/licenses/jackson-dataformat-smile-2.8.1.jar.sha1
deleted file mode 100644
index 114d656a388..00000000000
--- a/core/licenses/jackson-dataformat-smile-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-005b73867bc12224946fc67fc8d49d9f5e698d7f
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1
new file mode 100644
index 00000000000..19be9a2040b
--- /dev/null
+++ b/core/licenses/jackson-dataformat-smile-2.8.6.jar.sha1
@@ -0,0 +1 @@
+71590ad45cee21249774e2f93e5eca66e446cef3
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1 b/core/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1
deleted file mode 100644
index 32ce0f74344..00000000000
--- a/core/licenses/jackson-dataformat-yaml-2.8.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-eb63166c723b0b4b9fb5298fca232a2f6612ec34
\ No newline at end of file
diff --git a/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1 b/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1
new file mode 100644
index 00000000000..c61dad3bbcd
--- /dev/null
+++ b/core/licenses/jackson-dataformat-yaml-2.8.6.jar.sha1
@@ -0,0 +1 @@
+8bd44d50f9a6cdff9c7578ea39d524eb519e35ab
\ No newline at end of file
diff --git a/core/licenses/lucene-analyzers-common-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-analyzers-common-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 5cab7b2fef1..00000000000
--- a/core/licenses/lucene-analyzers-common-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-770114e0188dd8b4f30e5878b4f6c8677cecf1be
\ No newline at end of file
diff --git a/core/licenses/lucene-analyzers-common-6.4.0.jar.sha1 b/core/licenses/lucene-analyzers-common-6.4.0.jar.sha1
new file mode 100644
index 00000000000..f0934e93d51
--- /dev/null
+++ b/core/licenses/lucene-analyzers-common-6.4.0.jar.sha1
@@ -0,0 +1 @@
+e0feb9281a7da7a7df62398ab0fc655d51f68fed
\ No newline at end of file
diff --git a/core/licenses/lucene-backward-codecs-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-backward-codecs-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 02677cb1ff8..00000000000
--- a/core/licenses/lucene-backward-codecs-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f4eb0257e8419beaa9f84da6a51375fda4e491f2
\ No newline at end of file
diff --git a/core/licenses/lucene-backward-codecs-6.4.0.jar.sha1 b/core/licenses/lucene-backward-codecs-6.4.0.jar.sha1
new file mode 100644
index 00000000000..8ba9e440224
--- /dev/null
+++ b/core/licenses/lucene-backward-codecs-6.4.0.jar.sha1
@@ -0,0 +1 @@
+14698ecbca1437615ee31d412d0edd3440b4fccf
\ No newline at end of file
diff --git a/core/licenses/lucene-core-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-core-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index ea81fbaeb56..00000000000
--- a/core/licenses/lucene-core-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c80ad16cd36c41012abb8a8bb1c7328c6d680b4a
\ No newline at end of file
diff --git a/core/licenses/lucene-core-6.4.0.jar.sha1 b/core/licenses/lucene-core-6.4.0.jar.sha1
new file mode 100644
index 00000000000..e8e74ddc7a3
--- /dev/null
+++ b/core/licenses/lucene-core-6.4.0.jar.sha1
@@ -0,0 +1 @@
+09dd516b847dcaf8da4e9096bf3189b0b3607aef
\ No newline at end of file
diff --git a/core/licenses/lucene-grouping-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-grouping-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index d4442ded938..00000000000
--- a/core/licenses/lucene-grouping-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-070d4e370f4fe0b8a04b2bce5b4381201b0c783f
\ No newline at end of file
diff --git a/core/licenses/lucene-grouping-6.4.0.jar.sha1 b/core/licenses/lucene-grouping-6.4.0.jar.sha1
new file mode 100644
index 00000000000..9a54629dcd4
--- /dev/null
+++ b/core/licenses/lucene-grouping-6.4.0.jar.sha1
@@ -0,0 +1 @@
+68a8f986a0076ad784cbb20813b9465b94e4c846
\ No newline at end of file
diff --git a/core/licenses/lucene-highlighter-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-highlighter-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index e6fc043a287..00000000000
--- a/core/licenses/lucene-highlighter-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-131d9a86f5943675493a85def0e692842f396458
\ No newline at end of file
diff --git a/core/licenses/lucene-highlighter-6.4.0.jar.sha1 b/core/licenses/lucene-highlighter-6.4.0.jar.sha1
new file mode 100644
index 00000000000..fd11e9112fb
--- /dev/null
+++ b/core/licenses/lucene-highlighter-6.4.0.jar.sha1
@@ -0,0 +1 @@
+6d921c1242b608a4dcd0784e32bcd94097ad93cd
\ No newline at end of file
diff --git a/core/licenses/lucene-join-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-join-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 6c90673f498..00000000000
--- a/core/licenses/lucene-join-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-385b2202036b50a764e4d2b032e21496b74a1c8e
\ No newline at end of file
diff --git a/core/licenses/lucene-join-6.4.0.jar.sha1 b/core/licenses/lucene-join-6.4.0.jar.sha1
new file mode 100644
index 00000000000..171a8dfe26c
--- /dev/null
+++ b/core/licenses/lucene-join-6.4.0.jar.sha1
@@ -0,0 +1 @@
+74d3cdf1bc863e3836b06f1865c970127cc15f26
\ No newline at end of file
diff --git a/core/licenses/lucene-memory-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-memory-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index bdb3a168612..00000000000
--- a/core/licenses/lucene-memory-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e8742a44ef4849a17d5e59ef36e9a52a8f2370c2
\ No newline at end of file
diff --git a/core/licenses/lucene-memory-6.4.0.jar.sha1 b/core/licenses/lucene-memory-6.4.0.jar.sha1
new file mode 100644
index 00000000000..2ffd52a6f39
--- /dev/null
+++ b/core/licenses/lucene-memory-6.4.0.jar.sha1
@@ -0,0 +1 @@
+dd13729c0b401e3df11bce0c343d1e00f07b9a19
\ No newline at end of file
diff --git a/core/licenses/lucene-misc-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-misc-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index e29fc5f139c..00000000000
--- a/core/licenses/lucene-misc-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7ce2e4948fb66393a34f4200a6131cfde43e47bd
\ No newline at end of file
diff --git a/core/licenses/lucene-misc-6.4.0.jar.sha1 b/core/licenses/lucene-misc-6.4.0.jar.sha1
new file mode 100644
index 00000000000..69c41bc81b0
--- /dev/null
+++ b/core/licenses/lucene-misc-6.4.0.jar.sha1
@@ -0,0 +1 @@
+ce27abe3490bb8ccbebd2eefcb68f42a609ca986
\ No newline at end of file
diff --git a/core/licenses/lucene-queries-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-queries-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 4998ff5b2e4..00000000000
--- a/core/licenses/lucene-queries-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c1c385a597ce797b0049d9b2281b09593e1488a
\ No newline at end of file
diff --git a/core/licenses/lucene-queries-6.4.0.jar.sha1 b/core/licenses/lucene-queries-6.4.0.jar.sha1
new file mode 100644
index 00000000000..1ce80d5ba80
--- /dev/null
+++ b/core/licenses/lucene-queries-6.4.0.jar.sha1
@@ -0,0 +1 @@
+bd1978e3fdac2fadf1068828b0b1b534a56873c3
\ No newline at end of file
diff --git a/core/licenses/lucene-queryparser-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-queryparser-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 9ba51f22f25..00000000000
--- a/core/licenses/lucene-queryparser-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fafaa22906c067e6894f9f2b18ad03ded98e2f38
\ No newline at end of file
diff --git a/core/licenses/lucene-queryparser-6.4.0.jar.sha1 b/core/licenses/lucene-queryparser-6.4.0.jar.sha1
new file mode 100644
index 00000000000..155c688840d
--- /dev/null
+++ b/core/licenses/lucene-queryparser-6.4.0.jar.sha1
@@ -0,0 +1 @@
+fb8fe41948fccf13b5dbb5d50441cac974544ade
\ No newline at end of file
diff --git a/core/licenses/lucene-sandbox-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-sandbox-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index cce2045942b..00000000000
--- a/core/licenses/lucene-sandbox-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-19c64a84617f42bb4c11b1e266df4009cd37fdd0
\ No newline at end of file
diff --git a/core/licenses/lucene-sandbox-6.4.0.jar.sha1 b/core/licenses/lucene-sandbox-6.4.0.jar.sha1
new file mode 100644
index 00000000000..7df93d742c3
--- /dev/null
+++ b/core/licenses/lucene-sandbox-6.4.0.jar.sha1
@@ -0,0 +1 @@
+e7f7d1ad298c4af264199d9199f34f2e4d9ca2b5
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-spatial-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 8169bea2fae..00000000000
--- a/core/licenses/lucene-spatial-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bc8613fb61c0ae95dd3680b0f65e3380c3fd0d6c
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-6.4.0.jar.sha1 b/core/licenses/lucene-spatial-6.4.0.jar.sha1
new file mode 100644
index 00000000000..78dd7de93d0
--- /dev/null
+++ b/core/licenses/lucene-spatial-6.4.0.jar.sha1
@@ -0,0 +1 @@
+5d4b3ce4df83d0509e0b5f7eecda72af458ba225
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-extras-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-spatial-extras-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 2614704c057..00000000000
--- a/core/licenses/lucene-spatial-extras-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0fa2c3e722294e863f3c70a15e97a18397391fb4
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial-extras-6.4.0.jar.sha1 b/core/licenses/lucene-spatial-extras-6.4.0.jar.sha1
new file mode 100644
index 00000000000..016fd2b6ddd
--- /dev/null
+++ b/core/licenses/lucene-spatial-extras-6.4.0.jar.sha1
@@ -0,0 +1 @@
+3b486b51d3aede074ab6de890b427379d40c0438
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial3d-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-spatial3d-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 9b1c45581a1..00000000000
--- a/core/licenses/lucene-spatial3d-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db74c6313965ffdd10d9b19be2eed4ae2c76d2e3
\ No newline at end of file
diff --git a/core/licenses/lucene-spatial3d-6.4.0.jar.sha1 b/core/licenses/lucene-spatial3d-6.4.0.jar.sha1
new file mode 100644
index 00000000000..3d7cd4e751f
--- /dev/null
+++ b/core/licenses/lucene-spatial3d-6.4.0.jar.sha1
@@ -0,0 +1 @@
+344097014aeaaa0f94a217f3697e14ceee06581f
\ No newline at end of file
diff --git a/core/licenses/lucene-suggest-6.4.0-snapshot-ec38570.jar.sha1 b/core/licenses/lucene-suggest-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 91841f474ef..00000000000
--- a/core/licenses/lucene-suggest-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b85ae1121b5fd56df985615a3cdd7b3879e9b92d
\ No newline at end of file
diff --git a/core/licenses/lucene-suggest-6.4.0.jar.sha1 b/core/licenses/lucene-suggest-6.4.0.jar.sha1
new file mode 100644
index 00000000000..2cfe897d139
--- /dev/null
+++ b/core/licenses/lucene-suggest-6.4.0.jar.sha1
@@ -0,0 +1 @@
+6c4706b86718f2653120e0dbfd24e03248dd2ea7
\ No newline at end of file
diff --git a/core/src/main/java/org/apache/lucene/analysis/synonym/GraphTokenStreamFiniteStrings.java b/core/src/main/java/org/apache/lucene/analysis/synonym/GraphTokenStreamFiniteStrings.java
deleted file mode 100644
index 3d806588eca..00000000000
--- a/core/src/main/java/org/apache/lucene/analysis/synonym/GraphTokenStreamFiniteStrings.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.synonym;
-
-import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES;
-
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.FiniteStringsIterator;
-import org.apache.lucene.util.automaton.Operations;
-import org.apache.lucene.util.automaton.Transition;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Creates a list of {@link TokenStream} where each stream is the tokens that make up a finite string in graph token stream.  To do this,
- * the graph token stream is converted to an {@link Automaton} and from there we use a {@link FiniteStringsIterator} to collect the various
- * token streams for each finite string.
- */
-public class GraphTokenStreamFiniteStrings {
-    private final Automaton.Builder builder;
-    Automaton det;
-    private final Map<BytesRef, Integer> termToID = new HashMap<>();
-    private final Map<Integer, BytesRef> idToTerm = new HashMap<>();
-    private int anyTermID = -1;
-
-    public GraphTokenStreamFiniteStrings() {
-        this.builder = new Automaton.Builder();
-    }
-
-    private static class BytesRefArrayTokenStream extends TokenStream {
-        private final BytesTermAttribute termAtt = addAttribute(BytesTermAttribute.class);
-        private final BytesRef[] terms;
-        private int offset;
-
-        BytesRefArrayTokenStream(BytesRef[] terms) {
-            this.terms = terms;
-            offset = 0;
-        }
-
-        @Override
-        public boolean incrementToken() throws IOException {
-            if (offset < terms.length) {
-                clearAttributes();
-                termAtt.setBytesRef(terms[offset]);
-                offset = offset + 1;
-                return true;
-            }
-
-            return false;
-        }
-    }
-
-    /**
-     * Gets
-     */
-    public List<TokenStream> getTokenStreams(final TokenStream in) throws IOException {
-        // build automation
-        build(in);
-
-        List<TokenStream> tokenStreams = new ArrayList<>();
-        final FiniteStringsIterator finiteStrings = new FiniteStringsIterator(det);
-        for (IntsRef string; (string = finiteStrings.next()) != null; ) {
-            final BytesRef[] tokens = new BytesRef[string.length];
-            for (int idx = string.offset, len = string.offset + string.length; idx < len; idx++) {
-                tokens[idx - string.offset] = idToTerm.get(string.ints[idx]);
-            }
-
-            tokenStreams.add(new BytesRefArrayTokenStream(tokens));
-        }
-
-        return tokenStreams;
-    }
-
-    private void build(final TokenStream in) throws IOException {
-        if (det != null) {
-            throw new IllegalStateException("Automation already built");
-        }
-
-        final TermToBytesRefAttribute termBytesAtt = in.addAttribute(TermToBytesRefAttribute.class);
-        final PositionIncrementAttribute posIncAtt = in.addAttribute(PositionIncrementAttribute.class);
-        final PositionLengthAttribute posLengthAtt = in.addAttribute(PositionLengthAttribute.class);
-        final OffsetAttribute offsetAtt = in.addAttribute(OffsetAttribute.class);
-
-        in.reset();
-
-        int pos = -1;
-        int lastPos = 0;
-        int maxOffset = 0;
-        int maxPos = -1;
-        int state = -1;
-        while (in.incrementToken()) {
-            int posInc = posIncAtt.getPositionIncrement();
-            assert pos > -1 || posInc > 0;
-
-            if (posInc > 1) {
-                throw new IllegalArgumentException("cannot handle holes; to accept any term, use '*' term");
-            }
-
-            if (posInc > 0) {
-                // New node:
-                pos += posInc;
-            }
-
-            int endPos = pos + posLengthAtt.getPositionLength();
-            while (state < endPos) {
-                state = createState();
-            }
-
-            BytesRef term = termBytesAtt.getBytesRef();
-            //System.out.println(pos + "-" + endPos + ": " + term.utf8ToString() + ": posInc=" + posInc);
-            if (term.length == 1 && term.bytes[term.offset] == (byte) '*') {
-                addAnyTransition(pos, endPos);
-            } else {
-                addTransition(pos, endPos, term);
-            }
-
-            maxOffset = Math.max(maxOffset, offsetAtt.endOffset());
-            maxPos = Math.max(maxPos, endPos);
-        }
-
-        in.end();
-
-        // TODO: look at endOffset?  ts2a did...
-
-        // TODO: this (setting "last" state as the only accept state) may be too simplistic?
-        setAccept(state, true);
-        finish();
-    }
-
-    /**
-     * Returns a new state; state 0 is always the initial state.
-     */
-    private int createState() {
-        return builder.createState();
-    }
-
-    /**
-     * Marks the specified state as accept or not.
-     */
-    private void setAccept(int state, boolean accept) {
-        builder.setAccept(state, accept);
-    }
-
-    /**
-     * Adds a transition to the automaton.
-     */
-    private void addTransition(int source, int dest, String term) {
-        addTransition(source, dest, new BytesRef(term));
-    }
-
-    /**
-     * Adds a transition to the automaton.
-     */
-    private void addTransition(int source, int dest, BytesRef term) {
-        if (term == null) {
-            throw new NullPointerException("term should not be null");
-        }
-        builder.addTransition(source, dest, getTermID(term));
-    }
-
-    /**
-     * Adds a transition matching any term.
-     */
-    private void addAnyTransition(int source, int dest) {
-        builder.addTransition(source, dest, getTermID(null));
-    }
-
-    /**
-     * Call this once you are done adding states/transitions.
-     */
-    private void finish() {
-        finish(DEFAULT_MAX_DETERMINIZED_STATES);
-    }
-
-    /**
-     * Call this once you are done adding states/transitions.
-     *
-     * @param maxDeterminizedStates Maximum number of states created when determinizing the automaton.  Higher numbers allow this operation
-     *                              to consume more memory but allow more complex automatons.
-     */
-    private void finish(int maxDeterminizedStates) {
-        Automaton automaton = builder.finish();
-
-        // System.out.println("before det:\n" + automaton.toDot());
-
-        Transition t = new Transition();
-
-        // TODO: should we add "eps back to initial node" for all states,
-        // and det that?  then we don't need to revisit initial node at
-        // every position?  but automaton could blow up?  And, this makes it
-        // harder to skip useless positions at search time?
-
-        if (anyTermID != -1) {
-
-            // Make sure there are no leading or trailing ANY:
-            int count = automaton.initTransition(0, t);
-            for (int i = 0; i < count; i++) {
-                automaton.getNextTransition(t);
-                if (anyTermID >= t.min && anyTermID <= t.max) {
-                    throw new IllegalStateException("automaton cannot lead with an ANY transition");
-                }
-            }
-
-            int numStates = automaton.getNumStates();
-            for (int i = 0; i < numStates; i++) {
-                count = automaton.initTransition(i, t);
-                for (int j = 0; j < count; j++) {
-                    automaton.getNextTransition(t);
-                    if (automaton.isAccept(t.dest) && anyTermID >= t.min && anyTermID <= t.max) {
-                        throw new IllegalStateException("automaton cannot end with an ANY transition");
-                    }
-                }
-            }
-
-            int termCount = termToID.size();
-
-            // We have to carefully translate these transitions so automaton
-            // realizes they also match all other terms:
-            Automaton newAutomaton = new Automaton();
-            for (int i = 0; i < numStates; i++) {
-                newAutomaton.createState();
-                newAutomaton.setAccept(i, automaton.isAccept(i));
-            }
-
-            for (int i = 0; i < numStates; i++) {
-                count = automaton.initTransition(i, t);
-                for (int j = 0; j < count; j++) {
-                    automaton.getNextTransition(t);
-                    int min, max;
-                    if (t.min <= anyTermID && anyTermID <= t.max) {
-                        // Match any term
-                        min = 0;
-                        max = termCount - 1;
-                    } else {
-                        min = t.min;
-                        max = t.max;
-                    }
-                    newAutomaton.addTransition(t.source, t.dest, min, max);
-                }
-            }
-            newAutomaton.finishState();
-            automaton = newAutomaton;
-        }
-
-        det = Operations.removeDeadStates(Operations.determinize(automaton, maxDeterminizedStates));
-    }
-
-    private int getTermID(BytesRef term) {
-        Integer id = termToID.get(term);
-        if (id == null) {
-            id = termToID.size();
-            if (term != null) {
-                term = BytesRef.deepCopyOf(term);
-            }
-            termToID.put(term, id);
-            idToTerm.put(id, term);
-            if (term == null) {
-                anyTermID = id;
-            }
-        }
-
-        return id;
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java b/core/src/main/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java
deleted file mode 100644
index f2c27679ab6..00000000000
--- a/core/src/main/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.synonym;
-
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.apache.lucene.util.RollingBuffer;
-import org.apache.lucene.util.fst.FST;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-// TODO: maybe we should resolve token -> wordID then run
-// FST on wordIDs, for better perf?
-
-// TODO: a more efficient approach would be Aho/Corasick's
-// algorithm
-// http://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm
-// It improves over the current approach here
-// because it does not fully re-start matching at every
-// token.  For example if one pattern is "a b c x"
-// and another is "b c d" and the input is "a b c d", on
-// trying to parse "a b c x" but failing when you got to x,
-// rather than starting over again your really should
-// immediately recognize that "b c d" matches at the next
-// input.  I suspect this won't matter that much in
-// practice, but it's possible on some set of synonyms it
-// will.  We'd have to modify Aho/Corasick to enforce our
-// conflict resolving (eg greedy matching) because that algo
-// finds all matches.  This really amounts to adding a .*
-// closure to the FST and then determinizing it.
-//
-// Another possible solution is described at http://www.cis.uni-muenchen.de/people/Schulz/Pub/dictle5.ps
-
-/**
- * Applies single- or multi-token synonyms from a {@link SynonymMap}
- * to an incoming {@link TokenStream}, producing a fully correct graph
- * output.  This is a replacement for {@link SynonymFilter}, which produces
- * incorrect graphs for multi-token synonyms.
- *
- * <b>NOTE</b>: this cannot consume an incoming graph; results will
- * be undefined.
- */
-public final class SynonymGraphFilter extends TokenFilter {
-
-    public static final String TYPE_SYNONYM = "SYNONYM";
-    public static final int GRAPH_FLAG = 8;
-
-    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
-    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-    private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
-    private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
-
-    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
-    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
-
-    private final SynonymMap synonyms;
-    private final boolean ignoreCase;
-
-    private final FST<BytesRef> fst;
-
-    private final FST.BytesReader fstReader;
-    private final FST.Arc<BytesRef> scratchArc;
-    private final ByteArrayDataInput bytesReader = new ByteArrayDataInput();
-    private final BytesRef scratchBytes = new BytesRef();
-    private final CharsRefBuilder scratchChars = new CharsRefBuilder();
-    private final LinkedList<BufferedOutputToken> outputBuffer = new LinkedList<>();
-
-    private int nextNodeOut;
-    private int lastNodeOut;
-    private int maxLookaheadUsed;
-
-    // For testing:
-    private int captureCount;
-
-    private boolean liveToken;
-
-    // Start/end offset of the current match:
-    private int matchStartOffset;
-    private int matchEndOffset;
-
-    // True once the input TokenStream is exhausted:
-    private boolean finished;
-
-    private int lookaheadNextRead;
-    private int lookaheadNextWrite;
-
-    private RollingBuffer<BufferedInputToken> lookahead = new RollingBuffer<BufferedInputToken>() {
-        @Override
-        protected BufferedInputToken newInstance() {
-            return new BufferedInputToken();
-        }
-    };
-
-    static class BufferedInputToken implements RollingBuffer.Resettable {
-        final CharsRefBuilder term = new CharsRefBuilder();
-        AttributeSource.State state;
-        int startOffset = -1;
-        int endOffset = -1;
-
-        @Override
-        public void reset() {
-            state = null;
-            term.clear();
-
-            // Intentionally invalid to ferret out bugs:
-            startOffset = -1;
-            endOffset = -1;
-        }
-    }
-
-    static class BufferedOutputToken {
-        final String term;
-
-        // Non-null if this was an incoming token:
-        final State state;
-
-        final int startNode;
-        final int endNode;
-
-        public BufferedOutputToken(State state, String term, int startNode, int endNode) {
-            this.state = state;
-            this.term = term;
-            this.startNode = startNode;
-            this.endNode = endNode;
-        }
-    }
-
-    public SynonymGraphFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) {
-        super(input);
-        this.synonyms = synonyms;
-        this.fst = synonyms.fst;
-        if (fst == null) {
-            throw new IllegalArgumentException("fst must be non-null");
-        }
-        this.fstReader = fst.getBytesReader();
-        scratchArc = new FST.Arc<>();
-        this.ignoreCase = ignoreCase;
-    }
-
-    @Override
-    public boolean incrementToken() throws IOException {
-        //System.out.println("\nS: incrToken lastNodeOut=" + lastNodeOut + " nextNodeOut=" + nextNodeOut);
-
-        assert lastNodeOut <= nextNodeOut;
-
-        if (outputBuffer.isEmpty() == false) {
-            // We still have pending outputs from a prior synonym match:
-            releaseBufferedToken();
-            //System.out.println("  syn: ret buffered=" + this);
-            assert liveToken == false;
-            return true;
-        }
-
-        // Try to parse a new synonym match at the current token:
-
-        if (parse()) {
-            // A new match was found:
-            releaseBufferedToken();
-            //System.out.println("  syn: after parse, ret buffered=" + this);
-            assert liveToken == false;
-            return true;
-        }
-
-        if (lookaheadNextRead == lookaheadNextWrite) {
-
-            // Fast path: parse pulled one token, but it didn't match
-            // the start for any synonym, so we now return it "live" w/o having
-            // cloned all of its atts:
-            if (finished) {
-                //System.out.println("  syn: ret END");
-                return false;
-            }
-
-            assert liveToken;
-            liveToken = false;
-
-            // NOTE: no need to change posInc since it's relative, i.e. whatever
-            // node our output is upto will just increase by the incoming posInc.
-            // We also don't need to change posLen, but only because we cannot
-            // consume a graph, so the incoming token can never span a future
-            // synonym match.
-
-        } else {
-            // We still have buffered lookahead tokens from a previous
-            // parse attempt that required lookahead; just replay them now:
-            //System.out.println("  restore buffer");
-            assert lookaheadNextRead < lookaheadNextWrite : "read=" + lookaheadNextRead + " write=" + lookaheadNextWrite;
-            BufferedInputToken token = lookahead.get(lookaheadNextRead);
-            lookaheadNextRead++;
-
-            restoreState(token.state);
-
-            lookahead.freeBefore(lookaheadNextRead);
-
-            //System.out.println("  after restore offset=" + offsetAtt.startOffset() + "-" + offsetAtt.endOffset());
-            assert liveToken == false;
-        }
-
-        lastNodeOut += posIncrAtt.getPositionIncrement();
-        nextNodeOut = lastNodeOut + posLenAtt.getPositionLength();
-
-        //System.out.println("  syn: ret lookahead=" + this);
-
-        return true;
-    }
-
-    private void releaseBufferedToken() throws IOException {
-        //System.out.println("  releaseBufferedToken");
-
-        BufferedOutputToken token = outputBuffer.pollFirst();
-
-        if (token.state != null) {
-            // This is an original input token (keepOrig=true case):
-            //System.out.println("    hasState");
-            restoreState(token.state);
-            //System.out.println("    startOffset=" + offsetAtt.startOffset() + " endOffset=" + offsetAtt.endOffset());
-        } else {
-            clearAttributes();
-            //System.out.println("    no state");
-            termAtt.append(token.term);
-
-            // We better have a match already:
-            assert matchStartOffset != -1;
-
-            offsetAtt.setOffset(matchStartOffset, matchEndOffset);
-            //System.out.println("    startOffset=" + matchStartOffset + " endOffset=" + matchEndOffset);
-            typeAtt.setType(TYPE_SYNONYM);
-        }
-
-        //System.out.println("    lastNodeOut=" + lastNodeOut);
-        //System.out.println("    term=" + termAtt);
-
-        posIncrAtt.setPositionIncrement(token.startNode - lastNodeOut);
-        lastNodeOut = token.startNode;
-        posLenAtt.setPositionLength(token.endNode - token.startNode);
-        flagsAtt.setFlags(flagsAtt.getFlags() | GRAPH_FLAG);  // set the graph flag
-    }
-
-    /**
-     * Scans the next input token(s) to see if a synonym matches.  Returns true
-     * if a match was found.
-     */
-    private boolean parse() throws IOException {
-        // System.out.println(Thread.currentThread().getName() + ": S: parse: " + System.identityHashCode(this));
-
-        // Holds the longest match we've seen so far:
-        BytesRef matchOutput = null;
-        int matchInputLength = 0;
-
-        BytesRef pendingOutput = fst.outputs.getNoOutput();
-        fst.getFirstArc(scratchArc);
-
-        assert scratchArc.output == fst.outputs.getNoOutput();
-
-        // How many tokens in the current match
-        int matchLength = 0;
-        boolean doFinalCapture = false;
-
-        int lookaheadUpto = lookaheadNextRead;
-        matchStartOffset = -1;
-
-        byToken:
-        while (true) {
-            //System.out.println("  cycle lookaheadUpto=" + lookaheadUpto + " maxPos=" + lookahead.getMaxPos());
-
-            // Pull next token's chars:
-            final char[] buffer;
-            final int bufferLen;
-            final int inputEndOffset;
-
-            if (lookaheadUpto <= lookahead.getMaxPos()) {
-                // Still in our lookahead buffer
-                BufferedInputToken token = lookahead.get(lookaheadUpto);
-                lookaheadUpto++;
-                buffer = token.term.chars();
-                bufferLen = token.term.length();
-                inputEndOffset = token.endOffset;
-                //System.out.println("    use buffer now max=" + lookahead.getMaxPos());
-                if (matchStartOffset == -1) {
-                    matchStartOffset = token.startOffset;
-                }
-            } else {
-
-                // We used up our lookahead buffer of input tokens
-                // -- pull next real input token:
-
-                assert finished || liveToken == false;
-
-                if (finished) {
-                    //System.out.println("    break: finished");
-                    break;
-                } else if (input.incrementToken()) {
-                    //System.out.println("    input.incrToken");
-                    liveToken = true;
-                    buffer = termAtt.buffer();
-                    bufferLen = termAtt.length();
-                    if (matchStartOffset == -1) {
-                        matchStartOffset = offsetAtt.startOffset();
-                    }
-                    inputEndOffset = offsetAtt.endOffset();
-
-                    lookaheadUpto++;
-                } else {
-                    // No more input tokens
-                    finished = true;
-                    //System.out.println("    break: now set finished");
-                    break;
-                }
-            }
-
-            matchLength++;
-            //System.out.println("    cycle term=" + new String(buffer, 0, bufferLen));
-
-            // Run each char in this token through the FST:
-            int bufUpto = 0;
-            while (bufUpto < bufferLen) {
-                final int codePoint = Character.codePointAt(buffer, bufUpto, bufferLen);
-                if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc, fstReader) ==
-                    null) {
-                    break byToken;
-                }
-
-                // Accum the output
-                pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output);
-                bufUpto += Character.charCount(codePoint);
-            }
-
-            assert bufUpto == bufferLen;
-
-            // OK, entire token matched; now see if this is a final
-            // state in the FST (a match):
-            if (scratchArc.isFinal()) {
-                matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput);
-                matchInputLength = matchLength;
-                matchEndOffset = inputEndOffset;
-                //System.out.println("    ** match");
-            }
-
-            // See if the FST can continue matching (ie, needs to
-            // see the next input token):
-            if (fst.findTargetArc(SynonymMap.WORD_SEPARATOR, scratchArc, scratchArc, fstReader) == null) {
-                // No further rules can match here; we're done
-                // searching for matching rules starting at the
-                // current input position.
-                break;
-            } else {
-                // More matching is possible -- accum the output (if
-                // any) of the WORD_SEP arc:
-                pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output);
-                doFinalCapture = true;
-                if (liveToken) {
-                    capture();
-                }
-            }
-        }
-
-        if (doFinalCapture && liveToken && finished == false) {
-            // Must capture the final token if we captured any prior tokens:
-            capture();
-        }
-
-        if (matchOutput != null) {
-
-            if (liveToken) {
-                // Single input token synonym; we must buffer it now:
-                capture();
-            }
-
-            // There is a match!
-            bufferOutputTokens(matchOutput, matchInputLength);
-            lookaheadNextRead += matchInputLength;
-            //System.out.println("  precmatch; set lookaheadNextRead=" + lookaheadNextRead + " now max=" + lookahead.getMaxPos());
-            lookahead.freeBefore(lookaheadNextRead);
-            //System.out.println("  match; set lookaheadNextRead=" + lookaheadNextRead + " now max=" + lookahead.getMaxPos());
-            return true;
-        } else {
-            //System.out.println("  no match; lookaheadNextRead=" + lookaheadNextRead);
-            return false;
-        }
-
-        //System.out.println("  parse done inputSkipCount=" + inputSkipCount + " nextRead=" + nextRead + " nextWrite=" + nextWrite);
-    }
-
-    /**
-     * Expands the output graph into the necessary tokens, adding
-     * synonyms as side paths parallel to the input tokens, and
-     * buffers them in the output token buffer.
-     */
-    private void bufferOutputTokens(BytesRef bytes, int matchInputLength) {
-        bytesReader.reset(bytes.bytes, bytes.offset, bytes.length);
-
-        final int code = bytesReader.readVInt();
-        final boolean keepOrig = (code & 0x1) == 0;
-        //System.out.println("  buffer: keepOrig=" + keepOrig + " matchInputLength=" + matchInputLength);
-
-        // How many nodes along all paths; we need this to assign the
-        // node ID for the final end node where all paths merge back:
-        int totalPathNodes;
-        if (keepOrig) {
-            assert matchInputLength > 0;
-            totalPathNodes = matchInputLength - 1;
-        } else {
-            totalPathNodes = 0;
-        }
-
-        // How many synonyms we will insert over this match:
-        final int count = code >>> 1;
-
-        // TODO: we could encode this instead into the FST:
-
-        // 1st pass: count how many new nodes we need
-        List<List<String>> paths = new ArrayList<>();
-        for (int outputIDX = 0; outputIDX < count; outputIDX++) {
-            int wordID = bytesReader.readVInt();
-            synonyms.words.get(wordID, scratchBytes);
-            scratchChars.copyUTF8Bytes(scratchBytes);
-            int lastStart = 0;
-
-            List<String> path = new ArrayList<>();
-            paths.add(path);
-            int chEnd = scratchChars.length();
-            for (int chUpto = 0; chUpto <= chEnd; chUpto++) {
-                if (chUpto == chEnd || scratchChars.charAt(chUpto) == SynonymMap.WORD_SEPARATOR) {
-                    path.add(new String(scratchChars.chars(), lastStart, chUpto - lastStart));
-                    lastStart = 1 + chUpto;
-                }
-            }
-
-            assert path.size() > 0;
-            totalPathNodes += path.size() - 1;
-        }
-        //System.out.println("  totalPathNodes=" + totalPathNodes);
-
-        // 2nd pass: buffer tokens for the graph fragment
-
-        // NOTE: totalPathNodes will be 0 in the case where the matched
-        // input is a single token and all outputs are also a single token
-
-        // We "spawn" a side-path for each of the outputs for this matched
-        // synonym, all ending back at this end node:
-
-        int startNode = nextNodeOut;
-
-        int endNode = startNode + totalPathNodes + 1;
-        //System.out.println("  " + paths.size() + " new side-paths");
-
-        // First, fanout all tokens departing start node for these new side paths:
-        int newNodeCount = 0;
-        for (List<String> path : paths) {
-            int pathEndNode;
-            //System.out.println("    path size=" + path.size());
-            if (path.size() == 1) {
-                // Single token output, so there are no intermediate nodes:
-                pathEndNode = endNode;
-            } else {
-                pathEndNode = nextNodeOut + newNodeCount + 1;
-                newNodeCount += path.size() - 1;
-            }
-            outputBuffer.add(new BufferedOutputToken(null, path.get(0), startNode, pathEndNode));
-        }
-
-        // We must do the original tokens last, else the offsets "go backwards":
-        if (keepOrig) {
-            BufferedInputToken token = lookahead.get(lookaheadNextRead);
-            int inputEndNode;
-            if (matchInputLength == 1) {
-                // Single token matched input, so there are no intermediate nodes:
-                inputEndNode = endNode;
-            } else {
-                inputEndNode = nextNodeOut + newNodeCount + 1;
-            }
-
-            //System.out.println("    keepOrig first token: " + token.term);
-
-            outputBuffer.add(new BufferedOutputToken(token.state, token.term.toString(), startNode, inputEndNode));
-        }
-
-        nextNodeOut = endNode;
-
-        // Do full side-path for each syn output:
-        for (int pathID = 0; pathID < paths.size(); pathID++) {
-            List<String> path = paths.get(pathID);
-            if (path.size() > 1) {
-                int lastNode = outputBuffer.get(pathID).endNode;
-                for (int i = 1; i < path.size() - 1; i++) {
-                    outputBuffer.add(new BufferedOutputToken(null, path.get(i), lastNode, lastNode + 1));
-                    lastNode++;
-                }
-                outputBuffer.add(new BufferedOutputToken(null, path.get(path.size() - 1), lastNode, endNode));
-            }
-        }
-
-        if (keepOrig && matchInputLength > 1) {
-            // Do full "side path" with the original tokens:
-            int lastNode = outputBuffer.get(paths.size()).endNode;
-            for (int i = 1; i < matchInputLength - 1; i++) {
-                BufferedInputToken token = lookahead.get(lookaheadNextRead + i);
-                outputBuffer.add(new BufferedOutputToken(token.state, token.term.toString(), lastNode, lastNode + 1));
-                lastNode++;
-            }
-            BufferedInputToken token = lookahead.get(lookaheadNextRead + matchInputLength - 1);
-            outputBuffer.add(new BufferedOutputToken(token.state, token.term.toString(), lastNode, endNode));
-        }
-
-    /*
-    System.out.println("  after buffer: " + outputBuffer.size() + " tokens:");
-    for(BufferedOutputToken token : outputBuffer) {
-      System.out.println("    tok: " + token.term + " startNode=" + token.startNode + " endNode=" + token.endNode);
-    }
-    */
-    }
-
-    /**
-     * Buffers the current input token into lookahead buffer.
-     */
-    private void capture() {
-        assert liveToken;
-        liveToken = false;
-        BufferedInputToken token = lookahead.get(lookaheadNextWrite);
-        lookaheadNextWrite++;
-
-        token.state = captureState();
-        token.startOffset = offsetAtt.startOffset();
-        token.endOffset = offsetAtt.endOffset();
-        assert token.term.length() == 0;
-        token.term.append(termAtt);
-
-        captureCount++;
-        maxLookaheadUsed = Math.max(maxLookaheadUsed, lookahead.getBufferSize());
-        //System.out.println("  maxLookaheadUsed=" + maxLookaheadUsed);
-    }
-
-    @Override
-    public void reset() throws IOException {
-        super.reset();
-        lookahead.reset();
-        lookaheadNextWrite = 0;
-        lookaheadNextRead = 0;
-        captureCount = 0;
-        lastNodeOut = -1;
-        nextNodeOut = 0;
-        matchStartOffset = -1;
-        matchEndOffset = -1;
-        finished = false;
-        liveToken = false;
-        outputBuffer.clear();
-        maxLookaheadUsed = 0;
-        //System.out.println("S: reset");
-    }
-
-    // for testing
-    int getCaptureCount() {
-        return captureCount;
-    }
-
-    // for testing
-    int getMaxLookaheadUsed() {
-        return maxLookaheadUsed;
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
index 976c4706725..c1998c65000 100644
--- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
+++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java
@@ -19,6 +19,9 @@
 
 package org.apache.lucene.queryparser.classic;
 
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
+
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -30,6 +33,7 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -55,9 +59,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
-
 /**
  * A query parser that uses the {@link MapperService} in order to build smarter
  * queries based on the mapping information.
@@ -739,27 +740,48 @@ public class MapperQueryParser extends AnalyzingQueryParser {
 
     private Query applySlop(Query q, int slop) {
         if (q instanceof PhraseQuery) {
-            PhraseQuery pq = (PhraseQuery) q;
-            PhraseQuery.Builder builder = new PhraseQuery.Builder();
-            builder.setSlop(slop);
-            final Term[] terms = pq.getTerms();
-            final int[] positions = pq.getPositions();
-            for (int i = 0; i < terms.length; ++i) {
-                builder.add(terms[i], positions[i]);
-            }
-            pq = builder.build();
             //make sure that the boost hasn't been set beforehand, otherwise we'd lose it
             assert q instanceof BoostQuery == false;
-            return pq;
+            return addSlopToPhrase((PhraseQuery) q, slop);
         } else if (q instanceof MultiPhraseQuery) {
             MultiPhraseQuery.Builder builder = new MultiPhraseQuery.Builder((MultiPhraseQuery) q);
             builder.setSlop(slop);
             return builder.build();
+        } else if (q instanceof GraphQuery && ((GraphQuery) q).hasPhrase()) {
+            // we have a graph query that has at least one phrase sub-query
+            // re-build and set slop on all phrase queries
+            List<Query> oldQueries = ((GraphQuery) q).getQueries();
+            Query[] queries = new Query[oldQueries.size()];
+            for (int i = 0; i < queries.length; i++) {
+                Query oldQuery = oldQueries.get(i);
+                if (oldQuery instanceof PhraseQuery) {
+                    queries[i] = addSlopToPhrase((PhraseQuery) oldQuery, slop);
+                } else {
+                    queries[i] = oldQuery;
+                }
+            }
+
+            return new GraphQuery(queries);
         } else {
             return q;
         }
     }
 
+    /**
+     * Rebuild a phrase query with a slop value
+     */
+    private PhraseQuery addSlopToPhrase(PhraseQuery query, int slop) {
+        PhraseQuery.Builder builder = new PhraseQuery.Builder();
+        builder.setSlop(slop);
+        final Term[] terms = query.getTerms();
+        final int[] positions = query.getPositions();
+        for (int i = 0; i < terms.length; ++i) {
+            builder.add(terms[i], positions[i]);
+        }
+
+        return builder.build();
+    }
+
     private Collection<String> extractMultiFields(String field) {
         Collection<String> fields;
         if (field != null) {
diff --git a/core/src/main/java/org/apache/lucene/search/GraphQuery.java b/core/src/main/java/org/apache/lucene/search/GraphQuery.java
deleted file mode 100644
index cad316d701c..00000000000
--- a/core/src/main/java/org/apache/lucene/search/GraphQuery.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.search;
-
-import org.apache.lucene.index.IndexReader;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * A query that wraps multiple sub-queries generated from a graph token stream.
- */
-public final class GraphQuery extends Query {
-    private final Query[] queries;
-    private final boolean hasBoolean;
-
-    /**
-     * Constructor sets the queries and checks if any of them are
-     * a boolean query.
-     *
-     * @param queries the non-null array of queries
-     */
-    public GraphQuery(Query... queries) {
-        this.queries = Objects.requireNonNull(queries).clone();
-        for (Query query : queries) {
-            if (query instanceof BooleanQuery) {
-                hasBoolean = true;
-                return;
-            }
-        }
-        hasBoolean = false;
-    }
-
-    /**
-     * Gets the queries
-     *
-     * @return unmodifiable list of Query
-     */
-    public List<Query> getQueries() {
-        return Collections.unmodifiableList(Arrays.asList(queries));
-    }
-
-    /**
-     * If there is at least one boolean query or not.
-     *
-     * @return true if there is a boolean, false if not
-     */
-    public boolean hasBoolean() {
-        return hasBoolean;
-    }
-
-    /**
-     * Rewrites to a single query or a boolean query where each query is a SHOULD clause.
-     */
-    @Override
-    public Query rewrite(IndexReader reader) throws IOException {
-        if (queries.length == 0) {
-            return new BooleanQuery.Builder().build();
-        }
-
-        if (queries.length == 1) {
-            return queries[0];
-        }
-
-        BooleanQuery.Builder q = new BooleanQuery.Builder();
-        q.setDisableCoord(true);
-        for (Query clause : queries) {
-            q.add(clause, BooleanClause.Occur.SHOULD);
-        }
-
-        return q.build();
-    }
-
-    @Override
-    public String toString(String field) {
-        StringBuilder builder = new StringBuilder("Graph(");
-        for (int i = 0; i < queries.length; i++) {
-            if (i != 0) {
-                builder.append(", ");
-            }
-            builder.append(Objects.toString(queries[i]));
-        }
-        builder.append(")");
-        return builder.toString();
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        return sameClassAs(other) &&
-            Arrays.equals(queries, ((GraphQuery) other).queries);
-    }
-
-    @Override
-    public int hashCode() {
-        return 31 * classHash() + Arrays.hashCode(queries);
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java
new file mode 100644
index 00000000000..169a89edbcf
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.util.PriorityQueue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Represents hits returned by {@link CollapsingTopDocsCollector#getTopDocs()}.
+ */
+public class CollapseTopFieldDocs extends TopFieldDocs {
+    /** The field used for collapsing **/
+    public final String field;
+    /** The collapse value for each top doc */
+    public final Object[] collapseValues;
+
+    public CollapseTopFieldDocs(String field, int totalHits, ScoreDoc[] scoreDocs,
+                                SortField[] sortFields, Object[] values, float maxScore) {
+        super(totalHits, scoreDocs, sortFields, maxScore);
+        this.field = field;
+        this.collapseValues = values;
+    }
+
+    // Refers to one hit:
+    private static class ShardRef {
+        // Which shard (index into shardHits[]):
+        final int shardIndex;
+
+        // Which hit within the shard:
+        int hitIndex;
+
+        ShardRef(int shardIndex) {
+            this.shardIndex = shardIndex;
+        }
+
+        @Override
+        public String toString() {
+            return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")";
+        }
+    };
+
+    private static class MergeSortQueue extends PriorityQueue<ShardRef> {
+        // These are really FieldDoc instances:
+        final ScoreDoc[][] shardHits;
+        final FieldComparator<?>[] comparators;
+        final int[] reverseMul;
+
+        MergeSortQueue(Sort sort, CollapseTopFieldDocs[] shardHits) throws IOException {
+            super(shardHits.length);
+            this.shardHits = new ScoreDoc[shardHits.length][];
+            for (int shardIDX = 0; shardIDX < shardHits.length; shardIDX++) {
+                final ScoreDoc[] shard = shardHits[shardIDX].scoreDocs;
+                if (shard != null) {
+                    this.shardHits[shardIDX] = shard;
+                    // Fail gracefully if API is misused:
+                    for (int hitIDX = 0; hitIDX < shard.length; hitIDX++) {
+                        final ScoreDoc sd = shard[hitIDX];
+                        final FieldDoc gd = (FieldDoc) sd;
+                        assert gd.fields != null;
+                    }
+                }
+            }
+
+            final SortField[] sortFields = sort.getSort();
+            comparators = new FieldComparator[sortFields.length];
+            reverseMul = new int[sortFields.length];
+            for (int compIDX = 0; compIDX < sortFields.length; compIDX++) {
+                final SortField sortField = sortFields[compIDX];
+                comparators[compIDX] = sortField.getComparator(1, compIDX);
+                reverseMul[compIDX] = sortField.getReverse() ? -1 : 1;
+            }
+        }
+
+        // Returns true if first is < second
+        @Override
+        public boolean lessThan(ShardRef first, ShardRef second) {
+            assert first != second;
+            final FieldDoc firstFD = (FieldDoc) shardHits[first.shardIndex][first.hitIndex];
+            final FieldDoc secondFD = (FieldDoc) shardHits[second.shardIndex][second.hitIndex];
+
+            for (int compIDX = 0; compIDX < comparators.length; compIDX++) {
+                final FieldComparator comp = comparators[compIDX];
+
+                final int cmp =
+                    reverseMul[compIDX] * comp.compareValues(firstFD.fields[compIDX], secondFD.fields[compIDX]);
+
+                if (cmp != 0) {
+                    return cmp < 0;
+                }
+            }
+
+            // Tie break: earlier shard wins
+            if (first.shardIndex < second.shardIndex) {
+                return true;
+            } else if (first.shardIndex > second.shardIndex) {
+                return false;
+            } else {
+                // Tie break in same shard: resolve however the
+                // shard had resolved it:
+                assert first.hitIndex != second.hitIndex;
+                return first.hitIndex < second.hitIndex;
+            }
+        }
+    }
+
+    /**
+     * Returns a new CollapseTopDocs, containing topN collapsed results across
+     * the provided CollapseTopDocs, sorting by score. Each {@link CollapseTopFieldDocs} instance must be sorted.
+     **/
+    public static CollapseTopFieldDocs merge(Sort sort, int start, int size,
+                                             CollapseTopFieldDocs[] shardHits) throws IOException {
+        String collapseField = shardHits[0].field;
+        for (int i = 1; i < shardHits.length; i++) {
+            if (collapseField.equals(shardHits[i].field) == false) {
+                throw new IllegalArgumentException("collapse field differ across shards [" +
+                    collapseField + "] != [" + shardHits[i].field + "]");
+            }
+        }
+        final PriorityQueue<ShardRef> queue = new MergeSortQueue(sort, shardHits);
+
+        int totalHitCount = 0;
+        int availHitCount = 0;
+        float maxScore = Float.MIN_VALUE;
+        for(int shardIDX=0;shardIDX<shardHits.length;shardIDX++) {
+            final CollapseTopFieldDocs shard = shardHits[shardIDX];
+            // totalHits can be non-zero even if no hits were
+            // collected, when searchAfter was used:
+            totalHitCount += shard.totalHits;
+            if (shard.scoreDocs != null && shard.scoreDocs.length > 0) {
+                availHitCount += shard.scoreDocs.length;
+                queue.add(new ShardRef(shardIDX));
+                maxScore = Math.max(maxScore, shard.getMaxScore());
+            }
+        }
+
+        if (availHitCount == 0) {
+            maxScore = Float.NaN;
+        }
+
+        final ScoreDoc[] hits;
+        final Object[] values;
+        if (availHitCount <= start) {
+            hits = new ScoreDoc[0];
+            values = new Object[0];
+        } else {
+            List<ScoreDoc> hitList = new ArrayList<>();
+            List<Object> collapseList = new ArrayList<>();
+            int requestedResultWindow = start + size;
+            int numIterOnHits = Math.min(availHitCount, requestedResultWindow);
+            int hitUpto = 0;
+            Set<Object> seen = new HashSet<>();
+            while (hitUpto < numIterOnHits) {
+                if (queue.size() == 0) {
+                    break;
+                }
+                ShardRef ref = queue.top();
+                final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex];
+                final Object collapseValue = shardHits[ref.shardIndex].collapseValues[ref.hitIndex++];
+                if (seen.contains(collapseValue)) {
+                    if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+                        queue.updateTop();
+                    } else {
+                        queue.pop();
+                    }
+                    continue;
+                }
+                seen.add(collapseValue);
+                hit.shardIndex = ref.shardIndex;
+                if (hitUpto >= start) {
+                    hitList.add(hit);
+                    collapseList.add(collapseValue);
+                }
+
+                hitUpto++;
+
+                if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) {
+                    // Not done with this these TopDocs yet:
+                    queue.updateTop();
+                } else {
+                    queue.pop();
+                }
+            }
+            hits = hitList.toArray(new ScoreDoc[0]);
+            values = collapseList.toArray(new Object[0]);
+        }
+        return new CollapseTopFieldDocs(collapseField, totalHitCount, hits, sort.getSort(), values, maxScore);
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java
new file mode 100644
index 00000000000..5bc8afb347c
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingDocValuesSource.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+
+/**
+ * Utility class that ensures that a single collapse key is extracted per document.
+ */
+abstract class CollapsingDocValuesSource<T> {
+    protected final String field;
+
+    CollapsingDocValuesSource(String field) throws IOException {
+        this.field = field;
+    }
+
+    abstract T get(int doc);
+
+    abstract T copy(T value, T reuse);
+
+    abstract void setNextReader(LeafReader reader) throws IOException;
+
+    /**
+     * Implementation for {@link NumericDocValues} and {@link SortedNumericDocValues}.
+     * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field.
+     */
+    static class Numeric extends CollapsingDocValuesSource<Long> {
+        private NumericDocValues values;
+        private Bits docsWithField;
+
+        Numeric(String field) throws IOException {
+            super(field);
+        }
+
+        @Override
+        public Long get(int doc) {
+            if (docsWithField.get(doc)) {
+                return values.get(doc);
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public Long copy(Long value, Long reuse) {
+            return value;
+        }
+
+        @Override
+        public void setNextReader(LeafReader reader) throws IOException {
+            DocValuesType type = getDocValuesType(reader, field);
+            if (type == null || type == DocValuesType.NONE) {
+                values = DocValues.emptyNumeric();
+                docsWithField = new Bits.MatchNoBits(reader.maxDoc());
+                return ;
+            }
+            docsWithField = DocValues.getDocsWithField(reader, field);
+            switch (type) {
+                case NUMERIC:
+                    values = DocValues.getNumeric(reader, field);
+                    break;
+
+                case SORTED_NUMERIC:
+                    final SortedNumericDocValues sorted = DocValues.getSortedNumeric(reader, field);
+                    values = DocValues.unwrapSingleton(sorted);
+                    if (values == null) {
+                        values = new NumericDocValues() {
+                            @Override
+                            public long get(int docID) {
+                                sorted.setDocument(docID);
+                                assert sorted.count() > 0;
+                                if (sorted.count() > 1) {
+                                    throw new IllegalStateException("failed to collapse " + docID +
+                                        ", the collapse field must be single valued");
+                                }
+                                return sorted.valueAt(0);
+                            }
+                        };
+                    }
+                    break;
+
+                default:
+                    throw new IllegalStateException("unexpected doc values type " +
+                        type + "` for field `" + field + "`");
+            }
+        }
+    }
+
+    /**
+     * Implementation for {@link SortedDocValues} and {@link SortedSetDocValues}.
+     * Fails with an {@link IllegalStateException} if a document contains multiple values for the specified field.
+     */
+    static class Keyword extends CollapsingDocValuesSource<BytesRef> {
+        private Bits docsWithField;
+        private SortedDocValues values;
+
+        Keyword(String field) throws IOException {
+            super(field);
+        }
+
+        @Override
+        public BytesRef get(int doc) {
+            if (docsWithField.get(doc)) {
+                return values.get(doc);
+            } else {
+                return null;
+            }
+        }
+
+        @Override
+        public BytesRef copy(BytesRef value, BytesRef reuse) {
+            if (value == null) {
+                return null;
+            }
+            if (reuse != null) {
+                reuse.bytes = ArrayUtil.grow(reuse.bytes, value.length);
+                reuse.offset = 0;
+                reuse.length = value.length;
+                System.arraycopy(value.bytes, value.offset, reuse.bytes, 0, value.length);
+                return reuse;
+            } else {
+                return BytesRef.deepCopyOf(value);
+            }
+        }
+
+        @Override
+        public void setNextReader(LeafReader reader) throws IOException {
+            DocValuesType type = getDocValuesType(reader, field);
+            if (type == null || type == DocValuesType.NONE) {
+                values = DocValues.emptySorted();
+                docsWithField = new Bits.MatchNoBits(reader.maxDoc());
+                return ;
+            }
+            docsWithField = DocValues.getDocsWithField(reader, field);
+            switch (type) {
+                case SORTED:
+                    values = DocValues.getSorted(reader, field);
+                    break;
+
+                case SORTED_SET:
+                    final SortedSetDocValues sorted = DocValues.getSortedSet(reader, field);
+                    values = DocValues.unwrapSingleton(sorted);
+                    if (values == null) {
+                        values = new SortedDocValues() {
+                            @Override
+                            public int getOrd(int docID) {
+                                sorted.setDocument(docID);
+                                int ord = (int) sorted.nextOrd();
+                                if (sorted.nextOrd() != SortedSetDocValues.NO_MORE_ORDS) {
+                                    throw new IllegalStateException("failed to collapse " + docID +
+                                        ", the collapse field must be single valued");
+                                }
+                                return ord;
+                            }
+
+                            @Override
+                            public BytesRef lookupOrd(int ord) {
+                                return sorted.lookupOrd(ord);
+                            }
+
+                            @Override
+                            public int getValueCount() {
+                                return (int) sorted.getValueCount();
+                            }
+                        };
+                    }
+                    break;
+
+                default:
+                    throw new IllegalStateException("unexpected doc values type "
+                        + type + "` for field `" + field + "`");
+            }
+        }
+    }
+
+    private static DocValuesType getDocValuesType(LeafReader in, String field) {
+        FieldInfo fi = in.getFieldInfos().fieldInfo(field);
+        if (fi != null) {
+            return fi.getDocValuesType();
+        }
+        return null;
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
new file mode 100644
index 00000000000..955a63e5483
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/grouping/CollapsingTopDocsCollector.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.search.grouping;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+
+import static org.apache.lucene.search.SortField.Type.SCORE;
+
+/**
+ * A collector that groups documents based on field values and returns {@link CollapseTopFieldDocs}
+ * output. The collapsing is done in a single pass by selecting only the top sorted document per collapse key.
+ * The value used for the collapse key of each group can be found in {@link CollapseTopFieldDocs#collapseValues}.
+ */
+public abstract class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollector<T> {
+    protected final String collapseField;
+
+    protected final Sort sort;
+    protected Scorer scorer;
+
+    private int totalHitCount;
+    private float maxScore;
+    private final boolean trackMaxScore;
+
+    private CollapsingTopDocsCollector(String collapseField, Sort sort,
+                                       int topN, boolean trackMaxScore) throws IOException {
+        super(sort, topN);
+        this.collapseField = collapseField;
+        this.trackMaxScore = trackMaxScore;
+        if (trackMaxScore) {
+            maxScore = Float.NEGATIVE_INFINITY;
+        } else {
+            maxScore = Float.NaN;
+        }
+        this.sort = sort;
+    }
+
+    /**
+     * Transform {@link FirstPassGroupingCollector#getTopGroups(int, boolean)} output in
+     * {@link CollapseTopFieldDocs}. The collapsing needs only one pass so we can create the final top docs at the end
+     * of the first pass.
+     */
+    public CollapseTopFieldDocs getTopDocs() {
+        Collection<SearchGroup<T>> groups = super.getTopGroups(0, true);
+        if (groups == null) {
+            return new CollapseTopFieldDocs(collapseField, totalHitCount, new ScoreDoc[0],
+                sort.getSort(), new Object[0], Float.NaN);
+        }
+        FieldDoc[] docs = new FieldDoc[groups.size()];
+        Object[] collapseValues = new Object[groups.size()];
+        int scorePos = -1;
+        for (int index = 0; index < sort.getSort().length; index++) {
+            SortField sortField = sort.getSort()[index];
+            if (sortField.getType() == SCORE) {
+                scorePos = index;
+                break;
+            }
+        }
+        int pos = 0;
+        Iterator<CollectedSearchGroup<T>> it = orderedGroups.iterator();
+        for (SearchGroup<T> group : groups) {
+            assert it.hasNext();
+            CollectedSearchGroup<T> col = it.next();
+            float score = Float.NaN;
+            if (scorePos != -1) {
+                score = (float) group.sortValues[scorePos];
+            }
+            docs[pos] = new FieldDoc(col.topDoc, score, group.sortValues);
+            collapseValues[pos] = group.groupValue;
+            pos++;
+        }
+        return new CollapseTopFieldDocs(collapseField, totalHitCount, docs, sort.getSort(),
+            collapseValues, maxScore);
+    }
+
+    @Override
+    public boolean needsScores() {
+        if (super.needsScores() == false) {
+            return trackMaxScore;
+        }
+        return true;
+    }
+
+    @Override
+    public void setScorer(Scorer scorer) throws IOException {
+        super.setScorer(scorer);
+        this.scorer = scorer;
+    }
+
+    @Override
+    public void collect(int doc) throws IOException {
+        super.collect(doc);
+        if (trackMaxScore) {
+            maxScore = Math.max(maxScore, scorer.score());
+        }
+        totalHitCount++;
+    }
+
+    private static class Numeric extends CollapsingTopDocsCollector<Long> {
+        private final CollapsingDocValuesSource.Numeric source;
+
+        private Numeric(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException {
+            super(collapseField, sort, topN, trackMaxScore);
+            source = new CollapsingDocValuesSource.Numeric(collapseField);
+        }
+
+        @Override
+        protected void doSetNextReader(LeafReaderContext readerContext) throws IOException {
+            super.doSetNextReader(readerContext);
+            source.setNextReader(readerContext.reader());
+        }
+
+        @Override
+        protected Long getDocGroupValue(int doc) {
+            return source.get(doc);
+        }
+
+        @Override
+        protected Long copyDocGroupValue(Long groupValue, Long reuse) {
+            return source.copy(groupValue, reuse);
+        }
+    }
+
+    private static class Keyword extends CollapsingTopDocsCollector<BytesRef> {
+        private final CollapsingDocValuesSource.Keyword source;
+
+        private Keyword(String collapseField, Sort sort, int topN, boolean trackMaxScore) throws IOException {
+            super(collapseField, sort, topN, trackMaxScore);
+            source = new CollapsingDocValuesSource.Keyword(collapseField);
+
+        }
+
+        @Override
+        protected void doSetNextReader(LeafReaderContext readerContext) throws IOException {
+            super.doSetNextReader(readerContext);
+            source.setNextReader(readerContext.reader());
+        }
+
+        @Override
+        protected BytesRef getDocGroupValue(int doc) {
+            return source.get(doc);
+        }
+
+        @Override
+        protected BytesRef copyDocGroupValue(BytesRef groupValue, BytesRef reuse) {
+            return source.copy(groupValue, reuse);
+        }
+    }
+
+    /**
+     * Create a collapsing top docs collector on a {@link org.apache.lucene.index.NumericDocValues} field.
+     * It accepts also {@link org.apache.lucene.index.SortedNumericDocValues} field but
+     * the collect will fail with an {@link IllegalStateException} if a document contains more than one value for the
+     * field.
+     *
+     * @param collapseField The sort field used to group
+     *                      documents.
+     * @param sort          The {@link Sort} used to sort the collapsed hits.
+     *                      The collapsing keeps only the top sorted document per collapsed key.
+     *                      This must be non-null, ie, if you want to groupSort by relevance
+     *                      use Sort.RELEVANCE.
+     * @param topN          How many top groups to keep.
+     * @throws IOException When I/O related errors occur
+     */
+    public static CollapsingTopDocsCollector<?> createNumeric(String collapseField, Sort sort,
+                                                              int topN, boolean trackMaxScore) throws IOException {
+        return new Numeric(collapseField, sort, topN, trackMaxScore);
+    }
+
+    /**
+     * Create a collapsing top docs collector on a {@link org.apache.lucene.index.SortedDocValues} field.
+     * It accepts also {@link org.apache.lucene.index.SortedSetDocValues} field but
+     * the collect will fail with an {@link IllegalStateException} if a document contains more than one value for the
+     * field.
+     *
+     * @param collapseField The sort field used to group
+     *                      documents.
+     * @param sort          The {@link Sort} used to sort the collapsed hits. The collapsing keeps only the top sorted
+     *                      document per collapsed key.
+     *                      This must be non-null, ie, if you want to groupSort by relevance use Sort.RELEVANCE.
+     * @param topN          How many top groups to keep.
+     * @throws IOException When I/O related errors occur
+     */
+    public static CollapsingTopDocsCollector<?> createKeyword(String collapseField, Sort sort,
+                                                              int topN, boolean trackMaxScore) throws IOException {
+        return new Keyword(collapseField, sort, topN, trackMaxScore);
+    }
+}
+
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java b/core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
similarity index 96%
rename from core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java
rename to core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
index f3bfa1b9c65..81a3d406ea3 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/Snippet.java
+++ b/core/src/main/java/org/apache/lucene/search/highlight/Snippet.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.lucene.search.postingshighlight;
+package org.apache.lucene.search.highlight;
 
 /**
  * Represents a scored highlighted snippet.
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
index 889e7f741ed..a33bf16dee4 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
+++ b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatter.java
@@ -19,6 +19,7 @@
 
 package org.apache.lucene.search.postingshighlight;
 
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.highlight.Encoder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
 
@@ -46,10 +47,10 @@ public class CustomPassageFormatter extends PassageFormatter {
         for (int j = 0; j < passages.length; j++) {
             Passage passage = passages[j];
             StringBuilder sb = new StringBuilder();
-            pos = passage.startOffset;
-            for (int i = 0; i < passage.numMatches; i++) {
-                int start = passage.matchStarts[i];
-                int end = passage.matchEnds[i];
+            pos = passage.getStartOffset();
+            for (int i = 0; i < passage.getNumMatches(); i++) {
+                int start = passage.getMatchStarts()[i];
+                int end = passage.getMatchEnds()[i];
                 // its possible to have overlapping terms
                 if (start > pos) {
                     append(sb, content, pos, start);
@@ -62,7 +63,7 @@ public class CustomPassageFormatter extends PassageFormatter {
                 }
             }
             // its possible a "term" from the analyzer could span a sentence boundary.
-            append(sb, content, pos, Math.max(pos, passage.endOffset));
+            append(sb, content, pos, Math.max(pos, passage.getEndOffset()));
             //we remove the paragraph separator if present at the end of the snippet (we used it as separator between values)
             if (sb.charAt(sb.length() - 1) == HighlightUtils.PARAGRAPH_SEPARATOR) {
                 sb.deleteCharAt(sb.length() - 1);
@@ -70,7 +71,7 @@ public class CustomPassageFormatter extends PassageFormatter {
                 sb.deleteCharAt(sb.length() - 1);
             }
             //and we trim the snippets too
-            snippets[j] = new Snippet(sb.toString().trim(), passage.score, passage.numMatches > 0);
+            snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
         }
         return snippets;
     }
diff --git a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
index 30f57b2626c..ac90a3e57ae 100644
--- a/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
+++ b/core/src/main/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighter.java
@@ -22,6 +22,7 @@ package org.apache.lucene.search.postingshighlight;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.highlight.Snippet;
 
 import java.io.IOException;
 import java.text.BreakIterator;
diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
index c65f962dbb8..5eaf63369b9 100644
--- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
+++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
@@ -392,7 +392,7 @@ public long ramBytesUsed() {
     final BytesRefBuilder spare = new BytesRefBuilder();
     private char sepLabel;
 
-    public EscapingTokenStreamToAutomaton(char sepLabel) {
+    EscapingTokenStreamToAutomaton(char sepLabel) {
       this.sepLabel = sepLabel;
     }
 
@@ -432,7 +432,7 @@ public long ramBytesUsed() {
 
     private final boolean hasPayloads;
 
-    public AnalyzingComparator(boolean hasPayloads) {
+    AnalyzingComparator(boolean hasPayloads) {
       this.hasPayloads = hasPayloads;
     }
 
@@ -1114,7 +1114,7 @@ public long ramBytesUsed() {
             BytesRef payload;
             long weight;
 
-            public SurfaceFormAndPayload(BytesRef payload, long cost) {
+            SurfaceFormAndPayload(BytesRef payload, long cost) {
                 super();
                 this.payload = payload;
                 this.weight = cost;
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java
new file mode 100644
index 00000000000..7a34a805db6
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomPassageFormatter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.search.highlight.Encoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
+
+/**
+ * Custom passage formatter that allows us to:
+ * 1) extract different snippets (instead of a single big string) together with their scores ({@link Snippet})
+ * 2) use the {@link Encoder} implementations that are already used with the other highlighters
+ */
+public class CustomPassageFormatter extends PassageFormatter {
+
+    private final String preTag;
+    private final String postTag;
+    private final Encoder encoder;
+
+    public CustomPassageFormatter(String preTag, String postTag, Encoder encoder) {
+        this.preTag = preTag;
+        this.postTag = postTag;
+        this.encoder = encoder;
+    }
+
+    @Override
+    public Snippet[] format(Passage[] passages, String content) {
+        Snippet[] snippets = new Snippet[passages.length];
+        int pos;
+        for (int j = 0; j < passages.length; j++) {
+            Passage passage = passages[j];
+            StringBuilder sb = new StringBuilder();
+            pos = passage.getStartOffset();
+            for (int i = 0; i < passage.getNumMatches(); i++) {
+                int start = passage.getMatchStarts()[i];
+                int end = passage.getMatchEnds()[i];
+                // its possible to have overlapping terms
+                if (start > pos) {
+                    append(sb, content, pos, start);
+                }
+                if (end > pos) {
+                    sb.append(preTag);
+                    append(sb, content, Math.max(pos, start), end);
+                    sb.append(postTag);
+                    pos = end;
+                }
+            }
+            // its possible a "term" from the analyzer could span a sentence boundary.
+            append(sb, content, pos, Math.max(pos, passage.getEndOffset()));
+            //we remove the paragraph separator if present at the end of the snippet (we used it as separator between values)
+            if (sb.charAt(sb.length() - 1) == HighlightUtils.PARAGRAPH_SEPARATOR) {
+                sb.deleteCharAt(sb.length() - 1);
+            } else if (sb.charAt(sb.length() - 1) == HighlightUtils.NULL_SEPARATOR) {
+                sb.deleteCharAt(sb.length() - 1);
+            }
+            //and we trim the snippets too
+            snippets[j] = new Snippet(sb.toString().trim(), passage.getScore(), passage.getNumMatches() > 0);
+        }
+        return snippets;
+    }
+
+    private void append(StringBuilder dest, String content, int start, int end) {
+        dest.append(encoder.encodeText(content.substring(start, end)));
+    }
+}
diff --git a/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java
new file mode 100644
index 00000000000..4f1ec5fdb83
--- /dev/null
+++ b/core/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.CommonTermsQuery;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
+import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
+
+import java.io.IOException;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Subclass of the {@link UnifiedHighlighter} that works for a single field in a single document.
+ * Uses a custom {@link PassageFormatter}. Accepts field content as a constructor
+ * argument, given that loadings field value can be done reading from _source field.
+ * Supports using different {@link BreakIterator} to break the text into fragments. Considers every distinct field
+ * value as a discrete passage for highlighting (unless the whole content needs to be highlighted).
+ * Supports both returning empty snippets and non highlighted snippets when no highlighting can be performed.
+ */
+public class CustomUnifiedHighlighter extends UnifiedHighlighter {
+    private static final Snippet[] EMPTY_SNIPPET = new Snippet[0];
+
+    private final String fieldValue;
+    private final PassageFormatter passageFormatter;
+    private final BreakIterator breakIterator;
+    private final boolean returnNonHighlightedSnippets;
+
+    /**
+     * Creates a new instance of {@link CustomUnifiedHighlighter}
+     *
+     * @param analyzer the analyzer used for the field at index time, used for multi term queries internally
+     * @param passageFormatter our own {@link CustomPassageFormatter}
+     *                         which generates snippets in forms of {@link Snippet} objects
+     * @param breakIterator the {@link BreakIterator} to use for dividing text into passages.
+     *                      If null {@link BreakIterator#getSentenceInstance(Locale)} is used.
+     * @param fieldValue the original field values as constructor argument, loaded from the _source field or
+     *                   the relevant stored field.
+     * @param returnNonHighlightedSnippets whether non highlighted snippets should be
+     *                                     returned rather than empty snippets when no highlighting can be performed
+     */
+    public CustomUnifiedHighlighter(IndexSearcher searcher,
+                                    Analyzer analyzer,
+                                    PassageFormatter passageFormatter,
+                                    @Nullable BreakIterator breakIterator,
+                                    String fieldValue,
+                                    boolean returnNonHighlightedSnippets) {
+        super(searcher, analyzer);
+        this.breakIterator = breakIterator;
+        this.passageFormatter = passageFormatter;
+        this.fieldValue = fieldValue;
+        this.returnNonHighlightedSnippets = returnNonHighlightedSnippets;
+    }
+
+    /**
+     * Highlights terms extracted from the provided query within the content of the provided field name
+     */
+    public Snippet[] highlightField(String field, Query query, int docId, int maxPassages) throws IOException {
+        Map<String, Object[]> fieldsAsObjects = super.highlightFieldsAsObjects(new String[]{field}, query,
+            new int[]{docId}, new int[]{maxPassages});
+        Object[] snippetObjects = fieldsAsObjects.get(field);
+        if (snippetObjects != null) {
+            //one single document at a time
+            assert snippetObjects.length == 1;
+            Object snippetObject = snippetObjects[0];
+            if (snippetObject != null && snippetObject instanceof Snippet[]) {
+                return (Snippet[]) snippetObject;
+            }
+        }
+        return EMPTY_SNIPPET;
+    }
+
+    @Override
+    protected List<CharSequence[]> loadFieldValues(String[] fields, DocIdSetIterator docIter,
+                                                   int cacheCharsThreshold) throws IOException {
+        //we only highlight one field, one document at a time
+        return Collections.singletonList(new String[]{fieldValue});
+    }
+
+    @Override
+    protected BreakIterator getBreakIterator(String field) {
+        if (breakIterator != null) {
+            return breakIterator;
+        }
+        return super.getBreakIterator(field);
+    }
+
+    @Override
+    protected PassageFormatter getFormatter(String field) {
+        return passageFormatter;
+    }
+
+    @Override
+    protected int getMaxNoHighlightPassages(String field) {
+        if (returnNonHighlightedSnippets) {
+            return 1;
+        }
+        return 0;
+    }
+
+    @Override
+    protected Collection<Query> preMultiTermQueryRewrite(Query query) {
+        return rewriteCustomQuery(query);
+    }
+
+    @Override
+    protected Collection<Query> preSpanQueryRewrite(Query query) {
+        return rewriteCustomQuery(query);
+    }
+
+
+    /**
+     * Translate custom queries in queries that are supported by the unified highlighter.
+     */
+    private Collection<Query> rewriteCustomQuery(Query query) {
+        if (query instanceof MultiPhrasePrefixQuery) {
+            MultiPhrasePrefixQuery mpq = (MultiPhrasePrefixQuery) query;
+            Term[][] terms = mpq.getTerms();
+            int[] positions = mpq.getPositions();
+            SpanQuery[] positionSpanQueries = new SpanQuery[positions.length];
+            int sizeMinus1 = terms.length - 1;
+            for (int i = 0; i < positions.length; i++) {
+                SpanQuery[] innerQueries = new SpanQuery[terms[i].length];
+                for (int j = 0; j < terms[i].length; j++) {
+                    if (i == sizeMinus1) {
+                        innerQueries[j] = new SpanMultiTermQueryWrapper(new PrefixQuery(terms[i][j]));
+                    } else {
+                        innerQueries[j] = new SpanTermQuery(terms[i][j]);
+                    }
+                }
+                if (innerQueries.length > 1) {
+                    positionSpanQueries[i] = new SpanOrQuery(innerQueries);
+                } else {
+                    positionSpanQueries[i] = innerQueries[0];
+                }
+            }
+            // sum position increments beyond 1
+            int positionGaps = 0;
+            if (positions.length >= 2) {
+                // positions are in increasing order.   max(0,...) is just a safeguard.
+                positionGaps = Math.max(0, positions[positions.length - 1] - positions[0] - positions.length + 1);
+            }
+
+            //if original slop is 0 then require inOrder
+            boolean inorder = (mpq.getSlop() == 0);
+            return Collections.singletonList(new SpanNearQuery(positionSpanQueries,
+                mpq.getSlop() + positionGaps, inorder));
+        } else if (query instanceof CommonTermsQuery) {
+            CommonTermsQuery ctq = (CommonTermsQuery) query;
+            List<Query> tqs = new ArrayList<> ();
+            for (Term term : ctq.getTerms()) {
+                tqs.add(new TermQuery(term));
+            }
+            return tqs;
+        } else if (query instanceof AllTermQuery) {
+            AllTermQuery atq = (AllTermQuery) query;
+            return Collections.singletonList(new TermQuery(atq.getTerm()));
+        } else if (query instanceof FunctionScoreQuery) {
+            return Collections.singletonList(((FunctionScoreQuery) query).getSubQuery());
+        } else if (query instanceof FiltersFunctionScoreQuery) {
+            return Collections.singletonList(((FiltersFunctionScoreQuery) query).getSubQuery());
+        } else {
+            return null;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 97711eed427..389892a8652 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -21,6 +21,8 @@ package org.elasticsearch;
 
 import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,45 +36,49 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VALUE;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
-import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
 
 /**
  * A base class for all elasticsearch exceptions.
  */
 public class ElasticsearchException extends RuntimeException implements ToXContent, Writeable {
 
-    static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);
+    private static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);
 
     /**
-     * Passed in the {@link Params} of {@link #toXContent(XContentBuilder, org.elasticsearch.common.xcontent.ToXContent.Params, Throwable)}
+     * Passed in the {@link Params} of {@link #generateThrowableXContent(XContentBuilder, Params, Throwable)}
      * to control if the {@code caused_by} element should render. Unlike most parameters to {@code toXContent} methods this parameter is
      * internal only and not available as a URL parameter.
      */
-    public static final String REST_EXCEPTION_SKIP_CAUSE = "rest.exception.cause.skip";
+    private static final String REST_EXCEPTION_SKIP_CAUSE = "rest.exception.cause.skip";
     /**
-     * Passed in the {@link Params} of {@link #toXContent(XContentBuilder, org.elasticsearch.common.xcontent.ToXContent.Params, Throwable)}
+     * Passed in the {@link Params} of {@link #generateThrowableXContent(XContentBuilder, Params, Throwable)}
      * to control if the {@code stack_trace} element should render. Unlike most parameters to {@code toXContent} methods this parameter is
      * internal only and not available as a URL parameter. Use the {@code error_trace} parameter instead.
      */
     public static final String REST_EXCEPTION_SKIP_STACK_TRACE = "rest.exception.stacktrace.skip";
     public static final boolean REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT = true;
-    public static final boolean REST_EXCEPTION_SKIP_CAUSE_DEFAULT = false;
-    private static final String INDEX_HEADER_KEY = "es.index";
-    private static final String INDEX_HEADER_KEY_UUID = "es.index_uuid";
-    private static final String SHARD_HEADER_KEY = "es.shard";
-    private static final String RESOURCE_HEADER_TYPE_KEY = "es.resource.type";
-    private static final String RESOURCE_HEADER_ID_KEY = "es.resource.id";
+    private static final boolean REST_EXCEPTION_SKIP_CAUSE_DEFAULT = false;
+    private static final String INDEX_METADATA_KEY = "es.index";
+    private static final String INDEX_METADATA_KEY_UUID = "es.index_uuid";
+    private static final String SHARD_METADATA_KEY = "es.shard";
+    private static final String RESOURCE_METADATA_TYPE_KEY = "es.resource.type";
+    private static final String RESOURCE_METADATA_ID_KEY = "es.resource.id";
 
     private static final String TYPE = "type";
     private static final String REASON = "reason";
@@ -82,8 +88,9 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     private static final String ERROR = "error";
     private static final String ROOT_CAUSE = "root_cause";
 
-    private static final Map<Integer, FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException>> ID_TO_SUPPLIER;
+    private static final Map<Integer, CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException>> ID_TO_SUPPLIER;
     private static final Map<Class<? extends ElasticsearchException>, ElasticsearchExceptionHandle> CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE;
+    private final Map<String, List<String>> metadata = new HashMap<>();
     private final Map<String, List<String>> headers = new HashMap<>();
 
     /**
@@ -125,14 +132,56 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         super(in.readOptionalString(), in.readException());
         readStackTrace(this, in);
         headers.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
+        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            metadata.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
+        } else {
+            for (Iterator<Map.Entry<String, List<String>>> iterator = headers.entrySet().iterator(); iterator.hasNext(); ) {
+                Map.Entry<String, List<String>> header = iterator.next();
+                if (header.getKey().startsWith("es.")) {
+                    metadata.put(header.getKey(), header.getValue());
+                    iterator.remove();
+                }
+            }
+        }
     }
 
     /**
-     * Adds a new header with the given key.
-     * This method will replace existing header if a header with the same key already exists
+     * Adds a new piece of metadata with the given key.
+     * If the provided key is already present, the corresponding metadata will be replaced
      */
-    public void addHeader(String key, String... value) {
-        this.headers.put(key, Arrays.asList(value));
+    public void addMetadata(String key, String... values) {
+        addMetadata(key, Arrays.asList(values));
+    }
+
+    /**
+     * Adds a new piece of metadata with the given key.
+     * If the provided key is already present, the corresponding metadata will be replaced
+     */
+    public void addMetadata(String key, List<String> values) {
+        //we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
+        if (key.startsWith("es.") == false) {
+            throw new IllegalArgumentException("exception metadata must start with [es.], found [" + key + "] instead");
+        }
+        this.metadata.put(key, values);
+    }
+
+    /**
+     * Returns a set of all metadata keys on this exception
+     */
+    public Set<String> getMetadataKeys() {
+        return metadata.keySet();
+    }
+
+    /**
+     * Returns the list of metadata values for the given key or {@code null} if no metadata for the
+     * given key exists.
+     */
+    public List<String> getMetadata(String key) {
+        return metadata.get(key);
+    }
+
+    protected Map<String, List<String>> getMetadata() {
+        return metadata;
     }
 
     /**
@@ -140,9 +189,20 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
      * This method will replace existing header if a header with the same key already exists
      */
     public void addHeader(String key, List<String> value) {
+        //we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
+        if (key.startsWith("es.")) {
+            throw new IllegalArgumentException("exception headers must not start with [es.], found [" + key + "] instead");
+        }
         this.headers.put(key, value);
     }
 
+    /**
+     * Adds a new header with the given key.
+     * This method will replace existing header if a header with the same key already exists
+     */
+    public void addHeader(String key, String... value) {
+        addHeader(key, Arrays.asList(value));
+    }
 
     /**
      * Returns a set of all header keys on this exception
@@ -152,13 +212,17 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     }
 
     /**
-     * Returns the list of header values for the given key or {@code null} if not header for the
+     * Returns the list of header values for the given key or {@code null} if no header for the
      * given key exists.
      */
     public List<String> getHeader(String key) {
         return headers.get(key);
     }
 
+    protected Map<String, List<String>> getHeaders() {
+        return headers;
+    }
+
     /**
      * Returns the rest status code associated with this exception.
      */
@@ -219,11 +283,19 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         out.writeOptionalString(this.getMessage());
         out.writeException(this.getCause());
         writeStackTraces(this, out);
-        out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
+            out.writeMapOfLists(metadata, StreamOutput::writeString, StreamOutput::writeString);
+        } else {
+            HashMap<String, List<String>> finalHeaders = new HashMap<>(headers.size() + metadata.size());
+            finalHeaders.putAll(headers);
+            finalHeaders.putAll(metadata);
+            out.writeMapOfLists(finalHeaders, StreamOutput::writeString, StreamOutput::writeString);
+        }
     }
 
     public static ElasticsearchException readException(StreamInput input, int id) throws IOException {
-        FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException> elasticsearchException = ID_TO_SUPPLIER.get(id);
+        CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> elasticsearchException = ID_TO_SUPPLIER.get(id);
         if (elasticsearchException == null) {
             throw new IllegalStateException("unknown exception for id: " + id);
         }
@@ -256,64 +328,51 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         Throwable ex = ExceptionsHelper.unwrapCause(this);
         if (ex != this) {
-            toXContent(builder, params, this);
+            generateThrowableXContent(builder, params, this);
         } else {
-            builder.field(TYPE, getExceptionName());
-            builder.field(REASON, getMessage());
-            for (String key : headers.keySet()) {
-                if (key.startsWith("es.")) {
-                    List<String> values = headers.get(key);
-                    xContentHeader(builder, key.substring("es.".length()), values);
-                }
-            }
-            innerToXContent(builder, params);
-            renderHeader(builder, params);
-            if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
-                builder.field(STACK_TRACE, ExceptionsHelper.stackTrace(this));
-            }
+            innerToXContent(builder, params, this, getExceptionName(), getMessage(), headers, metadata, getCause());
         }
         return builder;
     }
 
-    /**
-     * Renders additional per exception information into the xcontent
-     */
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        causeToXContent(builder, params);
-    }
+    protected static void innerToXContent(XContentBuilder builder, Params params,
+                                          Throwable throwable, String type, String message, Map<String, List<String>> headers,
+                                          Map<String, List<String>> metadata, Throwable cause) throws IOException {
+        builder.field(TYPE, type);
+        builder.field(REASON, message);
 
-    /**
-     * Renders a cause exception as xcontent
-     */
-    protected void causeToXContent(XContentBuilder builder, Params params) throws IOException {
-        final Throwable cause = getCause();
-        if (cause != null && params.paramAsBoolean(REST_EXCEPTION_SKIP_CAUSE, REST_EXCEPTION_SKIP_CAUSE_DEFAULT) == false) {
-            builder.field(CAUSED_BY);
-            builder.startObject();
-            toXContent(builder, params, cause);
+        for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
+            headerToXContent(builder, entry.getKey().substring("es.".length()), entry.getValue());
+        }
+
+        if (throwable instanceof ElasticsearchException) {
+            ElasticsearchException exception = (ElasticsearchException) throwable;
+            exception.metadataToXContent(builder, params);
+        }
+
+        if (params.paramAsBoolean(REST_EXCEPTION_SKIP_CAUSE, REST_EXCEPTION_SKIP_CAUSE_DEFAULT) == false) {
+            if (cause != null) {
+                builder.field(CAUSED_BY);
+                builder.startObject();
+                generateThrowableXContent(builder, params, cause);
+                builder.endObject();
+            }
+        }
+
+        if (headers.isEmpty() == false) {
+            builder.startObject(HEADER);
+            for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
+                headerToXContent(builder, entry.getKey(), entry.getValue());
+            }
             builder.endObject();
         }
-    }
 
-    protected final void renderHeader(XContentBuilder builder, Params params) throws IOException {
-        boolean hasHeader = false;
-        for (String key : headers.keySet()) {
-            if (key.startsWith("es.")) {
-                continue;
-            }
-            if (hasHeader == false) {
-                builder.startObject(HEADER);
-                hasHeader = true;
-            }
-            List<String> values = headers.get(key);
-            xContentHeader(builder, key, values);
-        }
-        if (hasHeader) {
-            builder.endObject();
+        if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
+            builder.field(STACK_TRACE, ExceptionsHelper.stackTrace(throwable));
         }
     }
 
-    private void xContentHeader(XContentBuilder builder, String key, List<String> values) throws IOException {
+    private static void headerToXContent(XContentBuilder builder, String key, List<String> values) throws IOException {
         if (values != null && values.isEmpty() == false) {
             if (values.size() == 1) {
                 builder.field(key, values.get(0));
@@ -328,25 +387,9 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     }
 
     /**
-     * Static toXContent helper method that also renders non {@link org.elasticsearch.ElasticsearchException} instances as XContent.
+     * Renders additional per exception information into the XContent
      */
-    public static void toXContent(XContentBuilder builder, Params params, Throwable ex) throws IOException {
-        ex = ExceptionsHelper.unwrapCause(ex);
-        if (ex instanceof ElasticsearchException) {
-            ((ElasticsearchException) ex).toXContent(builder, params);
-        } else {
-            builder.field(TYPE, getExceptionName(ex));
-            builder.field(REASON, ex.getMessage());
-            if (ex.getCause() != null) {
-                builder.field(CAUSED_BY);
-                builder.startObject();
-                toXContent(builder, params, ex.getCause());
-                builder.endObject();
-            }
-            if (params.paramAsBoolean(REST_EXCEPTION_SKIP_STACK_TRACE, REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT) == false) {
-                builder.field(STACK_TRACE, ExceptionsHelper.stackTrace(ex));
-            }
-        }
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
     }
 
     /**
@@ -357,15 +400,25 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
      * instances.
      */
     public static ElasticsearchException fromXContent(XContentParser parser) throws IOException {
-        XContentParser.Token token = ensureFieldName(parser.nextToken(), parser::getTokenLocation);
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+        return innerFromXContent(parser, false);
+    }
+
+    private static ElasticsearchException innerFromXContent(XContentParser parser, boolean parseRootCauses) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
 
         String type = null, reason = null, stack = null;
         ElasticsearchException cause = null;
-        Map<String, Object> headers = new HashMap<>();
+        Map<String, List<String>> metadata = new HashMap<>();
+        Map<String, List<String>> headers = new HashMap<>();
+        List<ElasticsearchException> rootCauses = new ArrayList<>();
 
-        do {
+        for (; token == XContentParser.Token.FIELD_NAME; token = parser.nextToken()) {
             String currentFieldName = parser.currentName();
             token = parser.nextToken();
+
             if (token.isValue()) {
                 if (TYPE.equals(currentFieldName)) {
                     type = parser.text();
@@ -373,36 +426,173 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                     reason = parser.text();
                 } else if (STACK_TRACE.equals(currentFieldName)) {
                     stack = parser.text();
-                } else {
-                    // Everything else is considered as a header
-                    headers.put(currentFieldName, parser.text());
+                } else if (token == XContentParser.Token.VALUE_STRING) {
+                    metadata.put(currentFieldName, Collections.singletonList(parser.text()));
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (CAUSED_BY.equals(currentFieldName)) {
                     cause = fromXContent(parser);
                 } else if (HEADER.equals(currentFieldName)) {
-                    headers.putAll(parser.map());
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        } else {
+                            List<String> values = headers.getOrDefault(currentFieldName, new ArrayList<>());
+                            if (token == XContentParser.Token.VALUE_STRING) {
+                                values.add(parser.text());
+                            } else if (token == XContentParser.Token.START_ARRAY) {
+                                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                    if (token == XContentParser.Token.VALUE_STRING) {
+                                        values.add(parser.text());
+                                    } else {
+                                        parser.skipChildren();
+                                    }
+                                }
+                            } else if (token == XContentParser.Token.START_OBJECT) {
+                                parser.skipChildren();
+                            }
+                            headers.put(currentFieldName, values);
+                        }
+                    }
                 } else {
-                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                    // Any additional metadata object added by the metadataToXContent method is ignored
+                    // and skipped, so that the parser does not fail on unknown fields. The parser only
+                    // support metadata key-pairs and metadata arrays of values.
+                    parser.skipChildren();
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (parseRootCauses && ROOT_CAUSE.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        rootCauses.add(fromXContent(parser));
+                    }
+                } else {
+                    // Parse the array and add each item to the corresponding list of metadata.
+                    // Arrays of objects are not supported yet and just ignored and skipped.
+                    List<String> values = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            values.add(parser.text());
+                        } else {
+                            parser.skipChildren();
+                        }
+                    }
+                    if (values.size() > 0) {
+                        if (metadata.containsKey(currentFieldName)) {
+                            values.addAll(metadata.get(currentFieldName));
+                        }
+                        metadata.put(currentFieldName, values);
+                    }
                 }
             }
-        } while ((token = parser.nextToken()) == XContentParser.Token.FIELD_NAME);
-
-        StringBuilder message = new StringBuilder("Elasticsearch exception [");
-        message.append(TYPE).append('=').append(type).append(", ");
-        message.append(REASON).append('=').append(reason);
-        if (stack != null) {
-            message.append(", ").append(STACK_TRACE).append('=').append(stack);
         }
-        message.append(']');
 
-        ElasticsearchException e = new ElasticsearchException(message.toString(), cause);
-        for (Map.Entry<String, Object> header : headers.entrySet()) {
-            e.addHeader(header.getKey(), String.valueOf(header.getValue()));
+        ElasticsearchException e = new ElasticsearchException(buildMessage(type, reason, stack), cause);
+        for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
+            //subclasses can print out additional metadata through the metadataToXContent method. Simple key-value pairs will be
+            //parsed back and become part of this metadata set, while objects and arrays are not supported when parsing back.
+            //Those key-value pairs become part of the metadata set and inherit the "es." prefix as that is currently required
+            //by addMetadata. The prefix will get stripped out when printing metadata out so it will be effectively invisible.
+            //TODO move subclasses that print out simple metadata to using addMetadata directly and support also numbers and booleans.
+            //TODO rename metadataToXContent and have only SearchPhaseExecutionException use it, which prints out complex objects
+            e.addMetadata("es." + entry.getKey(), entry.getValue());
+        }
+        for (Map.Entry<String, List<String>> header : headers.entrySet()) {
+            e.addHeader(header.getKey(), header.getValue());
+        }
+
+        // Adds root causes as suppressed exception. This way they are not lost
+        // after parsing and can be retrieved using getSuppressed() method.
+        for (ElasticsearchException rootCause : rootCauses) {
+            e.addSuppressed(rootCause);
         }
         return e;
     }
 
+    /**
+     * Static toXContent helper method that renders {@link org.elasticsearch.ElasticsearchException} or {@link Throwable} instances
+     * as XContent, delegating the rendering to {@link #toXContent(XContentBuilder, Params)}
+     * or {@link #innerToXContent(XContentBuilder, Params, Throwable, String, String, Map, Map, Throwable)}.
+     *
+     * This method is usually used when the {@link Throwable} is rendered as a part of another XContent object, and its result can
+     * be parsed back using the {@link #fromXContent(XContentParser)} method.
+     */
+    public static void generateThrowableXContent(XContentBuilder builder, Params params, Throwable t) throws IOException {
+        t = ExceptionsHelper.unwrapCause(t);
+
+        if (t instanceof ElasticsearchException) {
+            ((ElasticsearchException) t).toXContent(builder, params);
+        } else {
+            innerToXContent(builder, params, t, getExceptionName(t), t.getMessage(), emptyMap(), emptyMap(), t.getCause());
+        }
+    }
+
+    /**
+     * Render any exception as a xcontent, encapsulated within a field or object named "error". The level of details that are rendered
+     * depends on the value of the "detailed" parameter: when it's false only a simple message based on the type and message of the
+     * exception is rendered. When it's true all detail are provided including guesses root causes, cause and potentially stack
+     * trace.
+     *
+     * This method is usually used when the {@link Exception} is rendered as a full XContent object, and its output can be parsed
+     * by the {@link #failureFromXContent(XContentParser)} method.
+     */
+    public static void generateFailureXContent(XContentBuilder builder, Params params, @Nullable Exception e, boolean detailed)
+            throws IOException {
+        // No exception to render as an error
+        if (e == null) {
+            builder.field(ERROR, "unknown");
+            return;
+        }
+
+        // Render the exception with a simple message
+        if (detailed == false) {
+            String message = "No ElasticsearchException found";
+            Throwable t = e;
+            for (int counter = 0; counter < 10 && t != null; counter++) {
+                if (t instanceof ElasticsearchException) {
+                    message = t.getClass().getSimpleName() + "[" + t.getMessage() + "]";
+                    break;
+                }
+                t = t.getCause();
+            }
+            builder.field(ERROR, message);
+            return;
+        }
+
+        // Render the exception with all details
+        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
+        builder.startObject(ERROR);
+        {
+            builder.startArray(ROOT_CAUSE);
+            for (ElasticsearchException rootCause : rootCauses) {
+                builder.startObject();
+                rootCause.toXContent(builder, new DelegatingMapParams(singletonMap(REST_EXCEPTION_SKIP_CAUSE, "true"), params));
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+        generateThrowableXContent(builder, params, e);
+        builder.endObject();
+    }
+
+    /**
+     * Parses the output of {@link #generateFailureXContent(XContentBuilder, Params, Exception, boolean)}
+     */
+    public static ElasticsearchException failureFromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureFieldName(parser, token, ERROR);
+
+        token = parser.nextToken();
+        if (token.isValue()) {
+            return new ElasticsearchException(buildMessage("exception", parser.text(), null));
+        }
+
+        ensureExpectedToken(token, XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+        token = parser.nextToken();
+
+        // Root causes are parsed in the innerFromXContent() and are added as suppressed exceptions.
+        return innerFromXContent(parser, true);
+    }
+
     /**
      * Returns the root cause of this exception or multiple if different shards caused different exceptions
      */
@@ -448,12 +638,23 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         return toUnderscoreCase(simpleName);
     }
 
+    static String buildMessage(String type, String reason, String stack) {
+        StringBuilder message = new StringBuilder("Elasticsearch exception [");
+        message.append(TYPE).append('=').append(type).append(", ");
+        message.append(REASON).append('=').append(reason);
+        if (stack != null) {
+            message.append(", ").append(STACK_TRACE).append('=').append(stack);
+        }
+        message.append(']');
+        return message.toString();
+    }
+
     @Override
     public String toString() {
         StringBuilder builder = new StringBuilder();
-        if (headers.containsKey(INDEX_HEADER_KEY)) {
+        if (metadata.containsKey(INDEX_METADATA_KEY)) {
             builder.append(getIndex());
-            if (headers.containsKey(SHARD_HEADER_KEY)) {
+            if (metadata.containsKey(SHARD_METADATA_KEY)) {
                 builder.append('[').append(getShardId()).append(']');
             }
             builder.append(' ');
@@ -752,8 +953,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                 org.elasticsearch.search.SearchContextException::new, 127, UNKNOWN_VERSION_ADDED),
         SEARCH_SOURCE_BUILDER_EXCEPTION(org.elasticsearch.search.builder.SearchSourceBuilderException.class,
                 org.elasticsearch.search.builder.SearchSourceBuilderException::new, 128, UNKNOWN_VERSION_ADDED),
-        ENGINE_CLOSED_EXCEPTION(org.elasticsearch.index.engine.EngineClosedException.class,
-                org.elasticsearch.index.engine.EngineClosedException::new, 129, UNKNOWN_VERSION_ADDED),
+        // 129 was EngineClosedException
         NO_SHARD_AVAILABLE_ACTION_EXCEPTION(org.elasticsearch.action.NoShardAvailableActionException.class,
                 org.elasticsearch.action.NoShardAvailableActionException::new, 130, UNKNOWN_VERSION_ADDED),
         UNAVAILABLE_SHARDS_EXCEPTION(org.elasticsearch.action.UnavailableShardsException.class,
@@ -786,15 +986,17 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         TASK_CANCELLED_EXCEPTION(org.elasticsearch.tasks.TaskCancelledException.class,
             org.elasticsearch.tasks.TaskCancelledException::new, 146, Version.V_5_1_1_UNRELEASED),
         SHARD_LOCK_OBTAIN_FAILED_EXCEPTION(org.elasticsearch.env.ShardLockObtainFailedException.class,
-                                           org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2);
+                                           org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2),
+        UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class,
+                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0_UNRELEASED);
 
         final Class<? extends ElasticsearchException> exceptionClass;
-        final FunctionThatThrowsIOException<StreamInput, ? extends ElasticsearchException> constructor;
+        final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;
         final int id;
         final Version versionAdded;
 
         <E extends ElasticsearchException> ElasticsearchExceptionHandle(Class<E> exceptionClass,
-                                                                        FunctionThatThrowsIOException<StreamInput, E> constructor, int id,
+                                                                        CheckedFunction<StreamInput, E, IOException> constructor, int id,
                                                                         Version versionAdded) {
             // We need the exceptionClass because you can't dig it out of the constructor reliably.
             this.exceptionClass = exceptionClass;
@@ -812,9 +1014,9 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     }
 
     public Index getIndex() {
-        List<String> index = getHeader(INDEX_HEADER_KEY);
+        List<String> index = getMetadata(INDEX_METADATA_KEY);
         if (index != null && index.isEmpty() == false) {
-            List<String> index_uuid = getHeader(INDEX_HEADER_KEY_UUID);
+            List<String> index_uuid = getMetadata(INDEX_METADATA_KEY_UUID);
             return new Index(index.get(0), index_uuid.get(0));
         }
 
@@ -822,7 +1024,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     }
 
     public ShardId getShardId() {
-        List<String> shard = getHeader(SHARD_HEADER_KEY);
+        List<String> shard = getMetadata(SHARD_METADATA_KEY);
         if (shard != null && shard.isEmpty() == false) {
             return new ShardId(getIndex(), Integer.parseInt(shard.get(0)));
         }
@@ -831,8 +1033,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
 
     public void setIndex(Index index) {
         if (index != null) {
-            addHeader(INDEX_HEADER_KEY, index.getName());
-            addHeader(INDEX_HEADER_KEY_UUID, index.getUUID());
+            addMetadata(INDEX_METADATA_KEY, index.getName());
+            addMetadata(INDEX_METADATA_KEY_UUID, index.getUUID());
         }
     }
 
@@ -845,27 +1047,22 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     public void setShard(ShardId shardId) {
         if (shardId != null) {
             setIndex(shardId.getIndex());
-            addHeader(SHARD_HEADER_KEY, Integer.toString(shardId.id()));
+            addMetadata(SHARD_METADATA_KEY, Integer.toString(shardId.id()));
         }
     }
 
-    public void setShard(String index, int shardId) {
-            setIndex(index);
-            addHeader(SHARD_HEADER_KEY, Integer.toString(shardId));
-    }
-
     public void setResources(String type, String... id) {
         assert type != null;
-        addHeader(RESOURCE_HEADER_ID_KEY, id);
-        addHeader(RESOURCE_HEADER_TYPE_KEY, type);
+        addMetadata(RESOURCE_METADATA_ID_KEY, id);
+        addMetadata(RESOURCE_METADATA_TYPE_KEY, type);
     }
 
     public List<String> getResourceId() {
-        return getHeader(RESOURCE_HEADER_ID_KEY);
+        return getMetadata(RESOURCE_METADATA_ID_KEY);
     }
 
     public String getResourceType() {
-        List<String> header = getHeader(RESOURCE_HEADER_TYPE_KEY);
+        List<String> header = getMetadata(RESOURCE_METADATA_TYPE_KEY);
         if (header != null && header.isEmpty() == false) {
             assert header.size() == 1;
             return header.get(0);
@@ -873,26 +1070,6 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         return null;
     }
 
-    public static void renderException(XContentBuilder builder, Params params, Exception e) throws IOException {
-        builder.startObject(ERROR);
-        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
-        builder.field(ROOT_CAUSE);
-        builder.startArray();
-        for (ElasticsearchException rootCause : rootCauses) {
-            builder.startObject();
-            rootCause.toXContent(builder, new ToXContent.DelegatingMapParams(
-                    Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE, "true"), params));
-            builder.endObject();
-        }
-        builder.endArray();
-        ElasticsearchException.toXContent(builder, params, e);
-        builder.endObject();
-    }
-
-    interface FunctionThatThrowsIOException<T, R> {
-        R apply(T t) throws IOException;
-    }
-
     // lower cases and adds underscores to transitions in a name
     private static String toUnderscoreCase(String value) {
         StringBuilder sb = new StringBuilder();
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java b/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
index cdccdb8da95..1711e9a3aaf 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchParseException.java
@@ -24,6 +24,9 @@ import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+/**
+ * Unchecked exception that is translated into a {@code 400 BAD REQUEST} error when it bubbles out over HTTP.
+ */
 public class ElasticsearchParseException extends ElasticsearchException {
 
     public ElasticsearchParseException(String msg, Object... args) {
diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
index c30662a0934..e89e04a301d 100644
--- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -214,7 +214,7 @@ public final class ExceptionsHelper {
         final String index;
         final Class<? extends Throwable> causeType;
 
-        public GroupBy(Throwable t) {
+        GroupBy(Throwable t) {
             if (t instanceof ElasticsearchException) {
                 final Index index = ((ElasticsearchException) t).getIndex();
                 if (index != null) {
diff --git a/core/src/main/java/org/elasticsearch/SpecialPermission.java b/core/src/main/java/org/elasticsearch/SpecialPermission.java
index 7d796346c64..9e5571a5b0a 100644
--- a/core/src/main/java/org/elasticsearch/SpecialPermission.java
+++ b/core/src/main/java/org/elasticsearch/SpecialPermission.java
@@ -57,6 +57,9 @@ import java.security.BasicPermission;
  * </code></pre>
  */
 public final class SpecialPermission extends BasicPermission {
+
+    public static final SpecialPermission INSTANCE = new SpecialPermission();
+
     /**
      * Creates a new SpecialPermision object.
      */
@@ -76,4 +79,14 @@ public final class SpecialPermission extends BasicPermission {
     public SpecialPermission(String name, String actions) {
         this();
     }
+
+    /**
+     * Check that the current stack has {@link SpecialPermission} access according to the {@link SecurityManager}.
+     */
+    public static void check() {
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(INSTANCE);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 474b220a9c5..ccd6f7c9a06 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -29,7 +29,7 @@ import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
 
-public class Version {
+public class Version implements Comparable<Version> {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
      * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99
@@ -73,6 +73,10 @@ public class Version {
     public static final Version V_2_4_2 = new Version(V_2_4_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
     public static final int V_2_4_3_ID = 2040399;
     public static final Version V_2_4_3 = new Version(V_2_4_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_4_ID = 2040499;
+    public static final Version V_2_4_4 = new Version(V_2_4_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
+    public static final int V_2_4_5_ID = 2040599;
+    public static final Version V_2_4_5 = new Version(V_2_4_5_ID, org.apache.lucene.util.Version.LUCENE_5_5_2);
     public static final int V_5_0_0_alpha1_ID = 5000001;
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
@@ -100,8 +104,14 @@ public class Version {
     public static final Version V_5_1_1_UNRELEASED = new Version(V_5_1_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
     public static final int V_5_1_2_ID_UNRELEASED = 5010299;
     public static final Version V_5_1_2_UNRELEASED = new Version(V_5_1_2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final int V_5_1_3_ID_UNRELEASED = 5010399;
+    public static final Version V_5_1_3_UNRELEASED = new Version(V_5_1_3_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
     public static final int V_5_2_0_ID_UNRELEASED = 5020099;
-    public static final Version V_5_2_0_UNRELEASED = new Version(V_5_2_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final Version V_5_2_0_UNRELEASED = new Version(V_5_2_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
+    public static final int V_5_2_1_ID_UNRELEASED = 5020199;
+    public static final Version V_5_2_1_UNRELEASED = new Version(V_5_2_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
+    public static final int V_5_3_0_ID_UNRELEASED = 5030099;
+    public static final Version V_5_3_0_UNRELEASED = new Version(V_5_3_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
     public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
     public static final Version V_6_0_0_alpha1_UNRELEASED =
         new Version(V_6_0_0_alpha1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
@@ -122,8 +132,14 @@ public class Version {
         switch (id) {
             case V_6_0_0_alpha1_ID_UNRELEASED:
                 return V_6_0_0_alpha1_UNRELEASED;
+            case V_5_3_0_ID_UNRELEASED:
+                return V_5_3_0_UNRELEASED;
+            case V_5_2_1_ID_UNRELEASED:
+                return V_5_2_1_UNRELEASED;
             case V_5_2_0_ID_UNRELEASED:
                 return V_5_2_0_UNRELEASED;
+            case V_5_1_3_ID_UNRELEASED:
+                return V_5_1_3_UNRELEASED;
             case V_5_1_2_ID_UNRELEASED:
                 return V_5_1_2_UNRELEASED;
             case V_5_1_1_ID_UNRELEASED:
@@ -150,6 +166,10 @@ public class Version {
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
                 return V_5_0_0_alpha1;
+            case V_2_4_5_ID:
+                return V_2_4_5;
+            case V_2_4_4_ID:
+                return V_2_4_4;
             case V_2_4_3_ID:
                 return V_2_4_3;
             case V_2_4_2_ID:
@@ -310,6 +330,11 @@ public class Version {
         return version.id >= id;
     }
 
+    @Override
+    public int compareTo(Version other) {
+        return Integer.compare(this.id, other.id);
+    }
+
     /**
      * Returns the minimum compatible version based on the current
      * version. Ie a node needs to have at least the return version in order
diff --git a/core/src/main/java/org/elasticsearch/action/ActionListener.java b/core/src/main/java/org/elasticsearch/action/ActionListener.java
index ef26867600e..f9fafa9f95a 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionListener.java
@@ -19,8 +19,11 @@
 
 package org.elasticsearch.action;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.CheckedConsumer;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.function.Consumer;
 
 /**
@@ -65,4 +68,41 @@ public interface ActionListener<Response> {
             }
         };
     }
+
+    /**
+     * Notifies every given listener with the response passed to {@link #onResponse(Object)}. If a listener itself throws an exception
+     * the exception is forwarded to {@link #onFailure(Exception)}. If in turn {@link #onFailure(Exception)} fails all remaining
+     * listeners will be processed and the caught exception will be re-thrown.
+     */
+    static <Response> void onResponse(Iterable<ActionListener<Response>> listeners, Response response) {
+        List<Exception> exceptionList = new ArrayList<>();
+        for (ActionListener<Response> listener : listeners) {
+            try {
+                listener.onResponse(response);
+            } catch (Exception ex) {
+                try {
+                    listener.onFailure(ex);
+                } catch (Exception ex1) {
+                    exceptionList.add(ex1);
+                }
+            }
+        }
+        ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptionList);
+    }
+
+    /**
+     * Notifies every given listener with the failure passed to {@link #onFailure(Exception)}. If a listener itself throws an exception
+     * all remaining listeners will be processed and the caught exception will be re-thrown.
+     */
+    static <Response> void onFailure(Iterable<ActionListener<Response>> listeners, Exception failure) {
+        List<Exception> exceptionList = new ArrayList<>();
+        for (ActionListener<Response> listener : listeners) {
+            try {
+                listener.onFailure(failure);
+            } catch (Exception ex) {
+                exceptionList.add(ex);
+            }
+        }
+        ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptionList);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index a24ed5f8083..4014c832b0f 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -19,14 +19,6 @@
 
 package org.elasticsearch.action;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.UnaryOperator;
-import java.util.stream.Collectors;
-
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
@@ -196,15 +188,19 @@ import org.elasticsearch.action.termvectors.TransportShardMultiTermsVectorAction
 import org.elasticsearch.action.termvectors.TransportTermVectorsAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.rest.RestController;
@@ -312,6 +308,15 @@ import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
+
 import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
 
@@ -324,6 +329,10 @@ public class ActionModule extends AbstractModule {
 
     private final boolean transportClient;
     private final Settings settings;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+    private final IndexScopedSettings indexScopedSettings;
+    private final ClusterSettings clusterSettings;
+    private final SettingsFilter settingsFilter;
     private final List<ActionPlugin> actionPlugins;
     private final Map<String, ActionHandler<?, ?>> actions;
     private final List<Class<? extends ActionFilter>> actionFilters;
@@ -331,14 +340,20 @@ public class ActionModule extends AbstractModule {
     private final DestructiveOperations destructiveOperations;
     private final RestController restController;
 
-    public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver resolver,
-                        ClusterSettings clusterSettings, ThreadPool threadPool, List<ActionPlugin> actionPlugins) {
+    public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
+                        IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
+                        ThreadPool threadPool, List<ActionPlugin> actionPlugins, NodeClient nodeClient,
+                        CircuitBreakerService circuitBreakerService) {
         this.transportClient = transportClient;
         this.settings = settings;
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+        this.indexScopedSettings = indexScopedSettings;
+        this.clusterSettings = clusterSettings;
+        this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
-        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, resolver);
+        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver);
         destructiveOperations = new DestructiveOperations(settings, clusterSettings);
         Set<String> headers = actionPlugins.stream().flatMap(p -> p.getRestHeaders().stream()).collect(Collectors.toSet());
         UnaryOperator<RestHandler> restWrapper = null;
@@ -352,9 +367,14 @@ public class ActionModule extends AbstractModule {
                 restWrapper = newRestWrapper;
             }
         }
-        restController = new RestController(settings, headers, restWrapper);
+        if (transportClient) {
+            restController = null;
+        } else {
+            restController = new RestController(settings, headers, restWrapper, nodeClient, circuitBreakerService);
+        }
     }
 
+
     public Map<String, ActionHandler<?, ?>> getActions() {
         return actions;
     }
@@ -362,7 +382,7 @@ public class ActionModule extends AbstractModule {
     static Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
         // Subclass NamedRegistry for easy registration
         class ActionRegistry extends NamedRegistry<ActionHandler<?, ?>> {
-            public ActionRegistry() {
+            ActionRegistry() {
                 super("action");
             }
 
@@ -478,147 +498,145 @@ public class ActionModule extends AbstractModule {
         return unmodifiableList(actionPlugins.stream().flatMap(p -> p.getActionFilters().stream()).collect(Collectors.toList()));
     }
 
-    static Set<Class<? extends RestHandler>> setupRestHandlers(List<ActionPlugin> actionPlugins) {
-        Set<Class<? extends RestHandler>> handlers = new HashSet<>();
-        registerRestHandler(handlers, RestMainAction.class);
-        registerRestHandler(handlers, RestNodesInfoAction.class);
-        registerRestHandler(handlers, RestNodesStatsAction.class);
-        registerRestHandler(handlers, RestNodesHotThreadsAction.class);
-        registerRestHandler(handlers, RestClusterAllocationExplainAction.class);
-        registerRestHandler(handlers, RestClusterStatsAction.class);
-        registerRestHandler(handlers, RestClusterStateAction.class);
-        registerRestHandler(handlers, RestClusterHealthAction.class);
-        registerRestHandler(handlers, RestClusterUpdateSettingsAction.class);
-        registerRestHandler(handlers, RestClusterGetSettingsAction.class);
-        registerRestHandler(handlers, RestClusterRerouteAction.class);
-        registerRestHandler(handlers, RestClusterSearchShardsAction.class);
-        registerRestHandler(handlers, RestPendingClusterTasksAction.class);
-        registerRestHandler(handlers, RestPutRepositoryAction.class);
-        registerRestHandler(handlers, RestGetRepositoriesAction.class);
-        registerRestHandler(handlers, RestDeleteRepositoryAction.class);
-        registerRestHandler(handlers, RestVerifyRepositoryAction.class);
-        registerRestHandler(handlers, RestGetSnapshotsAction.class);
-        registerRestHandler(handlers, RestCreateSnapshotAction.class);
-        registerRestHandler(handlers, RestRestoreSnapshotAction.class);
-        registerRestHandler(handlers, RestDeleteSnapshotAction.class);
-        registerRestHandler(handlers, RestSnapshotsStatusAction.class);
+    public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
+        List<AbstractCatAction> catActions = new ArrayList<>();
+        Consumer<RestHandler> registerHandler = a -> {
+            if (a instanceof AbstractCatAction) {
+                catActions.add((AbstractCatAction) a);
+            }
+        };
+        registerHandler.accept(new RestMainAction(settings, restController));
+        registerHandler.accept(new RestNodesInfoAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestNodesStatsAction(settings, restController));
+        registerHandler.accept(new RestNodesHotThreadsAction(settings, restController));
+        registerHandler.accept(new RestClusterAllocationExplainAction(settings, restController));
+        registerHandler.accept(new RestClusterStatsAction(settings, restController));
+        registerHandler.accept(new RestClusterStateAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestClusterHealthAction(settings, restController));
+        registerHandler.accept(new RestClusterUpdateSettingsAction(settings, restController));
+        registerHandler.accept(new RestClusterGetSettingsAction(settings, restController, clusterSettings, settingsFilter));
+        registerHandler.accept(new RestClusterRerouteAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestClusterSearchShardsAction(settings, restController));
+        registerHandler.accept(new RestPendingClusterTasksAction(settings, restController));
+        registerHandler.accept(new RestPutRepositoryAction(settings, restController));
+        registerHandler.accept(new RestGetRepositoriesAction(settings, restController, settingsFilter));
+        registerHandler.accept(new RestDeleteRepositoryAction(settings, restController));
+        registerHandler.accept(new RestVerifyRepositoryAction(settings, restController));
+        registerHandler.accept(new RestGetSnapshotsAction(settings, restController));
+        registerHandler.accept(new RestCreateSnapshotAction(settings, restController));
+        registerHandler.accept(new RestRestoreSnapshotAction(settings, restController));
+        registerHandler.accept(new RestDeleteSnapshotAction(settings, restController));
+        registerHandler.accept(new RestSnapshotsStatusAction(settings, restController));
 
-        registerRestHandler(handlers, RestIndicesExistsAction.class);
-        registerRestHandler(handlers, RestTypesExistsAction.class);
-        registerRestHandler(handlers, RestGetIndicesAction.class);
-        registerRestHandler(handlers, RestIndicesStatsAction.class);
-        registerRestHandler(handlers, RestIndicesSegmentsAction.class);
-        registerRestHandler(handlers, RestIndicesShardStoresAction.class);
-        registerRestHandler(handlers, RestGetAliasesAction.class);
-        registerRestHandler(handlers, RestAliasesExistAction.class);
-        registerRestHandler(handlers, RestIndexDeleteAliasesAction.class);
-        registerRestHandler(handlers, RestIndexPutAliasAction.class);
-        registerRestHandler(handlers, RestIndicesAliasesAction.class);
-        registerRestHandler(handlers, RestCreateIndexAction.class);
-        registerRestHandler(handlers, RestShrinkIndexAction.class);
-        registerRestHandler(handlers, RestRolloverIndexAction.class);
-        registerRestHandler(handlers, RestDeleteIndexAction.class);
-        registerRestHandler(handlers, RestCloseIndexAction.class);
-        registerRestHandler(handlers, RestOpenIndexAction.class);
+        registerHandler.accept(new RestIndicesExistsAction(settings, restController));
+        registerHandler.accept(new RestTypesExistsAction(settings, restController));
+        registerHandler.accept(new RestGetIndicesAction(settings, restController, indexScopedSettings, settingsFilter));
+        registerHandler.accept(new RestIndicesStatsAction(settings, restController));
+        registerHandler.accept(new RestIndicesSegmentsAction(settings, restController));
+        registerHandler.accept(new RestIndicesShardStoresAction(settings, restController));
+        registerHandler.accept(new RestGetAliasesAction(settings, restController));
+        registerHandler.accept(new RestAliasesExistAction(settings, restController));
+        registerHandler.accept(new RestIndexDeleteAliasesAction(settings, restController));
+        registerHandler.accept(new RestIndexPutAliasAction(settings, restController));
+        registerHandler.accept(new RestIndicesAliasesAction(settings, restController));
+        registerHandler.accept(new RestCreateIndexAction(settings, restController));
+        registerHandler.accept(new RestShrinkIndexAction(settings, restController));
+        registerHandler.accept(new RestRolloverIndexAction(settings, restController));
+        registerHandler.accept(new RestDeleteIndexAction(settings, restController));
+        registerHandler.accept(new RestCloseIndexAction(settings, restController));
+        registerHandler.accept(new RestOpenIndexAction(settings, restController));
 
-        registerRestHandler(handlers, RestUpdateSettingsAction.class);
-        registerRestHandler(handlers, RestGetSettingsAction.class);
+        registerHandler.accept(new RestUpdateSettingsAction(settings, restController));
+        registerHandler.accept(new RestGetSettingsAction(settings, restController, indexScopedSettings, settingsFilter));
 
-        registerRestHandler(handlers, RestAnalyzeAction.class);
-        registerRestHandler(handlers, RestGetIndexTemplateAction.class);
-        registerRestHandler(handlers, RestPutIndexTemplateAction.class);
-        registerRestHandler(handlers, RestDeleteIndexTemplateAction.class);
-        registerRestHandler(handlers, RestHeadIndexTemplateAction.class);
+        registerHandler.accept(new RestAnalyzeAction(settings, restController));
+        registerHandler.accept(new RestGetIndexTemplateAction(settings, restController));
+        registerHandler.accept(new RestPutIndexTemplateAction(settings, restController));
+        registerHandler.accept(new RestDeleteIndexTemplateAction(settings, restController));
+        registerHandler.accept(new RestHeadIndexTemplateAction(settings, restController));
 
-        registerRestHandler(handlers, RestPutMappingAction.class);
-        registerRestHandler(handlers, RestGetMappingAction.class);
-        registerRestHandler(handlers, RestGetFieldMappingAction.class);
+        registerHandler.accept(new RestPutMappingAction(settings, restController));
+        registerHandler.accept(new RestGetMappingAction(settings, restController));
+        registerHandler.accept(new RestGetFieldMappingAction(settings, restController));
 
-        registerRestHandler(handlers, RestRefreshAction.class);
-        registerRestHandler(handlers, RestFlushAction.class);
-        registerRestHandler(handlers, RestSyncedFlushAction.class);
-        registerRestHandler(handlers, RestForceMergeAction.class);
-        registerRestHandler(handlers, RestUpgradeAction.class);
-        registerRestHandler(handlers, RestClearIndicesCacheAction.class);
+        registerHandler.accept(new RestRefreshAction(settings, restController));
+        registerHandler.accept(new RestFlushAction(settings, restController));
+        registerHandler.accept(new RestSyncedFlushAction(settings, restController));
+        registerHandler.accept(new RestForceMergeAction(settings, restController));
+        registerHandler.accept(new RestUpgradeAction(settings, restController));
+        registerHandler.accept(new RestClearIndicesCacheAction(settings, restController));
 
-        registerRestHandler(handlers, RestIndexAction.class);
-        registerRestHandler(handlers, RestGetAction.class);
-        registerRestHandler(handlers, RestGetSourceAction.class);
-        registerRestHandler(handlers, RestHeadAction.Document.class);
-        registerRestHandler(handlers, RestHeadAction.Source.class);
-        registerRestHandler(handlers, RestMultiGetAction.class);
-        registerRestHandler(handlers, RestDeleteAction.class);
-        registerRestHandler(handlers, org.elasticsearch.rest.action.document.RestCountAction.class);
-        registerRestHandler(handlers, RestTermVectorsAction.class);
-        registerRestHandler(handlers, RestMultiTermVectorsAction.class);
-        registerRestHandler(handlers, RestBulkAction.class);
-        registerRestHandler(handlers, RestUpdateAction.class);
+        registerHandler.accept(new RestIndexAction(settings, restController));
+        registerHandler.accept(new RestGetAction(settings, restController));
+        registerHandler.accept(new RestGetSourceAction(settings, restController));
+        registerHandler.accept(new RestHeadAction.Document(settings, restController));
+        registerHandler.accept(new RestHeadAction.Source(settings, restController));
+        registerHandler.accept(new RestMultiGetAction(settings, restController));
+        registerHandler.accept(new RestDeleteAction(settings, restController));
+        registerHandler.accept(new org.elasticsearch.rest.action.document.RestCountAction(settings, restController));
+        registerHandler.accept(new RestTermVectorsAction(settings, restController));
+        registerHandler.accept(new RestMultiTermVectorsAction(settings, restController));
+        registerHandler.accept(new RestBulkAction(settings, restController));
+        registerHandler.accept(new RestUpdateAction(settings, restController));
 
-        registerRestHandler(handlers, RestSearchAction.class);
-        registerRestHandler(handlers, RestSearchScrollAction.class);
-        registerRestHandler(handlers, RestClearScrollAction.class);
-        registerRestHandler(handlers, RestMultiSearchAction.class);
+        registerHandler.accept(new RestSearchAction(settings, restController));
+        registerHandler.accept(new RestSearchScrollAction(settings, restController));
+        registerHandler.accept(new RestClearScrollAction(settings, restController));
+        registerHandler.accept(new RestMultiSearchAction(settings, restController));
 
-        registerRestHandler(handlers, RestValidateQueryAction.class);
+        registerHandler.accept(new RestValidateQueryAction(settings, restController));
 
-        registerRestHandler(handlers, RestExplainAction.class);
+        registerHandler.accept(new RestExplainAction(settings, restController));
 
-        registerRestHandler(handlers, RestRecoveryAction.class);
+        registerHandler.accept(new RestRecoveryAction(settings, restController));
 
         // Scripts API
-        registerRestHandler(handlers, RestGetStoredScriptAction.class);
-        registerRestHandler(handlers, RestPutStoredScriptAction.class);
-        registerRestHandler(handlers, RestDeleteStoredScriptAction.class);
+        registerHandler.accept(new RestGetStoredScriptAction(settings, restController));
+        registerHandler.accept(new RestPutStoredScriptAction(settings, restController));
+        registerHandler.accept(new RestDeleteStoredScriptAction(settings, restController));
 
-        registerRestHandler(handlers, RestFieldStatsAction.class);
+        registerHandler.accept(new RestFieldStatsAction(settings, restController));
 
         // Tasks API
-        registerRestHandler(handlers, RestListTasksAction.class);
-        registerRestHandler(handlers, RestGetTaskAction.class);
-        registerRestHandler(handlers, RestCancelTasksAction.class);
+        registerHandler.accept(new RestListTasksAction(settings, restController, nodesInCluster));
+        registerHandler.accept(new RestGetTaskAction(settings, restController));
+        registerHandler.accept(new RestCancelTasksAction(settings, restController, nodesInCluster));
 
         // Ingest API
-        registerRestHandler(handlers, RestPutPipelineAction.class);
-        registerRestHandler(handlers, RestGetPipelineAction.class);
-        registerRestHandler(handlers, RestDeletePipelineAction.class);
-        registerRestHandler(handlers, RestSimulatePipelineAction.class);
+        registerHandler.accept(new RestPutPipelineAction(settings, restController));
+        registerHandler.accept(new RestGetPipelineAction(settings, restController));
+        registerHandler.accept(new RestDeletePipelineAction(settings, restController));
+        registerHandler.accept(new RestSimulatePipelineAction(settings, restController));
 
         // CAT API
-        registerRestHandler(handlers, RestCatAction.class);
-        registerRestHandler(handlers, RestAllocationAction.class);
-        registerRestHandler(handlers, RestShardsAction.class);
-        registerRestHandler(handlers, RestMasterAction.class);
-        registerRestHandler(handlers, RestNodesAction.class);
-        registerRestHandler(handlers, RestTasksAction.class);
-        registerRestHandler(handlers, RestIndicesAction.class);
-        registerRestHandler(handlers, RestSegmentsAction.class);
+        registerHandler.accept(new RestAllocationAction(settings, restController));
+        registerHandler.accept(new RestShardsAction(settings, restController));
+        registerHandler.accept(new RestMasterAction(settings, restController));
+        registerHandler.accept(new RestNodesAction(settings, restController));
+        registerHandler.accept(new RestTasksAction(settings, restController, nodesInCluster));
+        registerHandler.accept(new RestIndicesAction(settings, restController, indexNameExpressionResolver));
+        registerHandler.accept(new RestSegmentsAction(settings, restController));
         // Fully qualified to prevent interference with rest.action.count.RestCountAction
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestCountAction.class);
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestCountAction(settings, restController));
         // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestRecoveryAction.class);
-        registerRestHandler(handlers, RestHealthAction.class);
-        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction.class);
-        registerRestHandler(handlers, RestAliasAction.class);
-        registerRestHandler(handlers, RestThreadPoolAction.class);
-        registerRestHandler(handlers, RestPluginsAction.class);
-        registerRestHandler(handlers, RestFielddataAction.class);
-        registerRestHandler(handlers, RestNodeAttrsAction.class);
-        registerRestHandler(handlers, RestRepositoriesAction.class);
-        registerRestHandler(handlers, RestSnapshotAction.class);
-        registerRestHandler(handlers, RestTemplatesAction.class);
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestRecoveryAction(settings, restController));
+        registerHandler.accept(new RestHealthAction(settings, restController));
+        registerHandler.accept(new org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction(settings, restController));
+        registerHandler.accept(new RestAliasAction(settings, restController));
+        registerHandler.accept(new RestThreadPoolAction(settings, restController));
+        registerHandler.accept(new RestPluginsAction(settings, restController));
+        registerHandler.accept(new RestFielddataAction(settings, restController));
+        registerHandler.accept(new RestNodeAttrsAction(settings, restController));
+        registerHandler.accept(new RestRepositoriesAction(settings, restController));
+        registerHandler.accept(new RestSnapshotAction(settings, restController));
+        registerHandler.accept(new RestTemplatesAction(settings, restController));
         for (ActionPlugin plugin : actionPlugins) {
-            for (Class<? extends RestHandler> handler : plugin.getRestHandlers()) {
-                registerRestHandler(handlers, handler);
+            for (RestHandler handler : plugin.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings,
+                    settingsFilter, indexNameExpressionResolver, nodesInCluster)) {
+                registerHandler.accept(handler);
             }
         }
-        return handlers;
-    }
-
-    private static void registerRestHandler(Set<Class<? extends RestHandler>> handlers, Class<? extends RestHandler> handler) {
-        if (handlers.contains(handler)) {
-            throw new IllegalArgumentException("can't register the same [rest_handler] more than once for [" + handler.getName() + "]");
-        }
-        handlers.add(handler);
+        registerHandler.accept(new RestCatAction(settings, restController, catActions));
     }
 
     @Override
@@ -647,23 +665,6 @@ public class ActionModule extends AbstractModule {
                     bind(supportAction).asEagerSingleton();
                 }
             }
-
-            // Bind the RestController which is required (by Node) even if rest isn't enabled.
-            bind(RestController.class).toInstance(restController);
-
-            // Setup the RestHandlers
-            if (NetworkModule.HTTP_ENABLED.get(settings)) {
-                Multibinder<RestHandler> restHandlers = Multibinder.newSetBinder(binder(), RestHandler.class);
-                Multibinder<AbstractCatAction> catHandlers = Multibinder.newSetBinder(binder(), AbstractCatAction.class);
-                for (Class<? extends RestHandler> handler : setupRestHandlers(actionPlugins)) {
-                    bind(handler).asEagerSingleton();
-                    if (AbstractCatAction.class.isAssignableFrom(handler)) {
-                        catHandlers.addBinding().to(handler.asSubclass(AbstractCatAction.class));
-                    } else {
-                        restHandlers.addBinding().to(handler);
-                    }
-                }
-            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 7a12ab8ace2..920ef1b5438 100644
--- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -18,15 +18,19 @@
  */
 package org.elasticsearch.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
@@ -34,12 +38,26 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Locale;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
 /**
  * A base class for the response of a write operation that involves a single doc
  */
-public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContent {
+public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject {
+
+    private static final String _SHARDS = "_shards";
+    private static final String _INDEX = "_index";
+    private static final String _TYPE = "_type";
+    private static final String _ID = "_id";
+    private static final String _VERSION = "_version";
+    private static final String _SEQ_NO = "_seq_no";
+    private static final String RESULT = "result";
+    private static final String FORCED_REFRESH = "forced_refresh";
 
     /**
      * An enum that represents the the results of CRUD operations, primarily used to communicate the type of
@@ -185,8 +203,9 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
     /**
      * Gets the location of the written document as a string suitable for a {@code Location} header.
      * @param routing any routing used in the request. If null the location doesn't include routing information.
+     *
      */
-    public String getLocation(@Nullable String routing) {
+    public String getLocation(@Nullable String routing) throws URISyntaxException {
         // Absolute path for the location of the document. This should be allowed as of HTTP/1.1:
         // https://tools.ietf.org/html/rfc7231#section-7.1.2
         String index = getIndex();
@@ -204,7 +223,9 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
         if (routing != null) {
             location.append(routingStart).append(routing);
         }
-        return location.toString();
+
+        URI uri = new URI(location.toString());
+        return uri.toASCIIString();
     }
 
     @Override
@@ -214,7 +235,11 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
         type = in.readString();
         id = in.readString();
         version = in.readZLong();
-        seqNo = in.readZLong();
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            seqNo = in.readZLong();
+        } else {
+            seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        }
         forcedRefresh = in.readBoolean();
         result = Result.readFrom(in);
     }
@@ -226,26 +251,49 @@ public abstract class DocWriteResponse extends ReplicationResponse implements Wr
         out.writeString(type);
         out.writeString(id);
         out.writeZLong(version);
-        out.writeZLong(seqNo);
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            out.writeZLong(seqNo);
+        }
         out.writeBoolean(forcedRefresh);
         result.writeTo(out);
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         ReplicationResponse.ShardInfo shardInfo = getShardInfo();
-        builder.field("_index", shardId.getIndexName())
-            .field("_type", type)
-            .field("_id", id)
-            .field("_version", version)
-            .field("result", getResult().getLowercase());
+        builder.field(_INDEX, shardId.getIndexName())
+                .field(_TYPE, type)
+                .field(_ID, id)
+                .field(_VERSION, version)
+                .field(RESULT, getResult().getLowercase());
         if (forcedRefresh) {
-            builder.field("forced_refresh", forcedRefresh);
+            builder.field(FORCED_REFRESH, true);
         }
-        shardInfo.toXContent(builder, params);
+        builder.field(_SHARDS, shardInfo);
         if (getSeqNo() >= 0) {
-            builder.field("_seq_no", getSeqNo());
+            builder.field(_SEQ_NO, getSeqNo());
         }
         return builder;
     }
+
+    /**
+     * Declare the {@link ObjectParser} fields to use when parsing a {@link DocWriteResponse}
+     */
+    protected static void declareParserFields(ConstructingObjectParser<? extends DocWriteResponse, Void> objParser) {
+        objParser.declareString(constructorArg(), new ParseField(_INDEX));
+        objParser.declareString(constructorArg(), new ParseField(_TYPE));
+        objParser.declareString(constructorArg(), new ParseField(_ID));
+        objParser.declareLong(constructorArg(), new ParseField(_VERSION));
+        objParser.declareString(constructorArg(), new ParseField(RESULT));
+        objParser.declareObject(optionalConstructorArg(), (p, c) -> ShardInfo.fromXContent(p), new ParseField(_SHARDS));
+        objParser.declareLong(optionalConstructorArg(), new ParseField(_SEQ_NO));
+        objParser.declareBoolean(DocWriteResponse::setForcedRefresh, new ParseField(FORCED_REFRESH));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
index 29b5a2a8774..87e4df3bc79 100644
--- a/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/ListenableActionFuture.java
@@ -29,5 +29,5 @@ public interface ListenableActionFuture<T> extends ActionFuture<T> {
     /**
      * Add an action listener to be invoked when a response has received.
      */
-    void addListener(final ActionListener<T> listener);
+    void addListener(ActionListener<T> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
index 6704f610ec0..519adc77b84 100644
--- a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
@@ -105,7 +105,7 @@ public final class TaskOperationFailure implements Writeable, ToXContent {
         if (reason != null) {
             builder.field("reason");
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, reason);
+            ElasticsearchException.generateThrowableXContent(builder, params, reason);
             builder.endObject();
         }
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
index f31b1d37376..eab7853eefd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java
@@ -20,12 +20,11 @@
 package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -40,36 +39,47 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAllocationExplainRequest> {
 
-    private static ObjectParser<ClusterAllocationExplainRequest, ParseFieldMatcherSupplier> PARSER = new ObjectParser(
-            "cluster/allocation/explain");
+    private static ObjectParser<ClusterAllocationExplainRequest, Void> PARSER = new ObjectParser<>("cluster/allocation/explain");
     static {
         PARSER.declareString(ClusterAllocationExplainRequest::setIndex, new ParseField("index"));
         PARSER.declareInt(ClusterAllocationExplainRequest::setShard, new ParseField("shard"));
         PARSER.declareBoolean(ClusterAllocationExplainRequest::setPrimary, new ParseField("primary"));
+        PARSER.declareString(ClusterAllocationExplainRequest::setCurrentNode, new ParseField("current_node"));
     }
 
+    @Nullable
     private String index;
+    @Nullable
     private Integer shard;
+    @Nullable
     private Boolean primary;
+    @Nullable
+    private String currentNode;
     private boolean includeYesDecisions = false;
     private boolean includeDiskInfo = false;
 
-    /** Explain the first unassigned shard */
+    /**
+     * Create a new allocation explain request to explain any unassigned shard in the cluster.
+     */
     public ClusterAllocationExplainRequest() {
         this.index = null;
         this.shard = null;
         this.primary = null;
+        this.currentNode = null;
     }
 
     /**
      * Create a new allocation explain request. If {@code primary} is false, the first unassigned replica
      * will be picked for explanation. If no replicas are unassigned, the first assigned replica will
      * be explained.
+     *
+     * Package private for testing.
      */
-    public ClusterAllocationExplainRequest(String index, int shard, boolean primary) {
+    ClusterAllocationExplainRequest(String index, int shard, boolean primary, @Nullable String currentNode) {
         this.index = index;
         this.shard = shard;
         this.primary = primary;
+        this.currentNode = currentNode;
     }
 
     @Override
@@ -93,54 +103,103 @@ public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAl
      * Returns {@code true} iff the first unassigned shard is to be used
      */
     public boolean useAnyUnassignedShard() {
-        return this.index == null && this.shard == null && this.primary == null;
+        return this.index == null && this.shard == null && this.primary == null && this.currentNode == null;
     }
 
+    /**
+     * Sets the index name of the shard to explain.
+     */
     public ClusterAllocationExplainRequest setIndex(String index) {
         this.index = index;
         return this;
     }
 
+    /**
+     * Returns the index name of the shard to explain, or {@code null} to use any unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public String getIndex() {
         return this.index;
     }
 
+    /**
+     * Sets the shard id of the shard to explain.
+     */
     public ClusterAllocationExplainRequest setShard(Integer shard) {
         this.shard = shard;
         return this;
     }
 
+    /**
+     * Returns the shard id of the shard to explain, or {@code null} to use any unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public Integer getShard() {
         return this.shard;
     }
 
+    /**
+     * Sets whether to explain the allocation of the primary shard or a replica shard copy
+     * for the shard id (see {@link #getShard()}).
+     */
     public ClusterAllocationExplainRequest setPrimary(Boolean primary) {
         this.primary = primary;
         return this;
     }
 
+    /**
+     * Returns {@code true} if explaining the primary shard for the shard id (see {@link #getShard()}),
+     * {@code false} if explaining a replica shard copy for the shard id, or {@code null} to use any
+     * unassigned shard (see {@link #useAnyUnassignedShard()}).
+     */
     @Nullable
     public Boolean isPrimary() {
         return this.primary;
     }
 
+    /**
+     * Requests the explain API to explain an already assigned replica shard currently allocated to
+     * the given node.
+     */
+    public ClusterAllocationExplainRequest setCurrentNode(String currentNodeId) {
+        this.currentNode = currentNodeId;
+        return this;
+    }
+
+    /**
+     * Returns the node holding the replica shard to be explained.  Returns {@code null} if any replica shard
+     * can be explained.
+     */
+    @Nullable
+    public String getCurrentNode() {
+        return currentNode;
+    }
+
+    /**
+     * Set to {@code true} to include yes decisions for a particular node.
+     */
     public void includeYesDecisions(boolean includeYesDecisions) {
         this.includeYesDecisions = includeYesDecisions;
     }
 
-    /** Returns true if all decisions should be included. Otherwise only "NO" and "THROTTLE" decisions are returned */
+    /**
+     * Returns {@code true} if yes decisions should be included.  Otherwise only "no" and "throttle"
+     * decisions are returned.
+     */
     public boolean includeYesDecisions() {
         return this.includeYesDecisions;
     }
 
-    /** {@code true} to include information about the gathered disk information of nodes in the cluster */
+    /**
+     * Set to {@code true} to include information about the gathered disk information of nodes in the cluster.
+     */
     public void includeDiskInfo(boolean includeDiskInfo) {
         this.includeDiskInfo = includeDiskInfo;
     }
 
-    /** Returns true if information about disk usage and shard sizes should also be returned */
+    /**
+     * Returns {@code true} if information about disk usage and shard sizes should also be returned.
+     */
     public boolean includeDiskInfo() {
         return this.includeDiskInfo;
     }
@@ -154,13 +213,16 @@ public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAl
             sb.append("index=").append(index);
             sb.append(",shard=").append(shard);
             sb.append(",primary?=").append(primary);
+            if (currentNode != null) {
+                sb.append(",currentNode=").append(currentNode);
+            }
         }
         sb.append(",includeYesDecisions?=").append(includeYesDecisions);
         return sb.toString();
     }
 
     public static ClusterAllocationExplainRequest parse(XContentParser parser) throws IOException {
-        ClusterAllocationExplainRequest req = PARSER.parse(parser, new ClusterAllocationExplainRequest(), () -> ParseFieldMatcher.STRICT);
+        ClusterAllocationExplainRequest req = PARSER.parse(parser, new ClusterAllocationExplainRequest(), null);
         Exception e = req.validate();
         if (e != null) {
             throw new ElasticsearchParseException("'index', 'shard', and 'primary' must be specified in allocation explain request", e);
@@ -170,21 +232,32 @@ public class ClusterAllocationExplainRequest extends MasterNodeRequest<ClusterAl
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
+        checkVersion(in.getVersion());
         super.readFrom(in);
         this.index = in.readOptionalString();
         this.shard = in.readOptionalVInt();
         this.primary = in.readOptionalBoolean();
+        this.currentNode = in.readOptionalString();
         this.includeYesDecisions = in.readBoolean();
         this.includeDiskInfo = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        checkVersion(out.getVersion());
         super.writeTo(out);
         out.writeOptionalString(index);
         out.writeOptionalVInt(shard);
         out.writeOptionalBoolean(primary);
+        out.writeOptionalString(currentNode);
         out.writeBoolean(includeYesDecisions);
         out.writeBoolean(includeDiskInfo);
     }
+
+    private void checkVersion(Version version) {
+        if (version.before(Version.V_5_2_0_UNRELEASED)) {
+            throw new IllegalArgumentException("cannot explain shards in a mixed-cluster with pre-" + Version.V_5_2_0_UNRELEASED +
+                                               " nodes, node version [" + version + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
index 343d1756d75..038657a277c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestBuilder.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
@@ -65,6 +64,15 @@ public class ClusterAllocationExplainRequestBuilder
         return this;
     }
 
+    /**
+     * Requests the explain API to explain an already assigned replica shard currently allocated to
+     * the given node.
+     */
+    public ClusterAllocationExplainRequestBuilder setCurrentNode(String currentNode) {
+        request.setCurrentNode(currentNode);
+        return this;
+    }
+
     /**
      * Signal that the first unassigned shard should be used
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
index cc586bd1a58..83a847ba01e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainResponse.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index fd06c685b95..e2911c227a8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -21,285 +21,184 @@ package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Locale;
+
+import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent;
 
 /**
- * A {@code ClusterAllocationExplanation} is an explanation of why a shard may or may not be allocated to nodes. It also includes weights
- * for where the shard is likely to be assigned. It is an immutable class
+ * A {@code ClusterAllocationExplanation} is an explanation of why a shard is unassigned,
+ * or if it is not unassigned, then which nodes it could possibly be relocated to.
+ * It is an immutable class.
  */
 public final class ClusterAllocationExplanation implements ToXContent, Writeable {
 
-    private final ShardId shard;
-    private final boolean primary;
-    private final boolean hasPendingAsyncFetch;
-    private final String assignedNodeId;
-    private final UnassignedInfo unassignedInfo;
-    private final long allocationDelayMillis;
-    private final long remainingDelayMillis;
-    private final Map<DiscoveryNode, NodeExplanation> nodeExplanations;
+    private final ShardRouting shardRouting;
+    private final DiscoveryNode currentNode;
+    private final DiscoveryNode relocationTargetNode;
     private final ClusterInfo clusterInfo;
+    private final ShardAllocationDecision shardAllocationDecision;
 
-    public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId, long allocationDelayMillis,
-                                        long remainingDelayMillis, @Nullable UnassignedInfo unassignedInfo, boolean hasPendingAsyncFetch,
-                                        Map<DiscoveryNode, NodeExplanation> nodeExplanations, @Nullable ClusterInfo clusterInfo) {
-        this.shard = shard;
-        this.primary = primary;
-        this.hasPendingAsyncFetch = hasPendingAsyncFetch;
-        this.assignedNodeId = assignedNodeId;
-        this.unassignedInfo = unassignedInfo;
-        this.allocationDelayMillis = allocationDelayMillis;
-        this.remainingDelayMillis = remainingDelayMillis;
-        this.nodeExplanations = nodeExplanations;
+    public ClusterAllocationExplanation(ShardRouting shardRouting, @Nullable DiscoveryNode currentNode,
+                                        @Nullable DiscoveryNode relocationTargetNode, @Nullable ClusterInfo clusterInfo,
+                                        ShardAllocationDecision shardAllocationDecision) {
+        this.shardRouting = shardRouting;
+        this.currentNode = currentNode;
+        this.relocationTargetNode = relocationTargetNode;
         this.clusterInfo = clusterInfo;
+        this.shardAllocationDecision = shardAllocationDecision;
     }
 
     public ClusterAllocationExplanation(StreamInput in) throws IOException {
-        this.shard = ShardId.readShardId(in);
-        this.primary = in.readBoolean();
-        this.hasPendingAsyncFetch = in.readBoolean();
-        this.assignedNodeId = in.readOptionalString();
-        this.unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
-        this.allocationDelayMillis = in.readVLong();
-        this.remainingDelayMillis = in.readVLong();
-
-        int mapSize = in.readVInt();
-        Map<DiscoveryNode, NodeExplanation> nodeToExplanation = new HashMap<>(mapSize);
-        for (int i = 0; i < mapSize; i++) {
-            NodeExplanation nodeExplanation = new NodeExplanation(in);
-            nodeToExplanation.put(nodeExplanation.getNode(), nodeExplanation);
-        }
-        this.nodeExplanations = nodeToExplanation;
-        if (in.readBoolean()) {
-            this.clusterInfo = new ClusterInfo(in);
-        } else {
-            this.clusterInfo = null;
-        }
+        this.shardRouting = new ShardRouting(in);
+        this.currentNode = in.readOptionalWriteable(DiscoveryNode::new);
+        this.relocationTargetNode = in.readOptionalWriteable(DiscoveryNode::new);
+        this.clusterInfo = in.readOptionalWriteable(ClusterInfo::new);
+        this.shardAllocationDecision = new ShardAllocationDecision(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        this.getShard().writeTo(out);
-        out.writeBoolean(this.isPrimary());
-        out.writeBoolean(this.isStillFetchingShardData());
-        out.writeOptionalString(this.getAssignedNodeId());
-        out.writeOptionalWriteable(this.getUnassignedInfo());
-        out.writeVLong(allocationDelayMillis);
-        out.writeVLong(remainingDelayMillis);
-
-        out.writeVInt(this.nodeExplanations.size());
-        for (NodeExplanation explanation : this.nodeExplanations.values()) {
-            explanation.writeTo(out);
-        }
-        if (this.clusterInfo != null) {
-            out.writeBoolean(true);
-            this.clusterInfo.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
+        shardRouting.writeTo(out);
+        out.writeOptionalWriteable(currentNode);
+        out.writeOptionalWriteable(relocationTargetNode);
+        out.writeOptionalWriteable(clusterInfo);
+        shardAllocationDecision.writeTo(out);
     }
 
-    /** Return the shard that the explanation is about */
+    /**
+     * Returns the shard that the explanation is about.
+     */
     public ShardId getShard() {
-        return this.shard;
+        return shardRouting.shardId();
     }
 
-    /** Return true if the explained shard is primary, false otherwise */
+    /**
+     * Returns {@code true} if the explained shard is primary, {@code false} otherwise.
+     */
     public boolean isPrimary() {
-        return this.primary;
+        return shardRouting.primary();
     }
 
-    /** Return turn if shard data is still being fetched for the allocation */
-    public boolean isStillFetchingShardData() {
-        return this.hasPendingAsyncFetch;
+    /**
+     * Returns the current {@link ShardRoutingState} of the shard.
+     */
+    public ShardRoutingState getShardState() {
+        return shardRouting.state();
     }
 
-    /** Return turn if the shard is assigned to a node */
-    public boolean isAssigned() {
-        return this.assignedNodeId != null;
-    }
-
-    /** Return the assigned node id or null if not assigned */
+    /**
+     * Returns the currently assigned node, or {@code null} if the shard is unassigned.
+     */
     @Nullable
-    public String getAssignedNodeId() {
-        return this.assignedNodeId;
+    public DiscoveryNode getCurrentNode() {
+        return currentNode;
     }
 
-    /** Return the unassigned info for the shard or null if the shard is assigned */
+    /**
+     * Returns the relocating target node, or {@code null} if the shard is not in the {@link ShardRoutingState#RELOCATING} state.
+     */
+    @Nullable
+    public DiscoveryNode getRelocationTargetNode() {
+        return relocationTargetNode;
+    }
+
+    /**
+     * Returns the unassigned info for the shard, or {@code null} if the shard is active.
+     */
     @Nullable
     public UnassignedInfo getUnassignedInfo() {
-        return this.unassignedInfo;
+        return shardRouting.unassignedInfo();
     }
 
-    /** Return the configured delay before the shard can be allocated in milliseconds */
-    public long getAllocationDelayMillis() {
-        return this.allocationDelayMillis;
-    }
-
-    /** Return the remaining allocation delay for this shard in milliseconds */
-    public long getRemainingDelayMillis() {
-        return this.remainingDelayMillis;
-    }
-
-    /** Return a map of node to the explanation for that node */
-    public Map<DiscoveryNode, NodeExplanation> getNodeExplanations() {
-        return this.nodeExplanations;
-    }
-
-    /** Return the cluster disk info for the cluster or null if none available */
+    /**
+     * Returns the cluster disk info for the cluster, or {@code null} if none available.
+     */
     @Nullable
     public ClusterInfo getClusterInfo() {
         return this.clusterInfo;
     }
 
+    /** \
+     * Returns the shard allocation decision for attempting to assign or move the shard.
+     */
+    public ShardAllocationDecision getShardAllocationDecision() {
+        return shardAllocationDecision;
+    }
+
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(); {
-            builder.startObject("shard"); {
-                builder.field("index", shard.getIndexName());
-                builder.field("index_uuid", shard.getIndex().getUUID());
-                builder.field("id", shard.getId());
-                builder.field("primary", primary);
+            builder.field("index", shardRouting.getIndexName());
+            builder.field("shard", shardRouting.getId());
+            builder.field("primary", shardRouting.primary());
+            builder.field("current_state", shardRouting.state().toString().toLowerCase(Locale.ROOT));
+            if (shardRouting.unassignedInfo() != null) {
+                unassignedInfoToXContent(shardRouting.unassignedInfo(), builder);
             }
-            builder.endObject(); // end shard
-            builder.field("assigned", this.assignedNodeId != null);
-            // If assigned, show the node id of the node it's assigned to
-            if (assignedNodeId != null) {
-                builder.field("assigned_node_id", this.assignedNodeId);
-            }
-            builder.field("shard_state_fetch_pending", this.hasPendingAsyncFetch);
-            // If we have unassigned info, show that
-            if (unassignedInfo != null) {
-                unassignedInfo.toXContent(builder, params);
-                builder.timeValueField("allocation_delay_in_millis", "allocation_delay", TimeValue.timeValueMillis(allocationDelayMillis));
-                builder.timeValueField("remaining_delay_in_millis", "remaining_delay", TimeValue.timeValueMillis(remainingDelayMillis));
-            }
-            builder.startObject("nodes"); {
-                for (NodeExplanation explanation : nodeExplanations.values()) {
-                    explanation.toXContent(builder, params);
+            if (currentNode != null) {
+                builder.startObject("current_node");
+                {
+                    discoveryNodeToXContent(currentNode, true, builder);
+                    if (shardAllocationDecision.getMoveDecision().isDecisionTaken()
+                            && shardAllocationDecision.getMoveDecision().getCurrentNodeRanking() > 0) {
+                        builder.field("weight_ranking", shardAllocationDecision.getMoveDecision().getCurrentNodeRanking());
+                    }
                 }
+                builder.endObject();
             }
-            builder.endObject(); // end nodes
             if (this.clusterInfo != null) {
                 builder.startObject("cluster_info"); {
                     this.clusterInfo.toXContent(builder, params);
                 }
                 builder.endObject(); // end "cluster_info"
             }
+            if (shardAllocationDecision.isDecisionTaken()) {
+                shardAllocationDecision.toXContent(builder, params);
+            } else {
+                String explanation;
+                if (shardRouting.state() == ShardRoutingState.RELOCATING) {
+                    explanation = "the shard is in the process of relocating from node [" + currentNode.getName() + "] " +
+                                  "to node [" + relocationTargetNode.getName() + "], wait until relocation has completed";
+                } else {
+                    assert shardRouting.state() == ShardRoutingState.INITIALIZING;
+                    explanation = "the shard is in the process of initializing on node [" + currentNode.getName() + "], " +
+                                  "wait until initialization has completed";
+                }
+                builder.field("explanation", explanation);
+            }
         }
         builder.endObject(); // end wrapping object
         return builder;
     }
 
-    /** An Enum representing the final decision for a shard allocation on a node */
-    public enum FinalDecision {
-        // Yes, the shard can be assigned
-        YES((byte) 0),
-        // No, the shard cannot be assigned
-        NO((byte) 1),
-        // The shard is already assigned to this node
-        ALREADY_ASSIGNED((byte) 2);
+    private XContentBuilder unassignedInfoToXContent(UnassignedInfo unassignedInfo, XContentBuilder builder)
+        throws IOException {
 
-        private final byte id;
-
-        FinalDecision (byte id) {
-            this.id = id;
+        builder.startObject("unassigned_info");
+        builder.field("reason", unassignedInfo.getReason());
+        builder.field("at", UnassignedInfo.DATE_TIME_FORMATTER.printer().print(unassignedInfo.getUnassignedTimeInMillis()));
+        if (unassignedInfo.getNumFailedAllocations() >  0) {
+            builder.field("failed_allocation_attempts", unassignedInfo.getNumFailedAllocations());
         }
-
-        private static FinalDecision fromId(byte id) {
-            switch (id) {
-                case 0: return YES;
-                case 1: return NO;
-                case 2: return ALREADY_ASSIGNED;
-                default:
-                    throw new IllegalArgumentException("unknown id for final decision: [" + id + "]");
-            }
-        }
-
-        @Override
-        public String toString() {
-            switch (id) {
-                case 0: return "YES";
-                case 1: return "NO";
-                case 2: return "ALREADY_ASSIGNED";
-                default:
-                    throw new IllegalArgumentException("unknown id for final decision: [" + id + "]");
-            }
-        }
-
-        static FinalDecision readFrom(StreamInput in) throws IOException {
-            return fromId(in.readByte());
-        }
-
-        void writeTo(StreamOutput out) throws IOException {
-            out.writeByte(id);
-        }
-    }
-
-    /** An Enum representing the state of the shard store's copy of the data on a node */
-    public enum StoreCopy {
-        // No data for this shard is on the node
-        NONE((byte) 0),
-        // A copy of the data is available on this node
-        AVAILABLE((byte) 1),
-        // The copy of the data on the node is corrupt
-        CORRUPT((byte) 2),
-        // There was an error reading this node's copy of the data
-        IO_ERROR((byte) 3),
-        // The copy of the data on the node is stale
-        STALE((byte) 4),
-        // It's unknown what the copy of the data is
-        UNKNOWN((byte) 5);
-
-        private final byte id;
-
-        StoreCopy (byte id) {
-            this.id = id;
-        }
-
-        private static StoreCopy fromId(byte id) {
-            switch (id) {
-                case 0: return NONE;
-                case 1: return AVAILABLE;
-                case 2: return CORRUPT;
-                case 3: return IO_ERROR;
-                case 4: return STALE;
-                case 5: return UNKNOWN;
-                default:
-                    throw new IllegalArgumentException("unknown id for store copy: [" + id + "]");
-            }
-        }
-
-        @Override
-        public String toString() {
-            switch (id) {
-                case 0: return "NONE";
-                case 1: return "AVAILABLE";
-                case 2: return "CORRUPT";
-                case 3: return "IO_ERROR";
-                case 4: return "STALE";
-                case 5: return "UNKNOWN";
-                default:
-                    throw new IllegalArgumentException("unknown id for store copy: [" + id + "]");
-            }
-        }
-
-        static StoreCopy readFrom(StreamInput in) throws IOException {
-            return fromId(in.readByte());
-        }
-
-        void writeTo(StreamOutput out) throws IOException {
-            out.writeByte(id);
+        String details = unassignedInfo.getDetails();
+        if (details != null) {
+            builder.field("details", details);
         }
+        builder.field("last_allocation_status", AllocationDecision.fromAllocationStatus(unassignedInfo.getLastAllocationStatus()));
+        builder.endObject();
+        return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
deleted file mode 100644
index 9fdf97b320c..00000000000
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Map;
-/** The cluster allocation explanation for a single node */
-public class NodeExplanation implements Writeable, ToXContent {
-    private final DiscoveryNode node;
-    private final Decision nodeDecision;
-    private final Float nodeWeight;
-    private final IndicesShardStoresResponse.StoreStatus storeStatus;
-    private final ClusterAllocationExplanation.FinalDecision finalDecision;
-    private final ClusterAllocationExplanation.StoreCopy storeCopy;
-    private final String finalExplanation;
-
-    public NodeExplanation(final DiscoveryNode node, final Decision nodeDecision, final Float nodeWeight,
-                           @Nullable final IndicesShardStoresResponse.StoreStatus storeStatus,
-                           final ClusterAllocationExplanation.FinalDecision finalDecision,
-                           final String finalExplanation,
-                           final ClusterAllocationExplanation.StoreCopy storeCopy) {
-        this.node = node;
-        this.nodeDecision = nodeDecision;
-        this.nodeWeight = nodeWeight;
-        this.storeStatus = storeStatus;
-        this.finalDecision = finalDecision;
-        this.finalExplanation = finalExplanation;
-        this.storeCopy = storeCopy;
-    }
-
-    public NodeExplanation(StreamInput in) throws IOException {
-        this.node = new DiscoveryNode(in);
-        this.nodeDecision = Decision.readFrom(in);
-        this.nodeWeight = in.readFloat();
-        if (in.readBoolean()) {
-            this.storeStatus = IndicesShardStoresResponse.StoreStatus.readStoreStatus(in);
-        } else {
-            this.storeStatus = null;
-        }
-        this.finalDecision = ClusterAllocationExplanation.FinalDecision.readFrom(in);
-        this.finalExplanation = in.readString();
-        this.storeCopy = ClusterAllocationExplanation.StoreCopy.readFrom(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        node.writeTo(out);
-        nodeDecision.writeTo(out);
-        out.writeFloat(nodeWeight);
-        if (storeStatus == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            storeStatus.writeTo(out);
-        }
-        finalDecision.writeTo(out);
-        out.writeString(finalExplanation);
-        storeCopy.writeTo(out);
-    }
-
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(node.getId()); {
-            builder.field("node_name", node.getName());
-            builder.startObject("node_attributes"); {
-                for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
-                    builder.field(attrEntry.getKey(), attrEntry.getValue());
-                }
-            }
-            builder.endObject(); // end attributes
-            builder.startObject("store"); {
-                builder.field("shard_copy", storeCopy.toString());
-                if (storeStatus != null) {
-                    final Throwable storeErr = storeStatus.getStoreException();
-                    if (storeErr != null) {
-                        builder.field("store_exception", ExceptionsHelper.detailedMessage(storeErr));
-                    }
-                }
-            }
-            builder.endObject(); // end store
-            builder.field("final_decision", finalDecision.toString());
-            builder.field("final_explanation", finalExplanation);
-            builder.field("weight", nodeWeight);
-            builder.startArray("decisions");
-            nodeDecision.toXContent(builder, params);
-            builder.endArray();
-        }
-        builder.endObject(); // end node <uuid>
-        return builder;
-    }
-
-    public DiscoveryNode getNode() {
-        return this.node;
-    }
-
-    public Decision getDecision() {
-        return this.nodeDecision;
-    }
-
-    public Float getWeight() {
-        return this.nodeWeight;
-    }
-
-    @Nullable
-    public IndicesShardStoresResponse.StoreStatus getStoreStatus() {
-        return this.storeStatus;
-    }
-
-    public ClusterAllocationExplanation.FinalDecision getFinalDecision() {
-        return this.finalDecision;
-    }
-
-    public String getFinalExplanation() {
-        return this.finalExplanation;
-    }
-
-    public ClusterAllocationExplanation.StoreCopy getStoreCopy() {
-        return this.storeCopy;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 5aa35a059fb..4d4796aaf3a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -19,13 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.apache.lucene.index.CorruptIndexException;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterInfo;
@@ -33,34 +27,25 @@ import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation.DebugMode;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
 
 /**
  * The {@code TransportClusterAllocationExplainAction} is responsible for actually executing the explanation of a shard's allocation on the
@@ -72,7 +57,6 @@ public class TransportClusterAllocationExplainAction
     private final ClusterInfoService clusterInfoService;
     private final AllocationDeciders allocationDeciders;
     private final ShardsAllocator shardAllocator;
-    private final TransportIndicesShardStoresAction shardStoresAction;
     private final GatewayAllocator gatewayAllocator;
 
     @Inject
@@ -80,14 +64,12 @@ public class TransportClusterAllocationExplainAction
                                                    ThreadPool threadPool, ActionFilters actionFilters,
                                                    IndexNameExpressionResolver indexNameExpressionResolver,
                                                    ClusterInfoService clusterInfoService, AllocationDeciders allocationDeciders,
-                                                   ShardsAllocator shardAllocator, TransportIndicesShardStoresAction shardStoresAction,
-                                                   GatewayAllocator gatewayAllocator) {
+                                                   ShardsAllocator shardAllocator, GatewayAllocator gatewayAllocator) {
         super(settings, ClusterAllocationExplainAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, ClusterAllocationExplainRequest::new);
         this.clusterInfoService = clusterInfoService;
         this.allocationDeciders = allocationDeciders;
         this.shardAllocator = shardAllocator;
-        this.shardStoresAction = shardStoresAction;
         this.gatewayAllocator = gatewayAllocator;
     }
 
@@ -106,172 +88,6 @@ public class TransportClusterAllocationExplainAction
         return new ClusterAllocationExplainResponse();
     }
 
-    /**
-     * Return the decisions for the given {@code ShardRouting} on the given {@code RoutingNode}. If {@code includeYesDecisions} is not true,
-     * only non-YES (NO and THROTTLE) decisions are returned.
-     */
-    public static Decision tryShardOnNode(ShardRouting shard, RoutingNode node, RoutingAllocation allocation, boolean includeYesDecisions) {
-        Decision d = allocation.deciders().canAllocate(shard, node, allocation);
-        if (includeYesDecisions) {
-            return d;
-        } else {
-            Decision.Multi nonYesDecisions = new Decision.Multi();
-            List<Decision> decisions = d.getDecisions();
-            for (Decision decision : decisions) {
-                if (decision.type() != Decision.Type.YES) {
-                    nonYesDecisions.add(decision);
-                }
-            }
-            return nonYesDecisions;
-        }
-    }
-
-    /**
-     * Construct a {@code WeightedDecision} object for the given shard given all the metadata. This also attempts to construct the human
-     * readable FinalDecision and final explanation as part of the explanation.
-     */
-    public static NodeExplanation calculateNodeExplanation(ShardRouting shard,
-                                                           IndexMetaData indexMetaData,
-                                                           DiscoveryNode node,
-                                                           Decision nodeDecision,
-                                                           Float nodeWeight,
-                                                           IndicesShardStoresResponse.StoreStatus storeStatus,
-                                                           String assignedNodeId,
-                                                           Set<String> activeAllocationIds,
-                                                           boolean hasPendingAsyncFetch) {
-        final ClusterAllocationExplanation.FinalDecision finalDecision;
-        final ClusterAllocationExplanation.StoreCopy storeCopy;
-        final String finalExplanation;
-
-        if (storeStatus == null) {
-            // No copies of the data
-            storeCopy = ClusterAllocationExplanation.StoreCopy.NONE;
-        } else {
-            final Exception storeErr = storeStatus.getStoreException();
-            if (storeErr != null) {
-                if (ExceptionsHelper.unwrapCause(storeErr) instanceof CorruptIndexException) {
-                    storeCopy = ClusterAllocationExplanation.StoreCopy.CORRUPT;
-                } else {
-                    storeCopy = ClusterAllocationExplanation.StoreCopy.IO_ERROR;
-                }
-            } else if (activeAllocationIds.isEmpty()) {
-                // The ids are only empty if dealing with a legacy index
-                // TODO: fetch the shard state versions and display here?
-                storeCopy = ClusterAllocationExplanation.StoreCopy.UNKNOWN;
-            } else if (activeAllocationIds.contains(storeStatus.getAllocationId())) {
-                storeCopy = ClusterAllocationExplanation.StoreCopy.AVAILABLE;
-            } else {
-                // Otherwise, this is a stale copy of the data (allocation ids don't match)
-                storeCopy = ClusterAllocationExplanation.StoreCopy.STALE;
-            }
-        }
-
-        if (node.getId().equals(assignedNodeId)) {
-            finalDecision = ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED;
-            finalExplanation = "the shard is already assigned to this node";
-        } else if (shard.unassigned() && shard.primary() == false &&
-                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && nodeDecision.type() != Decision.Type.YES) {
-            finalExplanation = "the shard cannot be assigned because allocation deciders return a " + nodeDecision.type().name() +
-                    " decision";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.unassigned() && shard.primary() == false &&
-                shard.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED && hasPendingAsyncFetch) {
-            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() &&
-                (shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE ||
-                    shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT)
-                && hasPendingAsyncFetch) {
-            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.STALE) {
-            finalExplanation = "the copy of the shard is stale, allocation ids do not match";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.NONE) {
-            finalExplanation = "there is no copy of the shard available";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.CORRUPT) {
-            finalExplanation = "the copy of the shard is corrupt";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else if (shard.primary() && shard.unassigned() && shard.recoverySource().getType() == RecoverySource.Type.EXISTING_STORE &&
-                storeCopy == ClusterAllocationExplanation.StoreCopy.IO_ERROR) {
-            finalExplanation = "the copy of the shard cannot be read";
-            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-        } else {
-            if (nodeDecision.type() == Decision.Type.NO) {
-                finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
-                finalExplanation = "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision";
-            } else {
-                // TODO: handle throttling decision better here
-                finalDecision = ClusterAllocationExplanation.FinalDecision.YES;
-                if (storeCopy == ClusterAllocationExplanation.StoreCopy.AVAILABLE) {
-                    finalExplanation = "the shard can be assigned and the node contains a valid copy of the shard data";
-                } else {
-                    finalExplanation = "the shard can be assigned";
-                }
-            }
-        }
-        return new NodeExplanation(node, nodeDecision, nodeWeight, storeStatus, finalDecision, finalExplanation, storeCopy);
-    }
-
-
-    /**
-     * For the given {@code ShardRouting}, return the explanation of the allocation for that shard on all nodes. If {@code
-     * includeYesDecisions} is true, returns all decisions, otherwise returns only 'NO' and 'THROTTLE' decisions.
-     */
-    public static ClusterAllocationExplanation explainShard(ShardRouting shard, RoutingAllocation allocation, RoutingNodes routingNodes,
-                                                            boolean includeYesDecisions, ShardsAllocator shardAllocator,
-                                                            List<IndicesShardStoresResponse.StoreStatus> shardStores,
-                                                            GatewayAllocator gatewayAllocator, ClusterInfo clusterInfo) {
-        // don't short circuit deciders, we want a full explanation
-        allocation.debugDecision(true);
-        // get the existing unassigned info if available
-        UnassignedInfo ui = shard.unassignedInfo();
-
-        Map<DiscoveryNode, Decision> nodeToDecision = new HashMap<>();
-        for (RoutingNode node : routingNodes) {
-            DiscoveryNode discoNode = node.node();
-            if (discoNode.isDataNode()) {
-                Decision d = tryShardOnNode(shard, node, allocation, includeYesDecisions);
-                nodeToDecision.put(discoNode, d);
-            }
-        }
-        long remainingDelayMillis = 0;
-        final MetaData metadata = allocation.metaData();
-        final IndexMetaData indexMetaData = metadata.index(shard.index());
-        long allocationDelayMillis = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).getMillis();
-        if (ui != null && ui.isDelayed()) {
-            long remainingDelayNanos = ui.getRemainingDelay(System.nanoTime(), indexMetaData.getSettings());
-            remainingDelayMillis = TimeValue.timeValueNanos(remainingDelayNanos).millis();
-        }
-
-        // Calculate weights for each of the nodes
-        Map<DiscoveryNode, Float> weights = shardAllocator.weighShard(allocation, shard);
-
-        Map<DiscoveryNode, IndicesShardStoresResponse.StoreStatus> nodeToStatus = new HashMap<>(shardStores.size());
-        for (IndicesShardStoresResponse.StoreStatus status : shardStores) {
-            nodeToStatus.put(status.getNode(), status);
-        }
-
-        Map<DiscoveryNode, NodeExplanation> explanations = new HashMap<>(shardStores.size());
-        for (Map.Entry<DiscoveryNode, Decision> entry : nodeToDecision.entrySet()) {
-            DiscoveryNode node = entry.getKey();
-            Decision decision = entry.getValue();
-            Float weight = weights.get(node);
-            IndicesShardStoresResponse.StoreStatus storeStatus = nodeToStatus.get(node);
-            NodeExplanation nodeExplanation = calculateNodeExplanation(shard, indexMetaData, node, decision, weight,
-                    storeStatus, shard.currentNodeId(), indexMetaData.inSyncAllocationIds(shard.getId()),
-                    allocation.hasPendingAsyncFetch());
-            explanations.put(node, nodeExplanation);
-        }
-        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(),
-                shard.currentNodeId(), allocationDelayMillis, remainingDelayMillis, ui,
-                gatewayAllocator.hasFetchPending(shard.shardId(), shard.primary()), explanations, clusterInfo);
-    }
-
     @Override
     protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
                                    final ActionListener<ClusterAllocationExplainResponse> listener) {
@@ -280,66 +96,105 @@ public class TransportClusterAllocationExplainAction
         final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
                 clusterInfo, System.nanoTime(), false);
 
+        ShardRouting shardRouting = findShardToExplain(request, allocation);
+        logger.debug("explaining the allocation for [{}], found shard [{}]", request, shardRouting);
+
+        ClusterAllocationExplanation cae = explainShard(shardRouting, allocation,
+            request.includeDiskInfo() ? clusterInfo : null, request.includeYesDecisions(), gatewayAllocator, shardAllocator);
+        listener.onResponse(new ClusterAllocationExplainResponse(cae));
+    }
+
+    // public for testing
+    public static ClusterAllocationExplanation explainShard(ShardRouting shardRouting, RoutingAllocation allocation,
+                                                            ClusterInfo clusterInfo, boolean includeYesDecisions,
+                                                            GatewayAllocator gatewayAllocator, ShardsAllocator shardAllocator) {
+        allocation.setDebugMode(includeYesDecisions ? DebugMode.ON : DebugMode.EXCLUDE_YES_DECISIONS);
+
+        ShardAllocationDecision shardDecision;
+        if (shardRouting.initializing() || shardRouting.relocating()) {
+            shardDecision = ShardAllocationDecision.NOT_TAKEN;
+        } else {
+            AllocateUnassignedDecision allocateDecision = shardRouting.unassigned() ?
+                gatewayAllocator.decideUnassignedShardAllocation(shardRouting, allocation) : AllocateUnassignedDecision.NOT_TAKEN;
+            if (allocateDecision.isDecisionTaken() == false) {
+                shardDecision = shardAllocator.decideShardAllocation(shardRouting, allocation);
+            } else {
+                shardDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN);
+            }
+        }
+
+        return new ClusterAllocationExplanation(shardRouting,
+            shardRouting.currentNodeId() != null ? allocation.nodes().get(shardRouting.currentNodeId()) : null,
+            shardRouting.relocatingNodeId() != null ? allocation.nodes().get(shardRouting.relocatingNodeId()) : null,
+            clusterInfo, shardDecision);
+    }
+
+    // public for testing
+    public static ShardRouting findShardToExplain(ClusterAllocationExplainRequest request, RoutingAllocation allocation) {
         ShardRouting foundShard = null;
         if (request.useAnyUnassignedShard()) {
             // If we can use any shard, just pick the first unassigned one (if there are any)
-            RoutingNodes.UnassignedShards.UnassignedIterator ui = routingNodes.unassigned().iterator();
+            RoutingNodes.UnassignedShards.UnassignedIterator ui = allocation.routingNodes().unassigned().iterator();
             if (ui.hasNext()) {
                 foundShard = ui.next();
             }
+            if (foundShard == null) {
+                throw new IllegalStateException("unable to find any unassigned shards to explain [" + request + "]");
+            }
         } else {
             String index = request.getIndex();
             int shard = request.getShard();
             if (request.isPrimary()) {
                 // If we're looking for the primary shard, there's only one copy, so pick it directly
                 foundShard = allocation.routingTable().shardRoutingTable(index, shard).primaryShard();
+                if (request.getCurrentNode() != null) {
+                    DiscoveryNode primaryNode = allocation.nodes().resolveNode(request.getCurrentNode());
+                    // the primary is assigned to a node other than the node specified in the request
+                    if (primaryNode.getId().equals(foundShard.currentNodeId()) == false) {
+                        throw new IllegalStateException("unable to find primary shard assigned to node [" + request.getCurrentNode() + "]");
+                    }
+                }
             } else {
                 // If looking for a replica, go through all the replica shards
                 List<ShardRouting> replicaShardRoutings = allocation.routingTable().shardRoutingTable(index, shard).replicaShards();
-                if (replicaShardRoutings.size() > 0) {
-                    // Pick the first replica at the very least
-                    foundShard = replicaShardRoutings.get(0);
-                    // In case there are multiple replicas where some are assigned and some aren't,
-                    // try to find one that is unassigned at least
+                if (request.getCurrentNode() != null) {
+                    // the request is to explain a replica shard already assigned on a particular node,
+                    // so find that shard copy
+                    DiscoveryNode replicaNode = allocation.nodes().resolveNode(request.getCurrentNode());
                     for (ShardRouting replica : replicaShardRoutings) {
-                        if (replica.unassigned()) {
+                        if (replicaNode.getId().equals(replica.currentNodeId())) {
                             foundShard = replica;
                             break;
                         }
                     }
+                    if (foundShard == null) {
+                        throw new IllegalStateException("unable to find a replica shard assigned to node [" +
+                                                            request.getCurrentNode() + "]");
+                    }
+                } else {
+                    if (replicaShardRoutings.size() > 0) {
+                        // Pick the first replica at the very least
+                        foundShard = replicaShardRoutings.get(0);
+                        for (ShardRouting replica : replicaShardRoutings) {
+                            // In case there are multiple replicas where some are assigned and some aren't,
+                            // try to find one that is unassigned at least
+                            if (replica.unassigned()) {
+                                foundShard = replica;
+                                break;
+                            } else if (replica.started() && (foundShard.initializing() || foundShard.relocating())) {
+                                // prefer started shards to initializing or relocating shards because started shards
+                                // can be explained
+                                foundShard = replica;
+                            }
+                        }
+                    }
                 }
             }
         }
 
         if (foundShard == null) {
-            listener.onFailure(new ElasticsearchException("unable to find any shards to explain [{}] in the routing table", request));
-            return;
+            throw new IllegalStateException("unable to find any shards to explain [" + request + "] in the routing table");
         }
-        final ShardRouting shardRouting = foundShard;
-        logger.debug("explaining the allocation for [{}], found shard [{}]", request, shardRouting);
-
-        getShardStores(shardRouting, new ActionListener<IndicesShardStoresResponse>() {
-            @Override
-            public void onResponse(IndicesShardStoresResponse shardStoreResponse) {
-                ImmutableOpenIntMap<List<IndicesShardStoresResponse.StoreStatus>> shardStatuses =
-                        shardStoreResponse.getStoreStatuses().get(shardRouting.getIndexName());
-                List<IndicesShardStoresResponse.StoreStatus> shardStoreStatus = shardStatuses.get(shardRouting.id());
-                ClusterAllocationExplanation cae = explainShard(shardRouting, allocation, routingNodes,
-                        request.includeYesDecisions(), shardAllocator, shardStoreStatus, gatewayAllocator,
-                        request.includeDiskInfo() ? clusterInfo : null);
-                listener.onResponse(new ClusterAllocationExplainResponse(cae));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                listener.onFailure(e);
-            }
-        });
-    }
-
-    private void getShardStores(ShardRouting shard, final ActionListener<IndicesShardStoresResponse> listener) {
-        IndicesShardStoresRequest request = new IndicesShardStoresRequest(shard.getIndexName());
-        request.shardStatuses("all");
-        shardStoresAction.execute(request, listener);
+        return foundShard;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
index e4a575dcf79..a9a2c36970e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -24,19 +24,19 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 
-public class ClusterHealthResponse extends ActionResponse implements StatusToXContent {
+public class ClusterHealthResponse extends ActionResponse implements StatusToXContentObject {
     private String clusterName;
     private int numberOfPendingTasks = 0;
     private int numberOfInFlightFetch = 0;
@@ -200,18 +200,9 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
         taskMaxWaitingTime.writeTo(out);
     }
 
-
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 
     @Override
@@ -240,6 +231,7 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.field(CLUSTER_NAME, getClusterName());
         builder.field(STATUS, getStatus().name().toLowerCase(Locale.ROOT));
         builder.field(TIMED_OUT, isTimedOut());
@@ -268,6 +260,7 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
             }
             builder.endObject();
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 7dd2c0df84c..44c604dc8b8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -85,37 +86,55 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
     protected void masterOperation(Task task, final ClusterHealthRequest request, final ClusterState unusedState, final ActionListener<ClusterHealthResponse> listener) {
         if (request.waitForEvents() != null) {
             final long endTimeMS = TimeValue.nsecToMSec(System.nanoTime()) + request.timeout().millis();
-            clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new ClusterStateUpdateTask(request.waitForEvents()) {
-                @Override
-                public ClusterState execute(ClusterState currentState) {
-                    return currentState;
-                }
+            if (request.local()) {
+                clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new LocalClusterUpdateTask(request.waitForEvents()) {
+                    @Override
+                    public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
+                        return unchanged();
+                    }
 
-                @Override
-                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                    final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
-                    final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
-                    request.timeout(newTimeout);
-                    executeHealth(request, listener);
-                }
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
+                        final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
+                        request.timeout(newTimeout);
+                        executeHealth(request, listener);
+                    }
 
-                @Override
-                public void onNoLongerMaster(String source) {
-                    logger.trace("stopped being master while waiting for events with priority [{}]. retrying.", request.waitForEvents());
-                    doExecute(task, request, listener);
-                }
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+                        listener.onFailure(e);
+                    }
+                });
+            } else {
+                clusterService.submitStateUpdateTask("cluster_health (wait_for_events [" + request.waitForEvents() + "])", new ClusterStateUpdateTask(request.waitForEvents()) {
+                    @Override
+                    public ClusterState execute(ClusterState currentState) {
+                        return currentState;
+                    }
 
-                @Override
-                public void onFailure(String source, Exception e) {
-                    logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
-                    listener.onFailure(e);
-                }
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        final long timeoutInMillis = Math.max(0, endTimeMS - TimeValue.nsecToMSec(System.nanoTime()));
+                        final TimeValue newTimeout = TimeValue.timeValueMillis(timeoutInMillis);
+                        request.timeout(newTimeout);
+                        executeHealth(request, listener);
+                    }
 
-                @Override
-                public boolean runOnlyOnMaster() {
-                    return !request.local();
-                }
-            });
+                    @Override
+                    public void onNoLongerMaster(String source) {
+                        logger.trace("stopped being master while waiting for events with priority [{}]. retrying.", request.waitForEvents());
+                        doExecute(task, request, listener);
+                    }
+
+                    @Override
+                    public void onFailure(String source, Exception e) {
+                        logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
+                        listener.onFailure(e);
+                    }
+                });
+            }
         } else {
             executeHealth(request, listener);
         }
@@ -141,8 +160,8 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
         }
 
         assert waitFor >= 0;
-        final ClusterStateObserver observer = new ClusterStateObserver(clusterService, logger, threadPool.getThreadContext());
-        final ClusterState state = observer.observedState();
+        final ClusterState state = clusterService.state();
+        final ClusterStateObserver observer = new ClusterStateObserver(state, clusterService, null, logger, threadPool.getThreadContext());
         if (request.timeout().millis() == 0) {
             listener.onResponse(getResponse(request, state, waitFor, request.timeout().millis() == 0));
             return;
@@ -163,8 +182,7 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
 
             @Override
             public void onTimeout(TimeValue timeout) {
-                final ClusterState clusterState = clusterService.state();
-                final ClusterHealthResponse response = getResponse(request, clusterState, concreteWaitFor, true);
+                final ClusterHealthResponse response = getResponse(request, observer.setAndGetObservedState(), concreteWaitFor, true);
                 listener.onResponse(response);
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index c26554b25e0..7d80b84d5d2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index b4cef38d28d..e4034582f96 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index ce5d92753a8..50786d36766 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -19,21 +19,21 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -49,9 +49,7 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 
 /**
@@ -116,18 +114,16 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
     @Override
     protected synchronized void taskOperation(CancelTasksRequest request, CancellableTask cancellableTask,
             ActionListener<TaskInfo> listener) {
-        final BanLock banLock = new BanLock(nodes -> removeBanOnNodes(cancellableTask, nodes));
-        Set<String> childNodes = taskManager.cancel(cancellableTask, request.getReason(), banLock::onTaskFinished);
-        if (childNodes != null) {
-            if (childNodes.isEmpty()) {
-                // The task has no child tasks, so we can return immediately
-                logger.trace("cancelling task {} with no children", cancellableTask.getId());
-                listener.onResponse(cancellableTask.taskInfo(clusterService.localNode().getId(), false));
-            } else {
-                // The task has some child tasks, we need to wait for until ban is set on all nodes
-                logger.trace("cancelling task {} with children on nodes [{}]", cancellableTask.getId(), childNodes);
-                String nodeId = clusterService.localNode().getId();
-                AtomicInteger responses = new AtomicInteger(childNodes.size());
+        String nodeId = clusterService.localNode().getId();
+        final boolean canceled;
+        if (cancellableTask.shouldCancelChildrenOnCancellation()) {
+            DiscoveryNodes childNodes = clusterService.state().nodes();
+            final BanLock banLock = new BanLock(childNodes.getSize(), () -> removeBanOnNodes(cancellableTask, childNodes));
+            canceled = taskManager.cancel(cancellableTask, request.getReason(), banLock::onTaskFinished);
+            if (canceled) {
+                // /In case the task has some child tasks, we need to wait for until ban is set on all nodes
+                logger.trace("cancelling task {} on child nodes", cancellableTask.getId());
+                AtomicInteger responses = new AtomicInteger(childNodes.getSize());
                 List<Exception> failures = new ArrayList<>();
                 setBanOnNodes(request.getReason(), cancellableTask, childNodes, new ActionListener<Void>() {
                     @Override
@@ -157,83 +153,74 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
                         }
                     }
                 });
-
             }
-        } else {
+        }  else {
+            canceled = taskManager.cancel(cancellableTask, request.getReason(),
+                () -> listener.onResponse(cancellableTask.taskInfo(nodeId, false)));
+            if (canceled) {
+                logger.trace("task {} doesn't have any children that should be cancelled", cancellableTask.getId());
+            }
+        }
+        if (canceled == false) {
             logger.trace("task {} is already cancelled", cancellableTask.getId());
             throw new IllegalStateException("task with id " + cancellableTask.getId() + " is already cancelled");
         }
     }
 
+
     @Override
     protected boolean accumulateExceptions() {
         return true;
     }
 
-    private void setBanOnNodes(String reason, CancellableTask task, Set<String> nodes, ActionListener<Void> listener) {
+    private void setBanOnNodes(String reason, CancellableTask task, DiscoveryNodes nodes, ActionListener<Void> listener) {
         sendSetBanRequest(nodes,
             BanParentTaskRequest.createSetBanParentTaskRequest(new TaskId(clusterService.localNode().getId(), task.getId()), reason),
             listener);
     }
 
-    private void removeBanOnNodes(CancellableTask task, Set<String> nodes) {
+    private void removeBanOnNodes(CancellableTask task, DiscoveryNodes nodes) {
         sendRemoveBanRequest(nodes,
             BanParentTaskRequest.createRemoveBanParentTaskRequest(new TaskId(clusterService.localNode().getId(), task.getId())));
     }
 
-    private void sendSetBanRequest(Set<String> nodes, BanParentTaskRequest request, ActionListener<Void> listener) {
-        ClusterState clusterState = clusterService.state();
-        for (String node : nodes) {
-            DiscoveryNode discoveryNode = clusterState.getNodes().get(node);
-            if (discoveryNode != null) {
-                // Check if node still in the cluster
-                logger.trace("Sending ban for tasks with the parent [{}] to the node [{}], ban [{}]", request.parentTaskId, node,
-                    request.ban);
-                transportService.sendRequest(discoveryNode, BAN_PARENT_ACTION_NAME, request,
-                    new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
-                        @Override
-                        public void handleResponse(TransportResponse.Empty response) {
-                            listener.onResponse(null);
-                        }
+    private void sendSetBanRequest(DiscoveryNodes nodes, BanParentTaskRequest request, ActionListener<Void> listener) {
+        for (ObjectObjectCursor<String, DiscoveryNode> node : nodes.getNodes()) {
+            logger.trace("Sending ban for tasks with the parent [{}] to the node [{}], ban [{}]", request.parentTaskId, node.key,
+                request.ban);
+            transportService.sendRequest(node.value, BAN_PARENT_ACTION_NAME, request,
+                new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {
+                    @Override
+                    public void handleResponse(TransportResponse.Empty response) {
+                        listener.onResponse(null);
+                    }
 
-                        @Override
-                        public void handleException(TransportException exp) {
-                            logger.warn("Cannot send ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node);
-                            listener.onFailure(exp);
-                        }
-                    });
-            } else {
-                listener.onResponse(null);
-                logger.debug("Cannot send ban for tasks with the parent [{}] to the node [{}] - the node no longer in the cluster",
-                    request.parentTaskId, node);
-            }
+                    @Override
+                    public void handleException(TransportException exp) {
+                        logger.warn("Cannot send ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node.key);
+                        listener.onFailure(exp);
+                    }
+                });
         }
     }
 
-    private void sendRemoveBanRequest(Set<String> nodes, BanParentTaskRequest request) {
-        ClusterState clusterState = clusterService.state();
-        for (String node : nodes) {
-            DiscoveryNode discoveryNode = clusterState.getNodes().get(node);
-            if (discoveryNode != null) {
-                // Check if node still in the cluster
-                logger.debug("Sending remove ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node);
-                transportService.sendRequest(discoveryNode, BAN_PARENT_ACTION_NAME, request, EmptyTransportResponseHandler
-                    .INSTANCE_SAME);
-            } else {
-                logger.debug("Cannot send remove ban request for tasks with the parent [{}] to the node [{}] - the node no longer in " +
-                    "the cluster", request.parentTaskId, node);
-            }
+    private void sendRemoveBanRequest(DiscoveryNodes nodes, BanParentTaskRequest request) {
+        for (ObjectObjectCursor<String, DiscoveryNode> node : nodes.getNodes()) {
+            logger.debug("Sending remove ban for tasks with the parent [{}] to the node [{}]", request.parentTaskId, node.key);
+            transportService.sendRequest(node.value, BAN_PARENT_ACTION_NAME, request, EmptyTransportResponseHandler
+                .INSTANCE_SAME);
         }
     }
 
     private static class BanLock {
-        private final Consumer<Set<String>> finish;
+        private final Runnable finish;
         private final AtomicInteger counter;
-        private final AtomicReference<Set<String>> nodes = new AtomicReference<>();
+        private final int nodesSize;
 
-        public BanLock(Consumer<Set<String>> finish) {
+        BanLock(int nodesSize, Runnable finish) {
             counter = new AtomicInteger(0);
             this.finish = finish;
+            this.nodesSize = nodesSize;
         }
 
         public void onBanSet() {
@@ -242,15 +229,14 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
             }
         }
 
-        public void onTaskFinished(Set<String> nodes) {
-            this.nodes.set(nodes);
-            if (counter.addAndGet(nodes.size()) == 0) {
+        public void onTaskFinished() {
+            if (counter.addAndGet(nodesSize) == 0) {
                 finish();
             }
         }
 
         public void finish() {
-            finish.accept(nodes.get());
+            finish.run();
         }
 
     }
@@ -282,7 +268,7 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
             this.ban = false;
         }
 
-        public BanParentTaskRequest() {
+        BanParentTaskRequest() {
         }
 
         @Override
@@ -322,5 +308,4 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
         }
     }
 
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
index ffd4b358314..72f26d2d576 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.TaskResult;
 
@@ -34,7 +34,7 @@ import static java.util.Objects.requireNonNull;
 /**
  * Returns the list of tasks currently running on the nodes
  */
-public class GetTaskResponse extends ActionResponse implements ToXContent {
+public class GetTaskResponse extends ActionResponse implements ToXContentObject {
     private TaskResult task;
 
     public GetTaskResponse() {
@@ -65,7 +65,10 @@ public class GetTaskResponse extends ActionResponse implements ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return task.innerToXContent(builder, params);
+        builder.startObject();
+        task.innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index b73adbf8112..30d71a992fd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -31,17 +31,17 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskResult;
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportException;
@@ -67,14 +67,17 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
     private final ClusterService clusterService;
     private final TransportService transportService;
     private final Client client;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportGetTaskAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client) {
+            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client,
+            NamedXContentRegistry xContentRegistry) {
         super(settings, GetTaskAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetTaskRequest::new);
         this.clusterService = clusterService;
         this.transportService = transportService;
         this.client = client;
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -117,7 +120,6 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
             return;
         }
         GetTaskRequest nodeRequest = request.nodeRequest(clusterService.localNode().getId(), thisTask.getId());
-        taskManager.registerChildTask(thisTask, node.getId());
         transportService.sendRequest(node, GetTaskAction.NAME, nodeRequest, builder.build(),
                 new TransportResponseHandler<GetTaskResponse>() {
                     @Override
@@ -246,8 +248,8 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
             listener.onFailure(new ElasticsearchException("Stored task status for [{}] didn't contain any source!", response.getId()));
             return;
         }
-        try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
-            TaskResult result = TaskResult.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+        try (XContentParser parser = XContentHelper.createParser(xContentRegistry, response.getSourceAsBytesRef())) {
+            TaskResult result = TaskResult.PARSER.apply(parser, null);
             listener.onResponse(new GetTaskResponse(result));
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index b33226b973b..a203dd35b47 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -27,7 +27,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -43,7 +43,7 @@ import java.util.stream.Collectors;
 /**
  * Returns the list of tasks currently running on the nodes
  */
-public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
+public class ListTasksResponse extends BaseTasksResponse implements ToXContentObject {
 
     private List<TaskInfo> tasks;
 
@@ -161,8 +161,9 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
             }
             builder.startObject("tasks");
             for(TaskInfo task : entry.getValue()) {
-                builder.field(task.getTaskId().toString());
+                builder.startObject(task.getTaskId().toString());
                 task.toXContent(builder, params);
+                builder.endObject();
             }
             builder.endObject();
             builder.endObject();
@@ -187,7 +188,10 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        return toXContentGroupedByParents(builder, params);
+        builder.startObject();
+        toXContentGroupedByParents(builder, params);
+        builder.endObject();
+        return builder;
     }
 
     private void toXContentCommon(XContentBuilder builder, Params params) throws IOException {
@@ -214,6 +218,6 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
 
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
index b254137163d..87bf70acede 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
@@ -81,7 +81,7 @@ public class TaskGroup implements ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        task.innerToXContent(builder, params);
+        task.toXContent(builder, params);
         if (childTasks.isEmpty() == false) {
             builder.startArray("children");
             for (TaskGroup taskGroup : childTasks) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
index a06175a598b..f0f8d50b4c1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java
@@ -142,14 +142,28 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque
     /**
      * Sets the repository settings.
      *
-     * @param source repository settings in json, yaml or properties format
+     * @param source repository settings in json or yaml format
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type auto-detection
      */
+    @Deprecated
     public PutRepositoryRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the repository settings.
+     *
+     * @param source repository settings in json or yaml format
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public PutRepositoryRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the repository settings.
      *
@@ -160,7 +174,7 @@ public class PutRepositoryRequest extends AcknowledgedRequest<PutRepositoryReque
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
index 39cfa6af7f7..aed09daff2a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.repositories.put;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -89,16 +90,30 @@ public class PutRepositoryRequestBuilder extends AcknowledgedRequestBuilder<PutR
     }
 
     /**
-     * Sets the repository settings in Json, Yaml or properties format
+     * Sets the repository settings in Json or Yaml format
      *
      * @param source repository settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} instead to avoid content type auto detection
      */
+    @Deprecated
     public PutRepositoryRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets the repository settings in Json or Yaml format
+     *
+     * @param source repository settings
+     * @param xContentType the contenty type of the source
+     * @return this builder
+     */
+    public PutRepositoryRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the repository settings
      *
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
index 5de45fafa6d..27612a3dab2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
@@ -22,18 +22,18 @@ package org.elasticsearch.action.admin.cluster.repositories.verify;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 
 import java.io.IOException;
 
 /**
  * Unregister repository response
  */
-public class VerifyRepositoryResponse extends ActionResponse implements ToXContent {
+public class VerifyRepositoryResponse extends ActionResponse implements ToXContentObject {
 
     private DiscoveryNode[] nodes;
 
@@ -83,6 +83,7 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject(Fields.NODES);
         for (DiscoveryNode node : nodes) {
             builder.startObject(node.getId());
@@ -90,11 +91,12 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte
             builder.endObject();
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 
     @Override
     public String toString() {
-        return XContentHelper.toString(this);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
index fbb6a8d18e8..74bf8f341f3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java
@@ -59,7 +59,7 @@ public class ClusterRerouteResponse extends AcknowledgedResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        state = ClusterState.Builder.readFrom(in, null);
+        state = ClusterState.readFrom(in, null);
         readAcknowledged(in);
         explanations = RoutingExplanations.readFrom(in);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
index 16310b58cbc..bd0110e644c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java
@@ -51,7 +51,7 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (transientSettings.getAsMap().isEmpty() && persistentSettings.getAsMap().isEmpty()) {
+        if (transientSettings.isEmpty() && persistentSettings.isEmpty()) {
             validationException = addValidationError("no settings to update", validationException);
         }
         return validationException;
@@ -83,12 +83,22 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #transientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest transientSettings(String source) {
         this.transientSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequest transientSettings(String source, XContentType xContentType) {
+        this.transientSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -97,7 +107,7 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            transientSettings(builder.string());
+            transientSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -122,12 +132,22 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #persistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequest persistentSettings(String source) {
         this.persistentSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequest persistentSettings(String source, XContentType xContentType) {
+        this.persistentSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */
@@ -136,7 +156,7 @@ public class ClusterUpdateSettingsRequest extends AcknowledgedRequest<ClusterUpd
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            persistentSettings(builder.string());
+            persistentSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
index f0492edfeb1..906b1867b1f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.settings;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -52,12 +53,22 @@ public class ClusterUpdateSettingsRequestBuilder extends AcknowledgedRequestBuil
 
     /**
      * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     * @deprecated use {@link #setTransientSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings) {
         request.transientSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the transient settings to be updated. They will not survive a full cluster restart
+     */
+    public ClusterUpdateSettingsRequestBuilder setTransientSettings(String settings, XContentType xContentType) {
+        request.transientSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the transient settings to be updated. They will not survive a full cluster restart
      */
@@ -84,12 +95,22 @@ public class ClusterUpdateSettingsRequestBuilder extends AcknowledgedRequestBuil
 
     /**
      * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     * @deprecated use {@link #setPersistentSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings) {
         request.persistentSettings(settings);
         return this;
     }
 
+    /**
+     * Sets the source containing the persistent settings to be updated. They will get applied cross restarts
+     */
+    public ClusterUpdateSettingsRequestBuilder setPersistentSettings(String settings, XContentType xContentType) {
+        request.persistentSettings(settings, xContentType);
+        return this;
+    }
+
     /**
      * Sets the persistent settings to be updated. They will get applied cross restarts
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index ab6cdb94e1f..913dbfff20c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -67,10 +67,10 @@ public class TransportClusterUpdateSettingsAction extends
     @Override
     protected ClusterBlockException checkBlock(ClusterUpdateSettingsRequest request, ClusterState state) {
         // allow for dedicated changes to the metadata blocks, so we don't block those to allow to "re-enable" it
-        if ((request.transientSettings().getAsMap().isEmpty() &&
-            request.persistentSettings().getAsMap().size() == 1 &&
+        if ((request.transientSettings().isEmpty() &&
+            request.persistentSettings().size() == 1 &&
             MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())) ||
-            (request.persistentSettings().getAsMap().isEmpty() && request.transientSettings().getAsMap().size() == 1 &&
+            (request.persistentSettings().isEmpty() && request.transientSettings().size() == 1 &&
                 MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings()))) {
             return null;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
index 473d31754eb..79a014ebda7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
@@ -38,7 +38,7 @@ public class ClusterSearchShardsGroup implements Streamable, ToXContent {
 
     }
 
-    ClusterSearchShardsGroup(ShardId shardId, ShardRouting[] shards) {
+    public ClusterSearchShardsGroup(ShardId shardId, ShardRouting[] shards) {
         this.shardId = shardId;
         this.shards = shards;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
index 6f9a2ae55b1..3ee5f56ebdc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 
@@ -32,7 +32,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-public class ClusterSearchShardsResponse extends ActionResponse implements ToXContent {
+public class ClusterSearchShardsResponse extends ActionResponse implements ToXContentObject {
 
     private ClusterSearchShardsGroup[] groups;
     private DiscoveryNode[] nodes;
@@ -42,7 +42,8 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
 
     }
 
-    ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes, Map<String, AliasFilter> indicesAndFilters) {
+    public ClusterSearchShardsResponse(ClusterSearchShardsGroup[] groups, DiscoveryNode[] nodes,
+                                       Map<String, AliasFilter> indicesAndFilters) {
         this.groups = groups;
         this.nodes = nodes;
         this.indicesAndFilters = indicesAndFilters;
@@ -104,6 +105,7 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject("nodes");
         for (DiscoveryNode node : nodes) {
             node.toXContent(builder, params);
@@ -129,7 +131,7 @@ public class ClusterSearchShardsResponse extends ActionResponse implements ToXCo
             group.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
index 9cbc1b65632..3267b6d9c96 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequest.java
@@ -42,7 +42,7 @@ import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
  * Create snapshot request
@@ -288,18 +288,34 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
     }
 
     /**
-     * Sets repository-specific snapshot settings in JSON, YAML or properties format
+     * Sets repository-specific snapshot settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public CreateSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>
@@ -312,7 +328,7 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -366,14 +382,14 @@ public class CreateSnapshotRequest extends MasterNodeRequest<CreateSnapshotReque
                     throw new IllegalArgumentException("malformed indices section, should be an array of strings");
                 }
             } else if (name.equals("partial")) {
-                partial(lenientNodeBooleanValue(entry.getValue()));
+                partial(nodeBooleanValue(entry.getValue(), "partial"));
             } else if (name.equals("settings")) {
                 if (!(entry.getValue() instanceof Map)) {
                     throw new IllegalArgumentException("malformed settings section, should indices an inner object");
                 }
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
-                includeGlobalState = lenientNodeBooleanValue(entry.getValue());
+                includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             }
         }
         indicesOptions(IndicesOptions.fromMap((Map<String, Object>) source, IndicesOptions.lenientExpandOpen()));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
index ebdd206b5c3..d3b5e12351c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestBuilder.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -147,12 +148,28 @@ public class CreateSnapshotRequestBuilder extends MasterNodeOperationRequestBuil
      *
      * @param source repository-specific snapshot settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateSnapshotRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets repository-specific snapshot settings in YAML or JSON format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public CreateSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets repository-specific snapshot settings.
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index efc2fbeb5b5..1f9f77f9ed3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -33,7 +33,7 @@ import java.io.IOException;
 /**
  * Create snapshot response
  */
-public class CreateSnapshotResponse extends ActionResponse implements ToXContent {
+public class CreateSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
     private SnapshotInfo snapshotInfo;
@@ -83,12 +83,14 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         if (snapshotInfo != null) {
             builder.field("snapshot");
             snapshotInfo.toXContent(builder, params);
         } else {
             builder.field("accepted", true);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 3c37d1870e5..5cce5482ec5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -75,6 +75,6 @@ public class TransportDeleteSnapshotAction extends TransportMasterNodeAction<Del
             public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
-        });
+        }, false);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 924f5a90d42..308a846c905 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.snapshots.SnapshotInfo;
 
@@ -34,7 +35,7 @@ import java.util.List;
 /**
  * Get snapshots response
  */
-public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
+public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject {
 
     private List<SnapshotInfo> snapshots = Collections.emptyList();
 
@@ -76,11 +77,13 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         builder.startArray("snapshots");
         for (SnapshotInfo snapshotInfo : snapshots) {
             snapshotInfo.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 573bb0ea263..ad8541ce9fd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -88,7 +88,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
                 currentSnapshotIds.add(snapshotId);
             }
             if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
-                for (SnapshotId snapshotId : snapshotsService.snapshotIds(repository)) {
+                for (SnapshotId snapshotId : snapshotsService.getRepositoryData(repository).getAllSnapshotIds()) {
                     allSnapshotIds.put(snapshotId.getName(), snapshotId);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 641525f00e8..9d8ed49aaa0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -40,7 +40,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
  * Restore snapshot request
@@ -313,18 +313,34 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     }
 
     /**
-     * Sets repository-specific restore settings in JSON, YAML or properties format
+     * Sets repository-specific restore settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this request
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this request
+     */
+    public RestoreSnapshotRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets repository-specific restore settings
      * <p>
@@ -337,7 +353,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -436,12 +452,22 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
 
     /**
      * Sets settings that should be added/changed in all restored indices
+     * @deprecated use {@link #indexSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequest indexSettings(String source) {
         this.indexSettings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets settings that should be added/changed in all restored indices
+     */
+    public RestoreSnapshotRequest indexSettings(String source, XContentType xContentType) {
+        this.indexSettings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Sets settings that should be added/changed in all restored indices
      */
@@ -449,7 +475,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            indexSettings(builder.string());
+            indexSettings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -481,16 +507,16 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
                     throw new IllegalArgumentException("malformed indices section, should be an array of strings");
                 }
             } else if (name.equals("partial")) {
-                partial(lenientNodeBooleanValue(entry.getValue()));
+                partial(nodeBooleanValue(entry.getValue(), "partial"));
             } else if (name.equals("settings")) {
                 if (!(entry.getValue() instanceof Map)) {
                     throw new IllegalArgumentException("malformed settings section");
                 }
                 settings((Map<String, Object>) entry.getValue());
             } else if (name.equals("include_global_state")) {
-                includeGlobalState = lenientNodeBooleanValue(entry.getValue());
+                includeGlobalState = nodeBooleanValue(entry.getValue(), "include_global_state");
             } else if (name.equals("include_aliases")) {
-                includeAliases = lenientNodeBooleanValue(entry.getValue());
+                includeAliases = nodeBooleanValue(entry.getValue(), "include_aliases");
             } else if (name.equals("rename_pattern")) {
                 if (entry.getValue() instanceof String) {
                     renamePattern((String) entry.getValue());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
index 661a1a1d018..807e2387243 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestBuilder.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.List;
 import java.util.Map;
@@ -153,18 +154,34 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui
     }
 
     /**
-     * Sets repository-specific restore settings in JSON, YAML or properties format
+     * Sets repository-specific restore settings in JSON or YAML format
      * <p>
      * See repository documentation for more information.
      *
      * @param source repository-specific snapshot settings
      * @return this builder
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * Sets repository-specific restore settings in JSON or YAML format
+     * <p>
+     * See repository documentation for more information.
+     *
+     * @param source repository-specific snapshot settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets repository-specific restore settings
      * <p>
@@ -251,12 +268,26 @@ public class RestoreSnapshotRequestBuilder extends MasterNodeOperationRequestBui
      *
      * @param source index settings
      * @return this builder
+     * @deprecated use {@link #setIndexSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public RestoreSnapshotRequestBuilder setIndexSettings(String source) {
         request.indexSettings(source);
         return this;
     }
 
+    /**
+     * Sets index settings that should be added or replaced during restore
+     *
+     * @param source index settings
+     * @param xContentType the content type of the source
+     * @return this builder
+     */
+    public RestoreSnapshotRequestBuilder setIndexSettings(String source, XContentType xContentType) {
+        request.indexSettings(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets index settings that should be added or replaced during restore
      *
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
index 70f4f2aa4f2..5a02e4bcb13 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.RestoreInfo;
@@ -33,7 +34,7 @@ import java.io.IOException;
 /**
  * Contains information about restores snapshot
  */
-public class RestoreSnapshotResponse extends ActionResponse implements ToXContent {
+public class RestoreSnapshotResponse extends ActionResponse implements ToXContentObject {
 
     @Nullable
     private RestoreInfo restoreInfo;
@@ -75,12 +76,14 @@ public class RestoreSnapshotResponse extends ActionResponse implements ToXConten
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         if (restoreInfo != null) {
             builder.field("snapshot");
             restoreInfo.toXContent(builder, params);
         } else {
             builder.field("accepted", true);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
index d96daa86f76..c523fbbac3b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStage.java
@@ -47,7 +47,7 @@ public enum SnapshotIndexShardStage {
 
     private boolean completed;
 
-    private SnapshotIndexShardStage(byte value, boolean completed) {
+    SnapshotIndexShardStage(byte value, boolean completed) {
         this.value = value;
         this.completed = completed;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index b9800a2d9ed..d44a490680c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.status;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -33,7 +33,7 @@ import java.util.List;
 /**
  * Snapshot status response
  */
-public class SnapshotsStatusResponse extends ActionResponse implements ToXContent {
+public class SnapshotsStatusResponse extends ActionResponse implements ToXContentObject {
 
     private List<SnapshotStatus> snapshots = Collections.emptyList();
 
@@ -75,11 +75,13 @@ public class SnapshotsStatusResponse extends ActionResponse implements ToXConten
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray("snapshots");
         for (SnapshotStatus snapshot : snapshots) {
             snapshot.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index c73ae48d070..7406b0fea4a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -36,7 +36,9 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.repositories.RepositoryData;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
@@ -201,7 +203,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
         final String repositoryName = request.repository();
         if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
             final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
-            final Map<String, SnapshotId> matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream()
+            final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName);
+            final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream()
                 .filter(s -> requestedSnapshotNames.contains(s.getName()))
                 .collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
             for (final String snapshotName : request.snapshots()) {
@@ -220,6 +223,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                     } else {
                         throw new SnapshotMissingException(repositoryName, snapshotName);
                     }
+                } else if (repositoryData.getIncompatibleSnapshotIds().contains(snapshotId)) {
+                    throw new SnapshotException(repositoryName, snapshotName, "cannot get the status for an incompatible snapshot");
                 }
                 SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
                 List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
@@ -243,7 +248,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                         default:
                             throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                     }
-                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotInfo.snapshotId()), state, Collections.unmodifiableList(shardStatusBuilder)));
+                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
index f4d8b891b86..6d6f0da34b5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
@@ -52,7 +52,7 @@ public class ClusterStateResponse extends ActionResponse {
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         clusterName = new ClusterName(in);
-        clusterState = ClusterState.Builder.readFrom(in, null);
+        clusterState = ClusterState.readFrom(in, null);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 45eb83dd9e1..d77bc599258 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
index d01c128cf36..c30eae12a82 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java
@@ -31,66 +31,79 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 public class DeleteStoredScriptRequest extends AcknowledgedRequest<DeleteStoredScriptRequest> {
 
     private String id;
-    private String scriptLang;
+    private String lang;
 
     DeleteStoredScriptRequest() {
+        super();
     }
 
-    public DeleteStoredScriptRequest(String scriptLang, String id) {
-        this.scriptLang = scriptLang;
+    public DeleteStoredScriptRequest(String id, String lang) {
+        super();
+
         this.id = id;
+        this.lang = lang;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (id == null) {
-            validationException = addValidationError("id is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
         } else if (id.contains("#")) {
-            validationException = addValidationError("id can't contain: '#'", validationException);
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (scriptLang == null) {
-            validationException = addValidationError("lang is missing", validationException);
-        } else if (scriptLang.contains("#")) {
-            validationException = addValidationError("lang can't contain: '#'", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public String scriptLang() {
-        return scriptLang;
-    }
-
-    public DeleteStoredScriptRequest scriptLang(String type) {
-        this.scriptLang = type;
-        return this;
-    }
-
     public String id() {
         return id;
     }
 
     public DeleteStoredScriptRequest id(String id) {
         this.id = id;
+
+        return this;
+    }
+
+    public String lang() {
+        return lang;
+    }
+
+    public DeleteStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
         return this;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        scriptLang = in.readString();
+
+        lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(scriptLang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeString(id);
     }
 
     @Override
     public String toString() {
-        return "delete script {[" + scriptLang + "][" + id + "]}";
+        return "delete stored script {id [" + id + "]" + (lang != null ? ", lang [" + lang + "]" : "") + "}";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
index caf55a03f18..8a65506dabd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java
@@ -29,13 +29,15 @@ public class DeleteStoredScriptRequestBuilder extends AcknowledgedRequestBuilder
         super(client, action, new DeleteStoredScriptRequest());
     }
 
-    public DeleteStoredScriptRequestBuilder setScriptLang(String scriptLang) {
-        request.scriptLang(scriptLang);
+    public DeleteStoredScriptRequestBuilder setLang(String lang) {
+        request.lang(lang);
+
         return this;
     }
 
     public DeleteStoredScriptRequestBuilder setId(String id) {
         request.id(id);
+
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
index bb7a9effd32..2bfd547362c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequest.java
@@ -28,61 +28,79 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
 public class GetStoredScriptRequest extends MasterNodeReadRequest<GetStoredScriptRequest> {
 
     protected String id;
     protected String lang;
 
     GetStoredScriptRequest() {
+        super();
     }
 
-    public GetStoredScriptRequest(String lang, String id) {
-        this.lang = lang;
+    public GetStoredScriptRequest(String id, String lang) {
+        super();
+
         this.id = id;
+        this.lang = lang;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (lang == null) {
-            validationException = ValidateActions.addValidationError("lang is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
+        } else if (id.contains("#")) {
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (id == null) {
-            validationException = ValidateActions.addValidationError("id is missing", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public GetStoredScriptRequest lang(@Nullable String type) {
-        this.lang = type;
-        return this;
-    }
-
-    public GetStoredScriptRequest id(String id) {
-        this.id = id;
-        return this;
-    }
-
-
-    public String lang() {
-        return lang;
-    }
-
     public String id() {
         return id;
     }
 
+    public GetStoredScriptRequest id(String id) {
+        this.id = id;
+
+        return this;
+    }
+
+    public String lang() {
+        return lang;
+    }
+
+    public GetStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
+        return this;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
+
         lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(lang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeString(id);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
index 36dd9beb38a..b8302a03c28 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java
@@ -19,49 +19,70 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.script.Script;
+import org.elasticsearch.script.StoredScriptSource;
 
 import java.io.IOException;
 
 public class GetStoredScriptResponse extends ActionResponse implements ToXContent {
 
-    private String storedScript;
+    private StoredScriptSource source;
 
     GetStoredScriptResponse() {
     }
 
-    GetStoredScriptResponse(String storedScript) {
-        this.storedScript = storedScript;
+    GetStoredScriptResponse(StoredScriptSource source) {
+        this.source = source;
     }
 
     /**
      * @return if a stored script and if not found <code>null</code>
      */
-    public String getStoredScript() {
-        return storedScript;
+    public StoredScriptSource getSource() {
+        return source;
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.value(storedScript);
+        source.toXContent(builder, params);
+
         return builder;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        storedScript = in.readOptionalString();
+
+        if (in.readBoolean()) {
+            if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+                source = new StoredScriptSource(in);
+            } else {
+                source = new StoredScriptSource(in.readString());
+            }
+        } else {
+            source = null;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeOptionalString(storedScript);
+
+        if (source == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+
+            if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+                source.writeTo(out);
+            } else {
+                out.writeString(source.getCode());
+            }
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
index cfe153d7d96..28c70dc45b8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java
@@ -19,108 +19,153 @@
 
 package org.elasticsearch.action.admin.cluster.storedscripts;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 public class PutStoredScriptRequest extends AcknowledgedRequest<PutStoredScriptRequest> {
 
     private String id;
-    private String scriptLang;
-    private BytesReference script;
+    private String lang;
+    private BytesReference content;
+    private XContentType xContentType;
 
     public PutStoredScriptRequest() {
         super();
     }
 
-    public PutStoredScriptRequest(String scriptLang) {
-        super();
-        this.scriptLang = scriptLang;
+    @Deprecated
+    public PutStoredScriptRequest(String id, String lang, BytesReference content) {
+        this(id, lang, content, XContentFactory.xContentType(content));
     }
 
-    public PutStoredScriptRequest(String scriptLang, String id) {
+    public PutStoredScriptRequest(String id, String lang, BytesReference content, XContentType xContentType) {
         super();
-        this.scriptLang = scriptLang;
         this.id = id;
+        this.lang = lang;
+        this.content = content;
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (id == null) {
-            validationException = addValidationError("id is missing", validationException);
+
+        if (id == null || id.isEmpty()) {
+            validationException = addValidationError("must specify id for stored script", validationException);
         } else if (id.contains("#")) {
-            validationException = addValidationError("id can't contain: '#'", validationException);
+            validationException = addValidationError("id cannot contain '#' for stored script", validationException);
         }
-        if (scriptLang == null) {
-            validationException = addValidationError("lang is missing", validationException);
-        } else if (scriptLang.contains("#")) {
-            validationException = addValidationError("lang can't contain: '#'", validationException);
+
+        if (lang != null && lang.contains("#")) {
+            validationException = addValidationError("lang cannot contain '#' for stored script", validationException);
         }
-        if (script == null) {
-            validationException = addValidationError("script is missing", validationException);
+
+        if (content == null) {
+            validationException = addValidationError("must specify code for stored script", validationException);
         }
+
         return validationException;
     }
 
-    public String scriptLang() {
-        return scriptLang;
-    }
-
-    public PutStoredScriptRequest scriptLang(String scriptLang) {
-        this.scriptLang = scriptLang;
-        return this;
-    }
-
     public String id() {
         return id;
     }
 
     public PutStoredScriptRequest id(String id) {
         this.id = id;
+
         return this;
     }
 
-    public BytesReference script() {
-        return script;
+    public String lang() {
+        return lang;
     }
 
-    public PutStoredScriptRequest script(BytesReference source) {
-        this.script = source;
+    public PutStoredScriptRequest lang(String lang) {
+        this.lang = lang;
+
+        return this;
+    }
+
+    public BytesReference content() {
+        return content;
+    }
+
+    public XContentType xContentType() {
+        return xContentType;
+    }
+
+    /**
+     * Set the script source using bytes.
+     * @deprecated this method is deprecated as it relies on content type detection. Use {@link #content(BytesReference, XContentType)}
+     */
+    @Deprecated
+    public PutStoredScriptRequest content(BytesReference content) {
+        return content(content, XContentFactory.xContentType(content));
+    }
+
+    /**
+     * Set the script source and the content type of the bytes.
+     */
+    public PutStoredScriptRequest content(BytesReference content, XContentType xContentType) {
+        this.content = content;
+        this.xContentType = Objects.requireNonNull(xContentType);
         return this;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        scriptLang = in.readString();
+
+        lang = in.readString();
+
+        if (lang.isEmpty()) {
+            lang = null;
+        }
+
         id = in.readOptionalString();
-        script = in.readBytesReference();
+        content = in.readBytesReference();
+        if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(content);
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(scriptLang);
+
+        out.writeString(lang == null ? "" : lang);
         out.writeOptionalString(id);
-        out.writeBytesReference(script);
+        out.writeBytesReference(content);
+        if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType.writeTo(out);
+        }
     }
 
     @Override
     public String toString() {
-        String sSource = "_na_";
+        String source = "_na_";
+
         try {
-            sSource = XContentHelper.convertToJson(script, false);
+            source = XContentHelper.convertToJson(content, false, xContentType);
         } catch (Exception e) {
             // ignore
         }
-        return "put script {[" + id + "][" + scriptLang + "], script[" + sSource + "]}";
+
+        return "put stored script {id [" + id + "]" + (lang != null ? ", lang [" + lang + "]" : "") + ", content [" + source + "]}";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
index 15c51c2ccd7..f8223d69199 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.cluster.storedscripts;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<PutStoredScriptRequest,
         PutStoredScriptResponse, PutStoredScriptRequestBuilder> {
@@ -30,19 +31,31 @@ public class PutStoredScriptRequestBuilder extends AcknowledgedRequestBuilder<Pu
         super(client, action, new PutStoredScriptRequest());
     }
 
-    public PutStoredScriptRequestBuilder setScriptLang(String scriptLang) {
-        request.scriptLang(scriptLang);
-        return this;
-    }
-
     public PutStoredScriptRequestBuilder setId(String id) {
         request.id(id);
         return this;
     }
 
-    public PutStoredScriptRequestBuilder setSource(BytesReference source) {
-        request.script(source);
+    /**
+     * Set the source of the script.
+     * @deprecated this method requires content type detection. Use {@link #setContent(BytesReference, XContentType)} instead
+     */
+    @Deprecated
+    public PutStoredScriptRequestBuilder setContent(BytesReference content) {
+        request.content(content);
         return this;
     }
 
+    /**
+     * Set the source of the script along with the content type of the source
+     */
+    public PutStoredScriptRequestBuilder setContent(BytesReference source, XContentType xContentType) {
+        request.content(source, xContentType);
+        return this;
+    }
+
+    public PutStoredScriptRequestBuilder setLang(String lang) {
+        request.lang(lang);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
index a91dec8d9b3..8b4079aee73 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java
@@ -59,7 +59,7 @@ public class TransportPutStoredScriptAction extends TransportMasterNodeAction<Pu
     @Override
     protected void masterOperation(PutStoredScriptRequest request, ClusterState state,
                                    ActionListener<PutStoredScriptResponse> listener) throws Exception {
-        scriptService.storeScript(clusterService, request, listener);
+        scriptService.putStoredScript(clusterService, request, listener);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
index bb1afe5e19e..ec42e34ec96 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/tasks/PendingClusterTasksResponse.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -31,7 +31,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-public class PendingClusterTasksResponse extends ActionResponse implements Iterable<PendingClusterTask>, ToXContent {
+public class PendingClusterTasksResponse extends ActionResponse implements Iterable<PendingClusterTask>, ToXContentObject {
 
     private List<PendingClusterTask> pendingTasks;
 
@@ -63,13 +63,15 @@ public class PendingClusterTasksResponse extends ActionResponse implements Itera
         StringBuilder sb = new StringBuilder();
         sb.append("tasks: (").append(pendingTasks.size()).append("):\n");
         for (PendingClusterTask pendingClusterTask : this) {
-            sb.append(pendingClusterTask.getInsertOrder()).append("/").append(pendingClusterTask.getPriority()).append("/").append(pendingClusterTask.getSource()).append("/").append(pendingClusterTask.getTimeInQueue()).append("\n");
+            sb.append(pendingClusterTask.getInsertOrder()).append("/").append(pendingClusterTask.getPriority()).append("/")
+                    .append(pendingClusterTask.getSource()).append("/").append(pendingClusterTask.getTimeInQueue()).append("\n");
         }
         return sb.toString();
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.TASKS);
         for (PendingClusterTask pendingClusterTask : this) {
             builder.startObject();
@@ -82,6 +84,7 @@ public class PendingClusterTasksResponse extends ActionResponse implements Itera
             builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
index 524a21ec632..e97facd748f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java
@@ -29,7 +29,6 @@ import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -42,6 +41,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 
@@ -118,8 +118,8 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
         public static AliasActions removeIndex() {
             return new AliasActions(AliasActions.Type.REMOVE_INDEX);
         }
-        private static ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser(String name, Supplier<AliasActions> supplier) {
-            ObjectParser<AliasActions, ParseFieldMatcherSupplier> parser = new ObjectParser<>(name, supplier);
+        private static ObjectParser<AliasActions, Void> parser(String name, Supplier<AliasActions> supplier) {
+            ObjectParser<AliasActions, Void> parser = new ObjectParser<>(name, supplier);
             parser.declareString((action, index) -> {
                 if (action.indices() != null) {
                     throw new IllegalArgumentException("Only one of [index] and [indices] is supported");
@@ -147,7 +147,7 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
             return parser;
         }
 
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> ADD_PARSER = parser("add", AliasActions::add);
+        private static final ObjectParser<AliasActions, Void> ADD_PARSER = parser("add", AliasActions::add);
         static {
             ADD_PARSER.declareObject(AliasActions::filter, (parser, m) -> {
                 try {
@@ -157,18 +157,17 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
                 }
             }, new ParseField("filter"));
             // Since we need to support numbers AND strings here we have to use ValueType.INT.
-            ADD_PARSER.declareField(AliasActions::routing, p -> p.text(), new ParseField("routing"), ValueType.INT);
-            ADD_PARSER.declareField(AliasActions::indexRouting, p -> p.text(), new ParseField("index_routing"), ValueType.INT);
-            ADD_PARSER.declareField(AliasActions::searchRouting, p -> p.text(), new ParseField("search_routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::routing, XContentParser::text, new ParseField("routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::indexRouting, XContentParser::text, new ParseField("index_routing"), ValueType.INT);
+            ADD_PARSER.declareField(AliasActions::searchRouting, XContentParser::text, new ParseField("search_routing"), ValueType.INT);
         }
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> REMOVE_PARSER = parser("remove", AliasActions::remove);
-        private static final ObjectParser<AliasActions, ParseFieldMatcherSupplier> REMOVE_INDEX_PARSER = parser("remove_index",
-                AliasActions::removeIndex);
+        private static final ObjectParser<AliasActions, Void> REMOVE_PARSER = parser("remove", AliasActions::remove);
+        private static final ObjectParser<AliasActions, Void> REMOVE_INDEX_PARSER = parser("remove_index", AliasActions::removeIndex);
 
         /**
          * Parser for any one {@link AliasAction}.
          */
-        public static final ConstructingObjectParser<AliasActions, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+        public static final ConstructingObjectParser<AliasActions, Void> PARSER = new ConstructingObjectParser<>(
                 "alias_action", a -> {
                     // Take the first action and complain if there are more than one actions
                     AliasActions action = null;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
index 85282788898..5a63ce8d869 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java
@@ -57,7 +57,7 @@ public class GetAliasesResponse extends ActionResponse {
             int valueSize = in.readVInt();
             List<AliasMetaData> value = new ArrayList<>(valueSize);
             for (int j = 0; j < valueSize; j++) {
-                value.add(AliasMetaData.Builder.readFrom(in));
+                value.add(new AliasMetaData(in));
             }
             aliasesBuilder.put(key, Collections.unmodifiableList(value));
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
index 6d0824eeb31..08f220e0199 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequest.java
@@ -75,7 +75,7 @@ public class AnalyzeRequest extends SingleShardRequest<AnalyzeRequest> {
             try {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
                 builder.map(definition);
-                this.definition = Settings.builder().loadFromSource(builder.string()).build();
+                this.definition = Settings.builder().loadFromSource(builder.string(), builder.contentType()).build();
             } catch (IOException e) {
                 throw new IllegalArgumentException("Failed to parse [" + definition + "]", e);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
index 344681b997e..5070862ed69 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeRequestBuilder.java
@@ -113,7 +113,7 @@ public class AnalyzeRequestBuilder extends SingleShardOperationRequestBuilder<An
     /**
      * Sets attributes that will include results
      */
-    public AnalyzeRequestBuilder setAttributes(String attributes){
+    public AnalyzeRequestBuilder setAttributes(String... attributes){
         request.attributes(attributes);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
index 302597e0e09..e4c6b34d9c7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -32,25 +32,27 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeResponse.AnalyzeToken>, ToXContent {
+public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeResponse.AnalyzeToken>, ToXContentObject {
 
-    public static class AnalyzeToken implements Streamable, ToXContent {
+    public static class AnalyzeToken implements Streamable, ToXContentObject {
         private String term;
         private int startOffset;
         private int endOffset;
         private int position;
+        private int positionLength = 1;
         private Map<String, Object> attributes;
         private String type;
 
         AnalyzeToken() {
         }
 
-        public AnalyzeToken(String term, int position, int startOffset, int endOffset, String type,
-                            Map<String, Object> attributes) {
+        public AnalyzeToken(String term, int position, int startOffset, int endOffset, int positionLength,
+                            String type, Map<String, Object> attributes) {
             this.term = term;
             this.position = position;
             this.startOffset = startOffset;
             this.endOffset = endOffset;
+            this.positionLength = positionLength;
             this.type = type;
             this.attributes = attributes;
         }
@@ -71,6 +73,10 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
             return this.position;
         }
 
+        public int getPositionLength() {
+            return this.positionLength;
+        }
+
         public String getType() {
             return this.type;
         }
@@ -87,6 +93,9 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
             builder.field(Fields.END_OFFSET, endOffset);
             builder.field(Fields.TYPE, type);
             builder.field(Fields.POSITION, position);
+            if (positionLength > 1) {
+                builder.field(Fields.POSITION_LENGTH, positionLength);
+            }
             if (attributes != null && !attributes.isEmpty()) {
                 for (Map.Entry<String, Object> entity : attributes.entrySet()) {
                     builder.field(entity.getKey(), entity.getValue());
@@ -108,6 +117,14 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
             startOffset = in.readInt();
             endOffset = in.readInt();
             position = in.readVInt();
+            if (in.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+                Integer len = in.readOptionalVInt();
+                if (len != null) {
+                    positionLength = len;
+                } else {
+                    positionLength = 1;
+                }
+            }
             type = in.readOptionalString();
             if (in.getVersion().onOrAfter(Version.V_2_2_0)) {
                 attributes = (Map<String, Object>) in.readGenericValue();
@@ -120,6 +137,9 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
             out.writeInt(startOffset);
             out.writeInt(endOffset);
             out.writeVInt(position);
+            if (out.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+                out.writeOptionalVInt(positionLength > 1 ? positionLength : null);
+            }
             out.writeOptionalString(type);
             if (out.getVersion().onOrAfter(Version.V_2_2_0)) {
                 out.writeGenericValue(attributes);
@@ -154,6 +174,7 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         if (tokens != null) {
             builder.startArray(Fields.TOKENS);
             for (AnalyzeToken token : tokens) {
@@ -167,6 +188,7 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
             detail.toXContent(builder, params);
             builder.endObject();
         }
+        builder.endObject();
         return builder;
     }
 
@@ -206,6 +228,7 @@ public class AnalyzeResponse extends ActionResponse implements Iterable<AnalyzeR
         static final String END_OFFSET = "end_offset";
         static final String TYPE = "type";
         static final String POSITION = "position";
+        static final String POSITION_LENGTH = "positionLength";
         static final String DETAIL = "detail";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
index 7d7e9d2dd2e..43207f68dee 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
@@ -218,13 +219,15 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                 PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class);
                 OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class);
                 TypeAttribute type = stream.addAttribute(TypeAttribute.class);
+                PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class);
 
                 while (stream.incrementToken()) {
                     int increment = posIncr.getPositionIncrement();
                     if (increment > 0) {
                         lastPosition = lastPosition + increment;
                     }
-                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), type.type(), null));
+                    tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
+                        lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
 
                 }
                 stream.end();
@@ -381,6 +384,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                 PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class);
                 OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class);
                 TypeAttribute type = stream.addAttribute(TypeAttribute.class);
+                PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class);
 
                 while (stream.incrementToken()) {
                     int increment = posIncr.getPositionIncrement();
@@ -388,7 +392,7 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeRe
                         lastPosition = lastPosition + increment;
                     }
                     tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(),
-                        lastOffset + offset.endOffset(), type.type(), extractExtendedAttributes(stream, includeAttributes)));
+                        lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes)));
 
                 }
                 stream.end();
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index 17df06dbf4b..798c08bd180 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
@@ -35,6 +36,7 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -42,10 +44,11 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.io.UncheckedIOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -168,19 +171,29 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
     }
 
     /**
-     * The settings to create the index with (either json/yaml/properties format)
+     * The settings to create the index with (either json or yaml format)
+     * @deprecated use {@link #source(String, XContentType)} instead to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * The settings to create the index with (either json or yaml format)
+     */
+    public CreateIndexRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Allows to set the settings using a json builder.
      */
     public CreateIndexRequest settings(XContentBuilder builder) {
         try {
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate json settings from builder", e);
         }
@@ -195,7 +208,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -207,13 +220,42 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #mapping(String, String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequest mapping(String type, String source) {
+        return mapping(type, new BytesArray(source), XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The content type of the source
+     */
+    public CreateIndexRequest mapping(String type, String source, XContentType xContentType) {
+        return mapping(type, new BytesArray(source), xContentType);
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType the content type of the mapping source
+     */
+    private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) {
         if (mappings.containsKey(type)) {
             throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
         }
-        mappings.put(type, source);
-        return this;
+        Objects.requireNonNull(xContentType);
+        try {
+            mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType));
+            return this;
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to convert to json", e);
+        }
     }
 
     /**
@@ -231,15 +273,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
      * @param source The mapping source
      */
     public CreateIndexRequest mapping(String type, XContentBuilder source) {
-        if (mappings.containsKey(type)) {
-            throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
-        }
-        try {
-            mappings.put(type, source.string());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to build json for mapping request", e);
-        }
-        return this;
+        return mapping(type, source.bytes(), source.contentType());
     }
 
     /**
@@ -260,7 +294,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            return mapping(type, builder.string());
+            return mapping(type, builder);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -307,7 +341,8 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
      * Sets the aliases that will be associated with the index when it gets created
      */
     public CreateIndexRequest aliases(BytesReference source) {
-        try (XContentParser parser = XContentHelper.createParser(source)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, source)) {
             //move to the first alias
             parser.nextToken();
             while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -329,9 +364,18 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public CreateIndexRequest source(String source) {
-        return source(source.getBytes(StandardCharsets.UTF_8));
+        return source(new BytesArray(source));
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequest source(String source, XContentType xContentType) {
+        return source(new BytesArray(source), xContentType);
     }
 
     /**
@@ -343,7 +387,9 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(byte[], XContentType)}
      */
+    @Deprecated
     public CreateIndexRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
@@ -351,6 +397,15 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequest source(byte[] source, int offset, int length) {
         return source(new BytesArray(source, offset, length));
     }
@@ -358,17 +413,27 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequest source(BytesReference source) {
         XContentType xContentType = XContentFactory.xContentType(source);
-        if (xContentType != null) {
-            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                source(parser.map());
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("failed to parse source for create index", e);
-            }
-        } else {
-            settings(source.utf8ToString());
-        }
+        source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequest source(BytesReference source, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
+        source(XContentHelper.convertToMap(source, false, xContentType).v2());
         return this;
     }
 
@@ -485,7 +550,13 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
         readTimeout(in);
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            mappings.put(in.readString(), in.readString());
+            final String type = in.readString();
+            String source = in.readString();
+            if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to 5.3.0 after backport
+                // we do not know the content type that comes from earlier versions so we autodetect and convert
+                source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
+            }
+            mappings.put(type, source);
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
index eaae4d53b73..237c88244b4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -76,13 +77,23 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
     }
 
     /**
-     * The settings to create the index with (either json/yaml/properties format)
+     * The settings to create the index with (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public CreateIndexRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
+    /**
+     * The settings to create the index with (either json or yaml format)
+     */
+    public CreateIndexRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
     /**
      * A simplified version of settings that takes key value pairs settings.
      */
@@ -104,12 +115,26 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #addMapping(String, String, XContentType)} to avoid content type auto-detection
      */
+    @Deprecated
     public CreateIndexRequestBuilder addMapping(String type, String source) {
         request.mapping(type, source);
         return this;
     }
 
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The content type of the source
+     */
+    public CreateIndexRequestBuilder addMapping(String type, String source, XContentType xContentType) {
+        request.mapping(type, source, xContentType);
+        return this;
+    }
+
     /**
      * The cause for this index creation.
      */
@@ -191,7 +216,9 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
 
     /**
      * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(String source) {
         request.source(source);
         return this;
@@ -200,6 +227,16 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(String source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(BytesReference source) {
         request.source(source);
         return this;
@@ -208,6 +245,16 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(BytesReference source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(byte[] source) {
         request.source(source);
         return this;
@@ -216,11 +263,29 @@ public class CreateIndexRequestBuilder extends AcknowledgedRequestBuilder<Create
     /**
      * Sets the settings and mappings as a single source.
      */
+    public CreateIndexRequestBuilder setSource(byte[] source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings and mappings as a single source.
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length) {
         request.source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the settings and mappings as a single source.
+     */
+    public CreateIndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) {
+        request.source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the settings and mappings as a single source.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java
index 83eaf11ca3a..ac32b16eb57 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java
@@ -58,6 +58,6 @@ public class ShardFlushRequest extends ReplicationRequest<ShardFlushRequest> {
 
     @Override
     public String toString() {
-        return "flush {" + super.toString() + "}";
+        return "flush {" + shardId + "}";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 1ec7186393f..026946334ac 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -65,17 +65,7 @@ public class TransportShardFlushAction extends TransportReplicationAction<ShardF
     }
 
     @Override
-    protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected boolean shouldExecuteReplication(Settings settings) {
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
         return true;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
index 0dd81075b83..48886c38aa4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
@@ -34,7 +34,7 @@ import java.util.List;
  */
 public class GetIndexRequest extends ClusterInfoRequest<GetIndexRequest> {
 
-    public static enum Feature {
+    public enum Feature {
         ALIASES((byte) 0, "_aliases", "_alias"),
         MAPPINGS((byte) 1, "_mappings", "_mapping"),
         SETTINGS((byte) 2, "_settings");
@@ -52,7 +52,7 @@ public class GetIndexRequest extends ClusterInfoRequest<GetIndexRequest> {
         private final String preferredName;
         private final byte id;
 
-        private Feature(byte id, String... validNames) {
+        Feature(byte id, String... validNames) {
             assert validNames != null && validNames.length > 0;
             this.id = id;
             this.validNames = Arrays.asList(validNames);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
index 3a29237faeb..6c2e4627523 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
@@ -104,7 +104,7 @@ public class GetIndexResponse extends ActionResponse {
             int valueSize = in.readVInt();
             ImmutableOpenMap.Builder<String, MappingMetaData> mappingEntryBuilder = ImmutableOpenMap.builder();
             for (int j = 0; j < valueSize; j++) {
-                mappingEntryBuilder.put(in.readString(), MappingMetaData.PROTO.readFrom(in));
+                mappingEntryBuilder.put(in.readString(), new MappingMetaData(in));
             }
             mappingsMapBuilder.put(key, mappingEntryBuilder.build());
         }
@@ -116,7 +116,7 @@ public class GetIndexResponse extends ActionResponse {
             int valueSize = in.readVInt();
             List<AliasMetaData> aliasEntryBuilder = new ArrayList<>();
             for (int j = 0; j < valueSize; j++) {
-                aliasEntryBuilder.add(AliasMetaData.Builder.readFrom(in));
+                aliasEntryBuilder.add(new AliasMetaData(in));
             }
             aliasesMapBuilder.put(key, Collections.unmodifiableList(aliasEntryBuilder));
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index e0cedcf841e..3f4ddaf08db 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.Mapper;
 
 import java.io.IOException;
@@ -108,20 +109,25 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte
 
         /** Returns the mappings as a map. Note that the returned map has a single key which is always the field's {@link Mapper#name}. */
         public Map<String, Object> sourceAsMap() {
-            return XContentHelper.convertToMap(source, true).v2();
+            return XContentHelper.convertToMap(source, true, XContentType.JSON).v2();
         }
 
         public boolean isNull() {
             return NULL.fullName().equals(fullName) && NULL.source.length() == source.length();
         }
 
+        //pkg-private for testing
+        BytesReference getSource() {
+            return source;
+        }
+
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.field("full_name", fullName);
             if (params.paramAsBoolean("pretty", false)) {
                 builder.field("mapping", sourceAsMap());
             } else {
-                builder.rawField("mapping", source);
+                builder.rawField("mapping", source, XContentType.JSON);
             }
             return builder;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
index e092f1f148d..12975c765d0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java
@@ -57,7 +57,7 @@ public class GetMappingsResponse extends ActionResponse {
             int valueSize = in.readVInt();
             ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
             for (int j = 0; j < valueSize; j++) {
-                typeMapBuilder.put(in.readString(), MappingMetaData.PROTO.readFrom(in));
+                typeMapBuilder.put(in.readString(), new MappingMetaData(in));
             }
             indexMapBuilder.put(key, typeMapBuilder.build());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index 864c6703c48..92c23bb8568 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -50,12 +50,10 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonMap;
-import static org.elasticsearch.common.util.CollectionUtils.newLinkedList;
 
 /**
  * Transport action used to retrieve the mappings related to fields that belong to a specific index
@@ -174,24 +172,12 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
                     addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
                 }
             } else if (Regex.isSimpleMatchPattern(field)) {
-                // go through the field mappers 3 times, to make sure we give preference to the resolve order: full name, index name, name.
-                // also make sure we only store each mapper once.
-                Collection<FieldMapper> remainingFieldMappers = newLinkedList(allFieldMappers);
-                for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
-                    final FieldMapper fieldMapper = it.next();
+                for (FieldMapper fieldMapper : allFieldMappers) {
                     if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
-                        it.remove();
+                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings,
+                            request.includeDefaults());
                     }
                 }
-                for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
-                    final FieldMapper fieldMapper = it.next();
-                    if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
-                        it.remove();
-                    }
-                }
-
             } else {
                 // not a pattern
                 FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
@@ -220,4 +206,4 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
         }
     }
 
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
index 152bc516549..400701f91ca 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequest.java
@@ -22,19 +22,24 @@ package org.elasticsearch.action.admin.indices.mapping.put;
 import com.carrotsearch.hppc.ObjectHashSet;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Objects;
@@ -245,7 +250,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
      */
     public PutMappingRequest source(XContentBuilder mappingBuilder) {
         try {
-            return source(mappingBuilder.string());
+            return source(mappingBuilder.string(), mappingBuilder.contentType());
         } catch (IOException e) {
             throw new IllegalArgumentException("Failed to build json for mapping request", e);
         }
@@ -259,7 +264,7 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(mappingSource);
-            return source(builder.string());
+            return source(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + mappingSource + "]", e);
         }
@@ -267,10 +272,31 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
 
     /**
      * The mapping source definition.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public PutMappingRequest source(String mappingSource) {
-        this.source = mappingSource;
-        return this;
+        return source(mappingSource, XContentFactory.xContentType(mappingSource));
+    }
+
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequest source(String mappingSource, XContentType xContentType) {
+        return source(new BytesArray(mappingSource), xContentType);
+    }
+
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequest source(BytesReference mappingSource, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
+        try {
+            this.source = XContentHelper.convertToJson(mappingSource, false, false, xContentType);
+            return this;
+        } catch (IOException e) {
+            throw new UncheckedIOException("failed to convert source to json", e);
+        }
     }
 
     /** True if all fields that span multiple types should be updated, false otherwise */
@@ -291,6 +317,10 @@ public class PutMappingRequest extends AcknowledgedRequest<PutMappingRequest> im
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         type = in.readOptionalString();
         source = in.readString();
+        if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3 once backported
+            // we do not know the format from earlier versions so convert if necessary
+            source = XContentHelper.convertToJson(new BytesArray(source), false, false, XContentFactory.xContentType(source));
+        }
         updateAllTypes = in.readBoolean();
         readTimeout(in);
         concreteIndex = in.readOptionalWriteable(Index::new);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
index c21c40cf041..012a593ebc4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 
 import java.util.Map;
@@ -82,12 +83,22 @@ public class PutMappingRequestBuilder extends AcknowledgedRequestBuilder<PutMapp
 
     /**
      * The mapping source definition.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public PutMappingRequestBuilder setSource(String mappingSource) {
         request.source(mappingSource);
         return this;
     }
 
+    /**
+     * The mapping source definition.
+     */
+    public PutMappingRequestBuilder setSource(String mappingSource, XContentType xContentType) {
+        request.source(mappingSource, xContentType);
+        return this;
+    }
+
     /**
      * A specialized simplified mapping source method, takes the form of simple properties definition:
      * ("field1", "type=string,store=true").
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index d4b8d3532e9..d1d8b4078b6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.support.replication.BasicReplicationRequest;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -68,17 +68,7 @@ public class TransportShardRefreshAction
     }
 
     @Override
-    protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.METADATA_WRITE;
-    }
-
-    @Override
-    protected boolean shouldExecuteReplication(Settings settings) {
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
         return true;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index 8d9b48f2000..d6bfaf0a48c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.indices.rollover;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -32,8 +31,7 @@ import java.util.Set;
  */
 public abstract class Condition<T> implements NamedWriteable {
 
-    public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("conditions", null);
+    public static ObjectParser<Set<Condition>, Void> PARSER = new ObjectParser<>("conditions", null);
     static {
         PARSER.declareString((conditions, s) ->
             conditions.add(new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))),
@@ -49,7 +47,7 @@ public abstract class Condition<T> implements NamedWriteable {
         this.name = name;
     }
 
-    public abstract Result evaluate(final Stats stats);
+    public abstract Result evaluate(Stats stats);
 
     @Override
     public final String toString() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index ddd58705bea..4804bc577fc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
@@ -44,23 +43,19 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
-    public static final ObjectParser<RolloverRequest, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("conditions", null);
+    public static final ObjectParser<RolloverRequest, Void> PARSER = new ObjectParser<>("conditions", null);
     static {
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                Condition.PARSER.parse(parser, request.conditions, parseFieldMatcherSupplier),
+        PARSER.declareField((parser, request, context) -> Condition.PARSER.parse(parser, request.conditions, null),
             new ParseField("conditions"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.createIndexRequest.settings(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()),
             new ParseField("settings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) -> {
+        PARSER.declareField((parser, request, context) -> {
             for (Map.Entry<String, Object> mappingsEntry : parser.map().entrySet()) {
                 request.createIndexRequest.mapping(mappingsEntry.getKey(),
                     (Map<String, Object>) mappingsEntry.getValue());
             }
         }, new ParseField("mappings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.createIndexRequest.aliases(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()),
             new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index b495e3c6a0f..8c1be3501a8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -32,7 +32,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-public final class RolloverResponse extends ActionResponse implements ToXContent {
+public final class RolloverResponse extends ActionResponse implements ToXContentObject {
 
     private static final String NEW_INDEX = "new_index";
     private static final String OLD_INDEX = "old_index";
@@ -157,6 +157,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.field(OLD_INDEX, oldIndex);
         builder.field(NEW_INDEX, newIndex);
         builder.field(ROLLED_OVER, rolledOver);
@@ -168,6 +169,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
             builder.field(entry.getKey(), entry.getValue());
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index 67099b4d100..8a8d1063f3e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -62,7 +62,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
         if (globalBlock != null) {
             return globalBlock;
         }
-        if (request.settings().getAsMap().size() == 1 && IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings()) || IndexMetaData.INDEX_READ_ONLY_SETTING.exists(request.settings())) {
+        if (request.settings().size() == 1 && IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.exists(request.settings()) || IndexMetaData.INDEX_READ_ONLY_SETTING.exists(request.settings())) {
             return null;
         }
         return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
index 494b9df7bd3..80f3fb1a071 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequest.java
@@ -70,7 +70,7 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
-        if (settings.getAsMap().isEmpty()) {
+        if (settings.isEmpty()) {
             validationException = addValidationError("no settings to update", validationException);
         }
         return validationException;
@@ -121,13 +121,23 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     * @deprecated use {@link #settings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public UpdateSettingsRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
+    /**
+     * Sets the settings to be updated (either json or yaml format)
+     */
+    public UpdateSettingsRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
     /**
      * Returns <code>true</code> iff the settings update should only add but not update settings. If the setting already exists
      * it should not be overwritten by this update. The default is <code>false</code>
@@ -146,14 +156,14 @@ public class UpdateSettingsRequest extends AcknowledgedRequest<UpdateSettingsReq
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
      */
     @SuppressWarnings("unchecked")
     public UpdateSettingsRequest settings(Map source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), builder.contentType());
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
index 36dfbf3b2d4..a9cecbfc5a4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Map;
 
@@ -70,15 +71,25 @@ public class UpdateSettingsRequestBuilder extends AcknowledgedRequestBuilder<Upd
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)} to avoid content type detection
      */
+    @Deprecated
     public UpdateSettingsRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
     /**
-     * Sets the settings to be updated (either json/yaml/properties format)
+     * Sets the settings to be updated (either json or yaml format)
+     */
+    public UpdateSettingsRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the settings to be updated
      */
     public UpdateSettingsRequestBuilder setSettings(Map<String, Object> source) {
         request.settings(source);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index f1df6d53e18..3657e327265 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -207,7 +207,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
             builder.field(Fields.ALLOCATED, allocationStatus.value());
             if (storeException != null) {
                 builder.startObject(Fields.STORE_EXCEPTION);
-                ElasticsearchException.toXContent(builder, params, storeException);
+                ElasticsearchException.generateThrowableXContent(builder, params, storeException);
                 builder.endObject();
             }
             return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index e13578d66de..bce20df34e3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -226,7 +226,7 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
                 private final List<NodeGatewayStartedShards> responses;
                 private final List<FailedNodeException> failures;
 
-                public Response(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
+                Response(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
                     this.shardId = shardId;
                     this.responses = responses;
                     this.failures = failures;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
index 40a11402501..faa0a63c54d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -40,14 +39,11 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements IndicesRequest {
 
-    public static final ObjectParser<ShrinkRequest, ParseFieldMatcherSupplier> PARSER =
-        new ObjectParser<>("shrink_request", null);
+    public static final ObjectParser<ShrinkRequest, Void> PARSER = new ObjectParser<>("shrink_request", null);
     static {
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.getShrinkIndexRequest().settings(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().settings(parser.map()),
             new ParseField("settings"), ObjectParser.ValueType.OBJECT);
-        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-                request.getShrinkIndexRequest().aliases(parser.map()),
+        PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().aliases(parser.map()),
             new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
index 6d27b03db63..8c482eac10c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
@@ -131,6 +131,9 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
             }
 
         }
+        if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) {
+            throw new IllegalArgumentException("cannot provide a routing partition size value when shrinking an index");
+        }
         targetIndex.cause("shrink_index");
         Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);
         settingsBuilder.put("index.number_of_shards", numShards);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
index 877db0579a0..150b7c6a52b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -103,7 +104,9 @@ public class ShardStats implements Streamable, Writeable, ToXContent {
         statePath = in.readString();
         dataPath = in.readString();
         isCustomDataPath = in.readBoolean();
-        seqNoStats = in.readOptionalWriteable(SeqNoStats::new);
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            seqNoStats = in.readOptionalWriteable(SeqNoStats::new);
+        }
     }
 
     @Override
@@ -114,7 +117,9 @@ public class ShardStats implements Streamable, Writeable, ToXContent {
         out.writeString(statePath);
         out.writeString(dataPath);
         out.writeBoolean(isCustomDataPath);
-        out.writeOptionalWriteable(seqNoStats);
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            out.writeOptionalWriteable(seqNoStats);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
index a519bd8bf55..3c5fb36d6c6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -31,7 +32,7 @@ import java.util.List;
 
 import static java.util.Collections.singletonMap;
 
-public class GetIndexTemplatesResponse extends ActionResponse implements ToXContent {
+public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {
 
     private List<IndexTemplateMetaData> indexTemplates;
 
@@ -52,7 +53,7 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont
         int size = in.readVInt();
         indexTemplates = new ArrayList<>(size);
         for (int i = 0 ; i < size ; i++) {
-            indexTemplates.add(0, IndexTemplateMetaData.Builder.readFrom(in));
+            indexTemplates.add(0, IndexTemplateMetaData.readFrom(in));
         }
     }
 
@@ -68,10 +69,11 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params);
-
+        builder.startObject();
         for (IndexTemplateMetaData indexTemplateMetaData : getIndexTemplates()) {
             IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
index 77c3a1d4c29..b0c13540dfa 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -44,11 +45,13 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -178,21 +181,31 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
     }
 
     /**
-     * The settings to create the index template with (either json/yaml/properties format).
+     * The settings to create the index template with (either json/yaml format).
+     * @deprecated use {@link #settings(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest settings(String source) {
         this.settings = Settings.builder().loadFromSource(source).build();
         return this;
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format).
+     * The settings to create the index template with (either json/yaml format).
+     */
+    public PutIndexTemplateRequest settings(String source, XContentType xContentType) {
+        this.settings = Settings.builder().loadFromSource(source, xContentType).build();
+        return this;
+    }
+
+    /**
+     * The settings to create the index template with (either json or yaml format).
      */
     public PutIndexTemplateRequest settings(Map<String, Object> source) {
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            settings(builder.string());
+            settings(builder.string(), XContentType.JSON);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -208,10 +221,23 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #mapping(String, String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest mapping(String type, String source) {
-        mappings.put(type, source);
-        return this;
+        XContentType xContentType = XContentFactory.xContentType(source);
+        return mapping(type, source, xContentType);
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The type of content contained within the source
+     */
+    public PutIndexTemplateRequest mapping(String type, String source, XContentType xContentType) {
+        return mapping(type, new BytesArray(source), xContentType);
     }
 
     /**
@@ -233,12 +259,24 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
      * @param source The mapping source
      */
     public PutIndexTemplateRequest mapping(String type, XContentBuilder source) {
+        return mapping(type, source.bytes(), source.contentType());
+    }
+
+    /**
+     * Adds mapping that will be added when the index gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType the source content type
+     */
+    public PutIndexTemplateRequest mapping(String type, BytesReference source, XContentType xContentType) {
+        Objects.requireNonNull(xContentType);
         try {
-            mappings.put(type, source.string());
+            mappings.put(type, XContentHelper.convertToJson(source, false, false, xContentType));
+            return this;
         } catch (IOException e) {
-            throw new IllegalArgumentException("Failed to build json for mapping request", e);
+            throw new UncheckedIOException("failed to convert source to json", e);
         }
-        return this;
     }
 
     /**
@@ -255,7 +293,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
         try {
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.map(source);
-            return mapping(type, builder.string());
+            return mapping(type, builder);
         } catch (IOException e) {
             throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e);
         }
@@ -279,7 +317,7 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
      */
     public PutIndexTemplateRequest source(XContentBuilder templateBuilder) {
         try {
-            return source(templateBuilder.bytes());
+            return source(templateBuilder.bytes(), templateBuilder.contentType());
         } catch (Exception e) {
             throw new IllegalArgumentException("Failed to build json for template request", e);
         }
@@ -349,18 +387,25 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     /**
      * The template source definition.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequest source(String templateSource) {
-        try (XContentParser parser = XContentFactory.xContent(templateSource).createParser(templateSource)) {
-            return source(parser.mapOrdered());
-        } catch (Exception e) {
-            throw new IllegalArgumentException("failed to parse template source [" + templateSource + "]", e);
-        }
+        return source(XContentHelper.convertToMap(XContentFactory.xContent(templateSource), templateSource, true));
     }
 
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) {
+        return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true));
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(byte[], XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
@@ -368,23 +413,40 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(byte[] source, int offset, int length) {
-        try (XContentParser parser = XContentFactory.xContent(source, offset, length).createParser(source, offset, length)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
+        return source(new BytesArray(source, offset, length));
     }
 
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequest source(BytesReference source) {
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            return source(parser.mapOrdered());
-        } catch (IOException e) {
-            throw new IllegalArgumentException("failed to parse template source", e);
-        }
+        return source(XContentHelper.convertToMap(source, true).v2());
+    }
+
+    /**
+     * The template source definition.
+     */
+    public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) {
+        return source(XContentHelper.convertToMap(source, true, xContentType).v2());
     }
 
     public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) {
@@ -432,7 +494,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
      * Sets the aliases that will be associated with the index when it gets created
      */
     public PutIndexTemplateRequest aliases(BytesReference source) {
-        try (XContentParser parser = XContentHelper.createParser(source)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, source)) {
             //move to the first alias
             parser.nextToken();
             while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -481,7 +544,14 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
         settings = readSettingsFromStream(in);
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            mappings.put(in.readString(), in.readString());
+            final String type = in.readString();
+            String mappingSource = in.readString();
+            if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) { // TODO change to V_5_3_0 once backported
+                // we do not know the incoming type so convert it if needed
+                mappingSource =
+                    XContentHelper.convertToJson(new BytesArray(mappingSource), false, false, XContentFactory.xContentType(mappingSource));
+            }
+            mappings.put(type, mappingSource);
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
index c1db96ae7ce..e7ca25e5119 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.util.Collections;
 import java.util.List;
@@ -100,15 +101,25 @@ public class PutIndexTemplateRequestBuilder
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format)
+     * The settings to crete the index template with (either json or yaml format)
+     * @deprecated use {@link #setSettings(String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSettings(String source) {
         request.settings(source);
         return this;
     }
 
     /**
-     * The settings to crete the index template with (either json/yaml/properties format)
+     * The settings to crete the index template with (either json or yaml format)
+     */
+    public PutIndexTemplateRequestBuilder setSettings(String source, XContentType xContentType) {
+        request.settings(source, xContentType);
+        return this;
+    }
+
+    /**
+     * The settings to crete the index template with (either json or yaml format)
      */
     public PutIndexTemplateRequestBuilder setSettings(Map<String, Object> source) {
         request.settings(source);
@@ -120,12 +131,26 @@ public class PutIndexTemplateRequestBuilder
      *
      * @param type   The mapping type
      * @param source The mapping source
+     * @deprecated use {@link #addMapping(String, String, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder addMapping(String type, String source) {
         request.mapping(type, source);
         return this;
     }
 
+    /**
+     * Adds mapping that will be added when the index template gets created.
+     *
+     * @param type   The mapping type
+     * @param source The mapping source
+     * @param xContentType The type/format of the source
+     */
+    public PutIndexTemplateRequestBuilder addMapping(String type, String source, XContentType xContentType) {
+        request.mapping(type, source, xContentType);
+        return this;
+    }
+
     /**
      * A specialized simplified mapping source method, takes the form of simple properties definition:
      * ("field1", "type=string,store=true").
@@ -226,7 +251,9 @@ public class PutIndexTemplateRequestBuilder
 
     /**
      * The template source definition.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(String templateSource) {
         request.source(templateSource);
         return this;
@@ -235,6 +262,16 @@ public class PutIndexTemplateRequestBuilder
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource, XContentType xContentType) {
+        request.source(templateSource, xContentType);
+        return this;
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(BytesReference templateSource) {
         request.source(templateSource);
         return this;
@@ -242,7 +279,9 @@ public class PutIndexTemplateRequestBuilder
 
     /**
      * The template source definition.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
      */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(byte[] templateSource) {
         request.source(templateSource);
         return this;
@@ -251,8 +290,26 @@ public class PutIndexTemplateRequestBuilder
     /**
      * The template source definition.
      */
+    public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, XContentType xContentType) {
+        request.source(templateSource, xContentType);
+        return this;
+    }
+
+    /**
+     * The template source definition.
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length) {
         request.source(templateSource, offset, length);
         return this;
     }
+
+    /**
+     * The template source definition.
+     */
+    public PutIndexTemplateRequestBuilder setSource(byte[] templateSource, int offset, int length, XContentType xContentType) {
+        request.source(templateSource, offset, length, xContentType);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
index bc8f8c347ab..81084e22377 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
@@ -171,7 +171,7 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
 
         private final int numberOfElements;
 
-        public ConstantBackoff(TimeValue delay, int numberOfElements) {
+        ConstantBackoff(TimeValue delay, int numberOfElements) {
             assert numberOfElements >= 0;
             this.delay = delay;
             this.numberOfElements = numberOfElements;
@@ -188,7 +188,7 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
         private final int numberOfElements;
         private int curr;
 
-        public ConstantBackoffIterator(TimeValue delay, int numberOfElements) {
+        ConstantBackoffIterator(TimeValue delay, int numberOfElements) {
             this.delay = delay;
             this.numberOfElements = numberOfElements;
         }
@@ -212,7 +212,7 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
         private final BackoffPolicy delegate;
         private final Runnable onBackoff;
 
-        public WrappedBackoffPolicy(BackoffPolicy delegate, Runnable onBackoff) {
+        WrappedBackoffPolicy(BackoffPolicy delegate, Runnable onBackoff) {
             this.delegate = delegate;
             this.onBackoff = onBackoff;
         }
@@ -227,7 +227,7 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
         private final Iterator<TimeValue> delegate;
         private final Runnable onBackoff;
 
-        public WrappedBackoffIterator(Iterator<TimeValue> delegate, Runnable onBackoff) {
+        WrappedBackoffIterator(Iterator<TimeValue> delegate, Runnable onBackoff) {
             this.delegate = delegate;
             this.onBackoff = onBackoff;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index e1a6e48e9e0..6b8d34098aa 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -22,8 +22,8 @@ package org.elasticsearch.action.bulk;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.DocWriteRequest.OpType;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.update.UpdateResponse;
@@ -32,7 +32,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
@@ -43,7 +43,7 @@ import java.io.IOException;
  * Represents a single item response for an action executed as part of the bulk API. Holds the index/type/id
  * of the relevant action, and if it has failed or not (with the failure message incase it failed).
  */
-public class BulkItemResponse implements Streamable, StatusToXContent {
+public class BulkItemResponse implements Streamable, StatusToXContentObject {
 
     @Override
     public RestStatus status() {
@@ -52,9 +52,10 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startObject(opType.getLowercase());
         if (failure == null) {
-            response.toXContent(builder, params);
+            response.innerToXContent(builder, params);
             builder.field(Fields.STATUS, response.status().getStatus());
         } else {
             builder.field(Fields._INDEX, failure.getIndex());
@@ -62,10 +63,11 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
             builder.field(Fields._ID, failure.getId());
             builder.field(Fields.STATUS, failure.getStatus().getStatus());
             builder.startObject(Fields.ERROR);
-            ElasticsearchException.toXContent(builder, params, failure.getCause());
+            ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause());
             builder.endObject();
         }
         builder.endObject();
+        builder.endObject();
         return builder;
     }
 
@@ -90,10 +92,10 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
         private final String index;
         private final String type;
         private final String id;
-        private final Throwable cause;
+        private final Exception cause;
         private final RestStatus status;
 
-        public Failure(String index, String type, String id, Throwable cause) {
+        public Failure(String index, String type, String id, Exception cause) {
             this.index = index;
             this.type = type;
             this.id = id;
@@ -159,7 +161,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
         /**
          * The actual cause of the failure.
          */
-        public Throwable getCause() {
+        public Exception getCause() {
             return cause;
         }
 
@@ -171,7 +173,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
                 builder.field(ID_FIELD, id);
             }
             builder.startObject(CAUSE_FIELD);
-            ElasticsearchException.toXContent(builder, params, cause);
+            ElasticsearchException.generateThrowableXContent(builder, params, cause);
             builder.endObject();
             builder.field(STATUS_FIELD, status.getStatus());
             return builder;
@@ -179,7 +181,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
 
         @Override
         public String toString() {
-            return Strings.toString(this, true);
+            return Strings.toString(this);
         }
     }
 
@@ -302,7 +304,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         id = in.readVInt();
-        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
             opType = OpType.fromId(in.readByte());
         } else {
             opType = OpType.fromString(in.readString());
@@ -315,6 +317,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
         } else if (type == 1) {
             response = new DeleteResponse();
             response.readFrom(in);
+
         } else if (type == 3) { // make 3 instead of 2, because 2 is already in use for 'no responses'
             response = new UpdateResponse();
             response.readFrom(in);
@@ -328,7 +331,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(id);
-        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
             out.writeByte(opType.getId());
         } else {
             out.writeString(opType.getLowercase());
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 5c3e8d9295e..371659586f9 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -39,9 +39,11 @@ import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
@@ -243,7 +245,9 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
 
     /**
      * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public BulkRequest add(byte[] data, int from, int length) throws IOException {
         return add(data, from, length, null, null);
     }
@@ -251,6 +255,15 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
     /**
      * Adds a framed data in binary format
      */
+    public BulkRequest add(byte[] data, int from, int length, XContentType xContentType) throws IOException {
+        return add(data, from, length, null, null, xContentType);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)}
+     */
+    @Deprecated
     public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException {
         return add(new BytesArray(data, from, length), defaultIndex, defaultType);
     }
@@ -258,6 +271,17 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
     /**
      * Adds a framed data in binary format
      */
+    public BulkRequest add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                           XContentType xContentType) throws IOException {
+        return add(new BytesArray(data, from, length), defaultIndex, defaultType, xContentType);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     *
+     * @deprecated use {@link #add(BytesReference, String, String, XContentType)}
+     */
+    @Deprecated
     public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType) throws IOException {
         return add(data, defaultIndex, defaultType, null, null, null, null, null, true);
     }
@@ -265,12 +289,40 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
     /**
      * Adds a framed data in binary format
      */
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType,
+                           XContentType xContentType) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, true, xContentType);
+    }
+
+    /**
+     * Adds a framed data in binary format
+     *
+     * @deprecated use {@link #add(BytesReference, String, String, boolean, XContentType)}
+     */
+    @Deprecated
     public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex) throws IOException {
         return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex);
     }
 
+    /**
+     * Adds a framed data in binary format
+     */
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, boolean allowExplicitIndex,
+                           XContentType xContentType) throws IOException {
+        return add(data, defaultIndex, defaultType, null, null, null, null, null, allowExplicitIndex, xContentType);
+    }
+
+    @Deprecated
     public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex) throws IOException {
-        XContent xContent = XContentFactory.xContent(data);
+        XContentType xContentType = XContentFactory.xContentType(data);
+        return add(data, defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, payload,
+            allowExplicitIndex, xContentType);
+    }
+
+    public BulkRequest add(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultType, @Nullable String
+        defaultRouting, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String
+        defaultPipeline, @Nullable Object payload, boolean allowExplicitIndex, XContentType xContentType) throws IOException {
+        XContent xContent = xContentType.xContent();
         int line = 0;
         int from = 0;
         int length = data.length();
@@ -283,7 +335,8 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
             line++;
 
             // now parse the action
-            try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
+            // EMPTY is safe here because we never call namedObject
+            try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, data.slice(from, nextMarker - from))) {
                 // move pointers
                 from = nextMarker + 1;
 
@@ -347,7 +400,7 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
                             } else if ("fields".equals(currentFieldName)) {
                                 throw new IllegalArgumentException("Action/metadata line [" + line + "] contains a simple value for parameter [fields] while a list is expected");
                             } else if ("_source".equals(currentFieldName)) {
-                                fetchSourceContext = FetchSourceContext.parse(parser);
+                                fetchSourceContext = FetchSourceContext.fromXContent(parser);
                             } else {
                                 throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]");
                             }
@@ -360,7 +413,7 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
                                 throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                             }
                         } else if (token == XContentParser.Token.START_OBJECT && "_source".equals(currentFieldName)) {
-                            fetchSourceContext = FetchSourceContext.parse(parser);
+                            fetchSourceContext = FetchSourceContext.fromXContent(parser);
                         } else if (token != XContentParser.Token.VALUE_NULL) {
                             throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                         }
@@ -385,22 +438,23 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
                     if ("index".equals(action)) {
                         if (opType == null) {
                             internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
-                                    .setPipeline(pipeline).source(data.slice(from, nextMarker - from)), payload);
+                                    .setPipeline(pipeline).source(data.slice(from, nextMarker - from), xContentType), payload);
                         } else {
                             internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
                                     .create("create".equals(opType)).setPipeline(pipeline)
-                                    .source(data.slice(from, nextMarker - from)), payload);
+                                    .source(data.slice(from, nextMarker - from), xContentType), payload);
                         }
                     } else if ("create".equals(action)) {
                         internalAdd(new IndexRequest(index, type, id).routing(routing).parent(parent).version(version).versionType(versionType)
                                 .create(true).setPipeline(pipeline)
-                                .source(data.slice(from, nextMarker - from)), payload);
+                                .source(data.slice(from, nextMarker - from), xContentType), payload);
                     } else if ("update".equals(action)) {
                         UpdateRequest updateRequest = new UpdateRequest(index, type, id).routing(routing).parent(parent).retryOnConflict(retryOnConflict)
                                 .version(version).versionType(versionType)
                                 .routing(routing)
                                 .parent(parent);
-                        try (XContentParser sliceParser = xContent.createParser(data.slice(from, nextMarker - from))) {
+                        // EMPTY is safe here because we never call namedObject
+                        try (XContentParser sliceParser = xContent.createParser(NamedXContentRegistry.EMPTY, data.slice(from, nextMarker - from))) {
                             updateRequest.fromXContent(sliceParser);
                         }
                         if (fetchSourceContext != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
index c48a8f507b8..8f634fa28a4 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
@@ -32,6 +32,7 @@ import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 
 /**
  * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes
@@ -97,7 +98,9 @@ public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkRe
 
     /**
      * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public BulkRequestBuilder add(byte[] data, int from, int length) throws Exception {
         request.add(data, from, length, null, null);
         return this;
@@ -106,11 +109,30 @@ public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkRe
     /**
      * Adds a framed data in binary format
      */
+    public BulkRequestBuilder add(byte[] data, int from, int length, XContentType xContentType) throws Exception {
+        request.add(data, from, length, null, null, xContentType);
+        return this;
+    }
+
+    /**
+     * Adds a framed data in binary format
+     * @deprecated use {@link #add(byte[], int, int, String, String, XContentType)}
+     */
+    @Deprecated
     public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception {
         request.add(data, from, length, defaultIndex, defaultType);
         return this;
     }
 
+    /**
+     * Adds a framed data in binary format
+     */
+    public BulkRequestBuilder add(byte[] data, int from, int length, @Nullable String defaultIndex, @Nullable String defaultType,
+                                  XContentType xContentType) throws Exception {
+        request.add(data, from, length, defaultIndex, defaultType, xContentType);
+        return this;
+    }
+
     /**
      * Sets the number of shard copies that must be active before proceeding with the write.
      * See {@link ReplicationRequest#waitForActiveShards(ActiveShardCount)} for details.
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
index 6ad566ca500..5d9910d9179 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
@@ -59,7 +59,7 @@ abstract class BulkRequestHandler {
         private final BulkProcessor.Listener listener;
         private final BackoffPolicy backoffPolicy;
 
-        public SyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
+        SyncBulkRequestHandler(Client client, BackoffPolicy backoffPolicy, BulkProcessor.Listener listener) {
             super(client);
             this.backoffPolicy = backoffPolicy;
             this.listener = listener;
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
index a41bd454979..c746894b78e 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
@@ -102,7 +102,7 @@ public class Retry {
         private volatile BulkRequest currentBulkRequest;
         private volatile ScheduledFuture<?> scheduledRequestFuture;
 
-        public AbstractRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
+        AbstractRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
             this.retryOnThrowable = retryOnThrowable;
             this.backoff = backoffPolicy.iterator();
             this.client = client;
@@ -213,7 +213,7 @@ public class Retry {
     }
 
     static class AsyncRetryHandler extends AbstractRetryHandler {
-        public AsyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
+        AsyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, ActionListener<BulkResponse> listener) {
             super(retryOnThrowable, backoffPolicy, client, listener);
         }
     }
@@ -226,7 +226,7 @@ public class Retry {
             return new SyncRetryHandler(retryOnThrowable, backoffPolicy, client, actionFuture);
         }
 
-        public SyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, PlainActionFuture<BulkResponse> actionFuture) {
+        SyncRetryHandler(Class<? extends Throwable> retryOnThrowable, BackoffPolicy backoffPolicy, Client client, PlainActionFuture<BulkResponse> actionFuture) {
             super(retryOnThrowable, backoffPolicy, client, actionFuture);
             this.actionFuture = actionFuture;
         }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 27a579db276..e7c2018ad31 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -31,8 +31,6 @@ import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.IngestActionForwarder;
 import org.elasticsearch.action.support.ActionFilters;
@@ -41,6 +39,8 @@ import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateObserver;
+import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -49,11 +49,15 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndexClosedException;
+import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -153,11 +157,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                         @Override
                         public void onResponse(CreateIndexResponse result) {
                             if (counter.decrementAndGet() == 0) {
-                                try {
-                                    executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Exception e) {
-                                    listener.onFailure(e);
-                                }
+                                executeBulk(task, bulkRequest, startTime, listener, responses);
                             }
                         }
 
@@ -173,12 +173,10 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                                 }
                             }
                             if (counter.decrementAndGet() == 0) {
-                                try {
-                                    executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Exception inner) {
+                                executeBulk(task, bulkRequest, startTime, ActionListener.wrap(listener::onResponse, inner -> {
                                     inner.addSuppressed(e);
                                     listener.onFailure(inner);
-                                }
+                                }), responses);
                             }
                         }
                     });
@@ -209,134 +207,200 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
         return false;
     }
 
-    /**
-     * This method executes the {@link BulkRequest} and calls the given listener once the request returns.
-     * This method will not create any indices even if auto-create indices is enabled.
-     *
-     * @see #doExecute(BulkRequest, org.elasticsearch.action.ActionListener)
-     */
-    public void executeBulk(final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
-        final long startTimeNanos = relativeTime();
-        executeBulk(null, bulkRequest, startTimeNanos, listener, new AtomicArray<>(bulkRequest.requests.size()));
-    }
-
     private long buildTookInMillis(long startTimeNanos) {
         return TimeUnit.NANOSECONDS.toMillis(relativeTime() - startTimeNanos);
     }
 
-    void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener, final AtomicArray<BulkItemResponse> responses ) {
-        final ClusterState clusterState = clusterService.state();
-        // TODO use timeout to wait here if its blocked...
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
+    /**
+     * retries on retryable cluster blocks, resolves item requests,
+     * constructs shard bulk requests and delegates execution to shard bulk action
+     * */
+    private final class BulkOperation extends AbstractRunnable {
+        private final Task task;
+        private final BulkRequest bulkRequest;
+        private final ActionListener<BulkResponse> listener;
+        private final AtomicArray<BulkItemResponse> responses;
+        private final long startTimeNanos;
+        private final ClusterStateObserver observer;
 
-        final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver);
-        MetaData metaData = clusterState.metaData();
-        for (int i = 0; i < bulkRequest.requests.size(); i++) {
-            DocWriteRequest docWriteRequest = bulkRequest.requests.get(i);
-            //the request can only be null because we set it to null in the previous step, so it gets ignored
-            if (docWriteRequest == null) {
-                continue;
+        BulkOperation(Task task, BulkRequest bulkRequest, ActionListener<BulkResponse> listener,
+                      AtomicArray<BulkItemResponse> responses, long startTimeNanos) {
+            this.task = task;
+            this.bulkRequest = bulkRequest;
+            this.listener = listener;
+            this.responses = responses;
+            this.startTimeNanos = startTimeNanos;
+            this.observer = new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext());
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            listener.onFailure(e);
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            final ClusterState clusterState = observer.setAndGetObservedState();
+            if (handleBlockExceptions(clusterState)) {
+                return;
             }
-            if (addFailureIfIndexIsUnavailable(docWriteRequest, bulkRequest, responses, i, concreteIndices, metaData)) {
-                continue;
-            }
-            Index concreteIndex = concreteIndices.resolveIfAbsent(docWriteRequest);
-            try {
-                switch (docWriteRequest.opType()) {
-                    case CREATE:
-                    case INDEX:
-                        IndexRequest indexRequest = (IndexRequest) docWriteRequest;
-                        MappingMetaData mappingMd = null;
-                        final IndexMetaData indexMetaData = metaData.index(concreteIndex);
-                        if (indexMetaData != null) {
-                            mappingMd = indexMetaData.mappingOrDefault(indexRequest.type());
-                        }
-                        indexRequest.resolveRouting(metaData);
-                        indexRequest.process(mappingMd, allowIdGeneration, concreteIndex.getName());
-                        break;
-                    case UPDATE:
-                        TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest);
-                        break;
-                    case DELETE:
-                        TransportDeleteAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (DeleteRequest) docWriteRequest);
-                        break;
-                    default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]");
+            final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver);
+            MetaData metaData = clusterState.metaData();
+            for (int i = 0; i < bulkRequest.requests.size(); i++) {
+                DocWriteRequest docWriteRequest = bulkRequest.requests.get(i);
+                //the request can only be null because we set it to null in the previous step, so it gets ignored
+                if (docWriteRequest == null) {
+                    continue;
+                }
+                if (addFailureIfIndexIsUnavailable(docWriteRequest, bulkRequest, responses, i, concreteIndices, metaData)) {
+                    continue;
+                }
+                Index concreteIndex = concreteIndices.resolveIfAbsent(docWriteRequest);
+                try {
+                    switch (docWriteRequest.opType()) {
+                        case CREATE:
+                        case INDEX:
+                            IndexRequest indexRequest = (IndexRequest) docWriteRequest;
+                            MappingMetaData mappingMd = null;
+                            final IndexMetaData indexMetaData = metaData.index(concreteIndex);
+                            if (indexMetaData != null) {
+                                mappingMd = indexMetaData.mappingOrDefault(indexRequest.type());
+                            }
+                            indexRequest.resolveRouting(metaData);
+                            indexRequest.process(mappingMd, allowIdGeneration, concreteIndex.getName());
+                            break;
+                        case UPDATE:
+                            TransportUpdateAction.resolveAndValidateRouting(metaData, concreteIndex.getName(), (UpdateRequest) docWriteRequest);
+                            break;
+                        case DELETE:
+                            docWriteRequest.routing(metaData.resolveIndexRouting(docWriteRequest.parent(), docWriteRequest.routing(), docWriteRequest.index()));
+                            // check if routing is required, if so, throw error if routing wasn't specified
+                            if (docWriteRequest.routing() == null && metaData.routingRequired(concreteIndex.getName(), docWriteRequest.type())) {
+                                throw new RoutingMissingException(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id());
+                            }
+                            break;
+                        default: throw new AssertionError("request type not supported: [" + docWriteRequest.opType() + "]");
+                    }
+                } catch (ElasticsearchParseException | RoutingMissingException e) {
+                    BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id(), e);
+                    BulkItemResponse bulkItemResponse = new BulkItemResponse(i, docWriteRequest.opType(), failure);
+                    responses.set(i, bulkItemResponse);
+                    // make sure the request gets never processed again
+                    bulkRequest.requests.set(i, null);
                 }
-            } catch (ElasticsearchParseException | RoutingMissingException e) {
-                BulkItemResponse.Failure failure = new BulkItemResponse.Failure(concreteIndex.getName(), docWriteRequest.type(), docWriteRequest.id(), e);
-                BulkItemResponse bulkItemResponse = new BulkItemResponse(i, docWriteRequest.opType(), failure);
-                responses.set(i, bulkItemResponse);
-                // make sure the request gets never processed again
-                bulkRequest.requests.set(i, null);
             }
-        }
 
-        // first, go over all the requests and create a ShardId -> Operations mapping
-        Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
-        for (int i = 0; i < bulkRequest.requests.size(); i++) {
-            DocWriteRequest request = bulkRequest.requests.get(i);
-            if (request == null) {
-                continue;
+            // first, go over all the requests and create a ShardId -> Operations mapping
+            Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
+            for (int i = 0; i < bulkRequest.requests.size(); i++) {
+                DocWriteRequest request = bulkRequest.requests.get(i);
+                if (request == null) {
+                    continue;
+                }
+                String concreteIndex = concreteIndices.getConcreteIndex(request.index()).getName();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, request.id(), request.routing()).shardId();
+                List<BulkItemRequest> shardRequests = requestsByShard.computeIfAbsent(shardId, shard -> new ArrayList<>());
+                shardRequests.add(new BulkItemRequest(i, request));
             }
-            String concreteIndex = concreteIndices.getConcreteIndex(request.index()).getName();
-            ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, request.id(), request.routing()).shardId();
-            List<BulkItemRequest> shardRequests = requestsByShard.computeIfAbsent(shardId, shard -> new ArrayList<>());
-            shardRequests.add(new BulkItemRequest(i, request));
-        }
 
-        if (requestsByShard.isEmpty()) {
-            listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
-            return;
-        }
-
-        final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
-        String nodeId = clusterService.localNode().getId();
-        for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
-            final ShardId shardId = entry.getKey();
-            final List<BulkItemRequest> requests = entry.getValue();
-            BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(),
-                    requests.toArray(new BulkItemRequest[requests.size()]));
-            bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
-            bulkShardRequest.timeout(bulkRequest.timeout());
-            if (task != null) {
-                bulkShardRequest.setParentTask(nodeId, task.getId());
+            if (requestsByShard.isEmpty()) {
+                listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                return;
             }
-            shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() {
-                @Override
-                public void onResponse(BulkShardResponse bulkShardResponse) {
-                    for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
-                        // we may have no response if item failed
-                        if (bulkItemResponse.getResponse() != null) {
-                            bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
+
+            final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
+            String nodeId = clusterService.localNode().getId();
+            for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
+                final ShardId shardId = entry.getKey();
+                final List<BulkItemRequest> requests = entry.getValue();
+                BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(),
+                        requests.toArray(new BulkItemRequest[requests.size()]));
+                bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards());
+                bulkShardRequest.timeout(bulkRequest.timeout());
+                if (task != null) {
+                    bulkShardRequest.setParentTask(nodeId, task.getId());
+                }
+                shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() {
+                    @Override
+                    public void onResponse(BulkShardResponse bulkShardResponse) {
+                        for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
+                            // we may have no response if item failed
+                            if (bulkItemResponse.getResponse() != null) {
+                                bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
+                            }
+                            responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
+                        }
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
                         }
-                        responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
                     }
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        // create failures for all relevant requests
+                        for (BulkItemRequest request : requests) {
+                            final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
+                            DocWriteRequest docWriteRequest = request.request();
+                            responses.set(request.id(), new BulkItemResponse(request.id(), docWriteRequest.opType(),
+                                    new BulkItemResponse.Failure(indexName, docWriteRequest.type(), docWriteRequest.id(), e)));
+                        }
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
+                        }
                     }
+
+                    private void finishHim() {
+                        listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                    }
+                });
+            }
+        }
+
+        private boolean handleBlockExceptions(ClusterState state) {
+            ClusterBlockException blockException = state.blocks().globalBlockedException(ClusterBlockLevel.WRITE);
+            if (blockException != null) {
+                if (blockException.retryable()) {
+                    logger.trace("cluster is blocked, scheduling a retry", blockException);
+                    retry(blockException);
+                } else {
+                    onFailure(blockException);
+                }
+                return true;
+            }
+            return false;
+        }
+
+        void retry(Exception failure) {
+            assert failure != null;
+            if (observer.isTimedOut()) {
+                // we running as a last attempt after a timeout has happened. don't retry
+                onFailure(failure);
+                return;
+            }
+            observer.waitForNextChange(new ClusterStateObserver.Listener() {
+                @Override
+                public void onNewClusterState(ClusterState state) {
+                    run();
                 }
 
                 @Override
-                public void onFailure(Exception e) {
-                    // create failures for all relevant requests
-                    for (BulkItemRequest request : requests) {
-                        final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
-                        DocWriteRequest docWriteRequest = request.request();
-                        responses.set(request.id(), new BulkItemResponse(request.id(), docWriteRequest.opType(),
-                            new BulkItemResponse.Failure(indexName, docWriteRequest.type(), docWriteRequest.id(), e)));
-                    }
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
+                public void onClusterServiceClose() {
+                    onFailure(new NodeClosedException(clusterService.localNode()));
                 }
 
-                private void finishHim() {
-                    listener.onResponse(new BulkResponse(responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTimeNanos)));
+                @Override
+                public void onTimeout(TimeValue timeout) {
+                    // Try one more time...
+                    run();
                 }
             });
         }
     }
 
+    void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener<BulkResponse> listener, final AtomicArray<BulkItemResponse> responses ) {
+        new BulkOperation(task, bulkRequest, listener, responses, startTimeNanos).run();
+    }
+
     private boolean addFailureIfIndexIsUnavailable(DocWriteRequest request, BulkRequest bulkRequest, AtomicArray<BulkItemResponse> responses, int idx,
                                                    final ConcreteIndices concreteIndices,
                                                    final MetaData metaData) {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index cef89e1ce78..f5faa184036 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.bulk;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -28,6 +29,8 @@ import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.replication.ReplicationOperation;
+import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
 import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.action.update.UpdateHelper;
@@ -47,28 +50,22 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
+import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.IndexShardClosedException;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.action.delete.TransportDeleteAction.executeDeleteRequestOnPrimary;
-import static org.elasticsearch.action.delete.TransportDeleteAction.executeDeleteRequestOnReplica;
-import static org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary;
-import static org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnReplica;
-import static org.elasticsearch.action.support.replication.ReplicationOperation.ignoreReplicaException;
-import static org.elasticsearch.action.support.replication.ReplicationOperation.isConflictException;
-
 /** Performs shard-level bulk (index, delete or update) operations */
 public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardRequest, BulkShardResponse> {
 
@@ -106,7 +103,8 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
     }
 
     @Override
-    protected WritePrimaryResult shardOperationOnPrimary(BulkShardRequest request, IndexShard primary) throws Exception {
+    public WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(
+            BulkShardRequest request, IndexShard primary) throws Exception {
         final IndexMetaData metaData = primary.indexSettings().getIndexMetaData();
 
         long[] preVersions = new long[request.items().length];
@@ -122,7 +120,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
             responses[i] = items[i].getPrimaryResponse();
         }
         BulkShardResponse response = new BulkShardResponse(request.shardId(), responses);
-        return new WritePrimaryResult(request, response, location, null, primary);
+        return new WritePrimaryResult<>(request, response, location, null, primary, logger);
     }
 
     /** Executes bulk item requests and handles request execution exceptions */
@@ -151,7 +149,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         final long version = indexResult.getVersion();
                         indexRequest.version(version);
                         indexRequest.versionType(indexRequest.versionType().versionTypeForReplicationAndRecovery());
-                        indexRequest.seqNo(indexResult.getSeqNo());
+                        indexRequest.setSeqNo(indexResult.getSeqNo());
                         assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());
                         response = new IndexResponse(primary.shardId(), indexRequest.type(), indexRequest.id(), indexResult.getSeqNo(),
                             indexResult.getVersion(), indexResult.isCreated());
@@ -175,7 +173,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         // update the request with the version so it will go to the replicas
                         deleteRequest.versionType(deleteRequest.versionType().versionTypeForReplicationAndRecovery());
                         deleteRequest.version(deleteResult.getVersion());
-                        deleteRequest.seqNo(deleteResult.getSeqNo());
+                        deleteRequest.setSeqNo(deleteResult.getSeqNo());
                         assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
                         response = new DeleteResponse(request.shardId(), deleteRequest.type(), deleteRequest.id(), deleteResult.getSeqNo(),
                             deleteResult.getVersion(), deleteResult.isFound());
@@ -235,6 +233,10 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
         return location;
     }
 
+    private static boolean isConflictException(final Exception e) {
+        return ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException;
+    }
+
     private static class UpdateResultHolder {
         final BulkItemRequest replicaRequest;
         final Engine.Result operationResult;
@@ -286,7 +288,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         final long version = updateOperationResult.getVersion();
                         indexRequest.version(version);
                         indexRequest.versionType(indexRequest.versionType().versionTypeForReplicationAndRecovery());
-                        indexRequest.seqNo(updateOperationResult.getSeqNo());
+                        indexRequest.setSeqNo(updateOperationResult.getSeqNo());
                         assert indexRequest.versionType().validateVersionForWrites(indexRequest.version());
                     }
                     break;
@@ -297,7 +299,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         // update the request with the version so it will go to the replicas
                         deleteRequest.versionType(deleteRequest.versionType().versionTypeForReplicationAndRecovery());
                         deleteRequest.version(updateOperationResult.getVersion());
-                        deleteRequest.seqNo(updateOperationResult.getSeqNo());
+                        deleteRequest.setSeqNo(updateOperationResult.getSeqNo());
                         assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
                     }
                     break;
@@ -326,7 +328,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) ||
                             (updateRequest.fields() != null && updateRequest.fields().length > 0)) {
                             Tuple<XContentType, Map<String, Object>> sourceAndContent =
-                                XContentHelper.convertToMap(indexSourceAsBytes, true);
+                                XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
                             updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(),
                                 indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
                         }
@@ -349,9 +351,9 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         break;
                 }
                 assert (replicaRequest.request() instanceof IndexRequest
-                    && ((IndexRequest) replicaRequest.request()).seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) ||
+                    && ((IndexRequest) replicaRequest.request()).getSeqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) ||
                     (replicaRequest.request() instanceof DeleteRequest
-                        && ((DeleteRequest) replicaRequest.request()).seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO);
+                        && ((DeleteRequest) replicaRequest.request()).getSeqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO);
                 // successful operation
                 break; // out of retry loop
             } else if (updateOperationResult.getFailure() instanceof VersionConflictEngineException == false) {
@@ -363,7 +365,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
     }
 
     @Override
-    protected WriteReplicaResult shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
+    public WriteReplicaResult<BulkShardRequest> shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
         Translog.Location location = null;
         for (int i = 0; i < request.items().length; i++) {
             BulkItemRequest item = request.items()[i];
@@ -388,11 +390,9 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                         Exception failure = operationResult.getFailure();
                         assert failure instanceof VersionConflictEngineException
                             || failure instanceof MapperParsingException
-                            || failure instanceof EngineClosedException
-                            || failure instanceof IndexShardClosedException
                             : "expected any one of [version conflict, mapper parsing, engine closed, index shard closed]" +
                             " failures. got " + failure;
-                        if (!ignoreReplicaException(failure)) {
+                        if (!TransportActions.isShardNotAvailableException(failure)) {
                             throw failure;
                         }
                     } else {
@@ -401,13 +401,13 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                 } catch (Exception e) {
                     // if its not an ignore replica failure, we need to make sure to bubble up the failure
                     // so we will fail the shard
-                    if (!ignoreReplicaException(e)) {
+                    if (!TransportActions.isShardNotAvailableException(e)) {
                         throw e;
                     }
                 }
             }
         }
-        return new WriteReplicaResult(request, location, null, replica);
+        return new WriteReplicaResult<>(request, location, null, replica, logger);
     }
 
     private Translog.Location locationToSync(Translog.Location current, Translog.Location next) {
@@ -421,4 +421,79 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
         return next;
     }
 
+    /**
+     * Execute the given {@link IndexRequest} on a replica shard, throwing a
+     * {@link RetryOnReplicaException} if the operation needs to be re-tried.
+     */
+    public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) throws IOException {
+        final ShardId shardId = replica.shardId();
+        SourceToParse sourceToParse =
+            SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source(),
+                request.getContentType()).routing(request.routing()).parent(request.parent());
+
+        final Engine.Index operation;
+        try {
+            operation = replica.prepareIndexOnReplica(sourceToParse, request.getSeqNo(), request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
+        } catch (MapperParsingException e) {
+            return new Engine.IndexResult(e, request.version(), request.getSeqNo());
+        }
+        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
+        if (update != null) {
+            throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update);
+        }
+        return replica.index(operation);
+    }
+
+    /** Utility method to prepare an index operation on primary shards */
+    static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) {
+        SourceToParse sourceToParse =
+            SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(),
+                request.getContentType()).routing(request.routing()).parent(request.parent());
+        return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
+    }
+
+    /** Executes index operation on primary shard after updates mapping if dynamic mappings are found */
+    public static Engine.IndexResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard primary,
+                                                                  MappingUpdatedAction mappingUpdatedAction) throws Exception {
+        Engine.Index operation;
+        try {
+            operation = prepareIndexOperationOnPrimary(request, primary);
+        } catch (MapperParsingException | IllegalArgumentException e) {
+            return new Engine.IndexResult(e, request.version(), request.getSeqNo());
+        }
+        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
+        final ShardId shardId = primary.shardId();
+        if (update != null) {
+            // can throw timeout exception when updating mappings or ISE for attempting to update default mappings
+            // which are bubbled up
+            try {
+                mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
+            } catch (IllegalArgumentException e) {
+                // throws IAE on conflicts merging dynamic mappings
+                return new Engine.IndexResult(e, request.version(), request.getSeqNo());
+            }
+            try {
+                operation = prepareIndexOperationOnPrimary(request, primary);
+            } catch (MapperParsingException | IllegalArgumentException e) {
+                return new Engine.IndexResult(e, request.version(), request.getSeqNo());
+            }
+            update = operation.parsedDoc().dynamicMappingsUpdate();
+            if (update != null) {
+                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
+                        "Dynamic mappings are not available on the node that holds the primary yet");
+            }
+        }
+        return primary.index(operation);
+    }
+
+    public static Engine.DeleteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard primary) throws IOException {
+        final Engine.Delete delete = primary.prepareDeleteOnPrimary(request.type(), request.id(), request.version(), request.versionType());
+        return primary.delete(delete);
+    }
+
+    public static Engine.DeleteResult executeDeleteRequestOnReplica(DeleteRequest request, IndexShard replica) throws IOException {
+        final Engine.Delete delete = replica.prepareDeleteOnReplica(request.type(), request.id(),
+                request.getSeqNo(), request.primaryTerm(), request.version(), request.versionType());
+        return replica.delete(delete);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java
new file mode 100644
index 00000000000..ed17971a77c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.function.Supplier;
+
+/** use transport bulk action directly */
+@Deprecated
+public abstract class TransportSingleItemBulkWriteAction<
+    Request extends ReplicatedWriteRequest<Request>,
+    Response extends ReplicationResponse & WriteResponse
+    > extends TransportWriteAction<Request, Request, Response> {
+
+    private final TransportBulkAction bulkAction;
+    private final TransportShardBulkAction shardBulkAction;
+
+
+    protected TransportSingleItemBulkWriteAction(Settings settings, String actionName, TransportService transportService,
+                                                 ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool,
+                                                 ShardStateAction shardStateAction, ActionFilters actionFilters,
+                                                 IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
+                                                 Supplier<Request> replicaRequest, String executor,
+                                                 TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
+        super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+            indexNameExpressionResolver, request, replicaRequest, executor);
+        this.bulkAction = bulkAction;
+        this.shardBulkAction = shardBulkAction;
+    }
+
+
+    @Override
+    protected void doExecute(Task task, final Request request, final ActionListener<Response> listener) {
+        bulkAction.execute(task, toSingleItemBulkRequest(request), wrapBulkResponse(listener));
+    }
+
+    @Override
+    protected WritePrimaryResult<Request, Response> shardOperationOnPrimary(
+        Request request, final IndexShard primary) throws Exception {
+        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
+        WriteRequest.RefreshPolicy refreshPolicy = request.getRefreshPolicy();
+        request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
+        itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) request));
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(request.shardId(), refreshPolicy, itemRequests);
+        WritePrimaryResult<BulkShardRequest, BulkShardResponse> bulkResult =
+            shardBulkAction.shardOperationOnPrimary(bulkShardRequest, primary);
+        assert bulkResult.finalResponseIfSuccessful.getResponses().length == 1 : "expected only one bulk shard response";
+        BulkItemResponse itemResponse = bulkResult.finalResponseIfSuccessful.getResponses()[0];
+        final Response response;
+        final Exception failure;
+        if (itemResponse.isFailed()) {
+            failure = itemResponse.getFailure().getCause();
+            response = null;
+        } else {
+            response = (Response) itemResponse.getResponse();
+            failure = null;
+        }
+        return new WritePrimaryResult<>(request, response, bulkResult.location, failure, primary, logger);
+    }
+
+    @Override
+    protected WriteReplicaResult<Request> shardOperationOnReplica(
+        Request replicaRequest, IndexShard replica) throws Exception {
+        BulkItemRequest[] itemRequests = new BulkItemRequest[1];
+        WriteRequest.RefreshPolicy refreshPolicy = replicaRequest.getRefreshPolicy();
+        itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) replicaRequest));
+        BulkShardRequest bulkShardRequest = new BulkShardRequest(replicaRequest.shardId(), refreshPolicy, itemRequests);
+        WriteReplicaResult<BulkShardRequest> result = shardBulkAction.shardOperationOnReplica(bulkShardRequest, replica);
+        // a replica operation can never throw a document-level failure,
+        // as the same document has been already indexed successfully in the primary
+        return new WriteReplicaResult<>(replicaRequest, result.location, null, replica, logger);
+    }
+
+
+    public static <Response extends ReplicationResponse & WriteResponse>
+    ActionListener<BulkResponse> wrapBulkResponse(ActionListener<Response> listener) {
+        return ActionListener.wrap(bulkItemResponses -> {
+            assert bulkItemResponses.getItems().length == 1 : "expected only one item in bulk request";
+            BulkItemResponse bulkItemResponse = bulkItemResponses.getItems()[0];
+            if (bulkItemResponse.isFailed() == false) {
+                final DocWriteResponse response = bulkItemResponse.getResponse();
+                listener.onResponse((Response) response);
+            } else {
+                listener.onFailure(bulkItemResponse.getFailure().getCause());
+            }
+        }, listener::onFailure);
+    }
+
+    public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) {
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(((DocWriteRequest) request));
+        bulkRequest.setRefreshPolicy(request.getRefreshPolicy());
+        bulkRequest.timeout(request.timeout());
+        bulkRequest.waitForActiveShards(request.waitForActiveShards());
+        request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
+        return bulkRequest;
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollAction.java
similarity index 51%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollAction.java
index 34ad6e2ee25..834321f1798 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollAction.java
@@ -17,16 +17,29 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.Retry;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.IndexFieldMapper;
@@ -40,27 +53,60 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiFunction;
 
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
+import static org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.rest.RestStatus.CONFLICT;
+import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
 
 /**
- * Abstract base for scrolling across a search and executing bulk indexes on all
- * results.
+ * Abstract base for scrolling across a search and executing bulk actions on all results. All package private methods are package private so
+ * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block.
  */
-public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends AbstractBulkByScrollRequest<Request>>
-        extends AbstractAsyncBulkByScrollAction<Request> {
-
+public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>> {
+    protected final Logger logger;
+    protected final WorkingBulkByScrollTask task;
+    protected final ThreadPool threadPool;
     protected final ScriptService scriptService;
     protected final ClusterState clusterState;
 
+    /**
+     * The request for this action. Named mainRequest because we create lots of <code>request</code> variables all representing child
+     * requests of this mainRequest.
+     */
+    protected final Request mainRequest;
+
+    private final AtomicLong startTime = new AtomicLong(-1);
+    private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+    private final ParentTaskAssigningClient client;
+    private final ActionListener<BulkByScrollResponse> listener;
+    private final Retry bulkRetry;
+    private final ScrollableHitSource scrollSource;
+
     /**
      * This BiFunction is used to apply various changes depending of the Reindex action and  the search hit,
      * from copying search hit metadata (parent, routing, etc) to potentially transforming the
@@ -68,54 +114,47 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
      */
     private final BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> scriptApplier;
 
-    public AbstractAsyncBulkIndexByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                                                ThreadPool threadPool, Request mainRequest,
-                                                ActionListener<BulkIndexByScrollResponse> listener,
-                                                ScriptService scriptService, ClusterState clusterState) {
-        super(task, logger, client, threadPool, mainRequest, listener);
+    public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+            ThreadPool threadPool, Request mainRequest, ScriptService scriptService, ClusterState clusterState,
+            ActionListener<BulkByScrollResponse> listener) {
+        this.task = task;
+        this.logger = logger;
+        this.client = client;
+        this.threadPool = threadPool;
         this.scriptService = scriptService;
         this.clusterState = clusterState;
-        this.scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
+        this.mainRequest = mainRequest;
+        this.listener = listener;
+        BackoffPolicy backoffPolicy = buildBackoffPolicy();
+        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry));
+        scrollSource = buildScrollableResultSource(backoffPolicy);
+        scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
+        /*
+         * Default to sorting by doc. We can't do this in the request itself because it is normal to *add* to the sorts rather than replace
+         * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't
+         * another sort.
+         */
+        List<SortBuilder<?>> sorts = mainRequest.getSearchRequest().source().sorts();
+        if (sorts == null || sorts.isEmpty()) {
+            mainRequest.getSearchRequest().source().sort(fieldSort("_doc"));
+        }
+        mainRequest.getSearchRequest().source().version(needsSourceDocumentVersions());
     }
 
     /**
      * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
+     *
+     * Public for testings....
      */
-    protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+    public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
         // The default script applier executes a no-op
         return (request, searchHit) -> request;
     }
 
-    @Override
-    protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
-        BulkRequest bulkRequest = new BulkRequest();
-        for (ScrollableHitSource.Hit doc : docs) {
-            if (accept(doc)) {
-                RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
-                if (request != null) {
-                    bulkRequest.add(request.self());
-                }
-            }
-        }
-        return bulkRequest;
-    }
-
     /**
-     * Used to accept or ignore a search hit. Ignored search hits will be excluded
-     * from the bulk request. It is also where we fail on invalid search hits, like
-     * when the document has no source but it's required.
+     * Does this operation need the versions of the source documents?
      */
-    protected boolean accept(ScrollableHitSource.Hit doc) {
-        if (doc.getSource() == null) {
-            /*
-             * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
-             * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
-             * Thus the error message assumes that it wasn't stored.
-             */
-            throw new IllegalArgumentException("[" + doc.getIndex() + "][" + doc.getType() + "][" + doc.getId() + "] didn't store _source");
-        }
-        return true;
-    }
+    protected abstract boolean needsSourceDocumentVersions();
 
     /**
      * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle
@@ -140,10 +179,328 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
         request.setRouting(routing);
     }
 
+    /**
+     * Used to accept or ignore a search hit. Ignored search hits will be excluded
+     * from the bulk request. It is also where we fail on invalid search hits, like
+     * when the document has no source but it's required.
+     */
+    protected boolean accept(ScrollableHitSource.Hit doc) {
+        if (doc.getSource() == null) {
+            /*
+             * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
+             * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
+             * Thus the error message assumes that it wasn't stored.
+             */
+            throw new IllegalArgumentException("[" + doc.getIndex() + "][" + doc.getType() + "][" + doc.getId() + "] didn't store _source");
+        }
+        return true;
+    }
+
+    private BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
+        BulkRequest bulkRequest = new BulkRequest();
+        for (ScrollableHitSource.Hit doc : docs) {
+            if (accept(doc)) {
+                RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
+                if (request != null) {
+                    bulkRequest.add(request.self());
+                }
+            }
+        }
+        return bulkRequest;
+    }
+
+    protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) {
+        return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client,
+                mainRequest.getSearchRequest());
+    }
+
+    /**
+     * Build the response for reindex actions.
+     */
+    protected BulkByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
+                                                      List<SearchFailure> searchFailures, boolean timedOut) {
+        return new BulkByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
+    }
+
+    /**
+     * Start the action by firing the initial search request.
+     */
+    public void start() {
+        if (task.isCancelled()) {
+            finishHim(null);
+            return;
+        }
+        try {
+            startTime.set(System.nanoTime());
+            scrollSource.start(response -> onScrollResponse(timeValueNanos(System.nanoTime()), 0, response));
+        } catch (Exception e) {
+            finishHim(e);
+        }
+    }
+
+    /**
+     * Process a scroll response.
+     * @param lastBatchStartTime the time when the last batch started. Used to calculate the throttling delay.
+     * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay.
+     * @param response the scroll response to process
+     */
+    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, ScrollableHitSource.Response response) {
+        if (task.isCancelled()) {
+            finishHim(null);
+            return;
+        }
+        if (    // If any of the shards failed that should abort the request.
+                (response.getFailures().size() > 0)
+                // Timeouts aren't shard failures but we still need to pass them back to the user.
+                || response.isTimedOut()
+                ) {
+            refreshAndFinish(emptyList(), response.getFailures(), response.isTimedOut());
+            return;
+        }
+        long total = response.getTotalHits();
+        if (mainRequest.getSize() > 0) {
+            total = min(total, mainRequest.getSize());
+        }
+        task.setTotal(total);
+        AbstractRunnable prepareBulkRequestRunnable = new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                /*
+                 * It is important that the batch start time be calculated from here, scroll response to scroll response. That way the time
+                 * waiting on the scroll doesn't count against this batch in the throttle.
+                 */
+                prepareBulkRequest(timeValueNanos(System.nanoTime()), response);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                finishHim(e);
+            }
+        };
+        prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable);
+        task.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable);
+    }
+
+    /**
+     * Prepare the bulk request. Called on the generic thread pool after some preflight checks have been done one the SearchResponse and any
+     * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
+     * thread may be blocked by the user script.
+     */
+    void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
+        if (task.isCancelled()) {
+            finishHim(null);
+            return;
+        }
+        if (response.getHits().isEmpty()) {
+            refreshAndFinish(emptyList(), emptyList(), false);
+            return;
+        }
+        task.countBatch();
+        List<? extends ScrollableHitSource.Hit> hits = response.getHits();
+        if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
+            // Truncate the hits if we have more than the request size
+            long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed());
+            if (remaining < hits.size()) {
+                hits = hits.subList(0, (int) remaining);
+            }
+        }
+        BulkRequest request = buildBulk(hits);
+        if (request.requests().isEmpty()) {
+            /*
+             * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
+             */
+            startNextScroll(thisBatchStartTime, 0);
+            return;
+        }
+        request.timeout(mainRequest.getTimeout());
+        request.waitForActiveShards(mainRequest.getWaitForActiveShards());
+        if (logger.isDebugEnabled()) {
+            logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(),
+                    new ByteSizeValue(request.estimatedSizeInBytes()));
+        }
+        sendBulkRequest(thisBatchStartTime, request);
+    }
+
+    /**
+     * Send a bulk request, handling retries.
+     */
+    void sendBulkRequest(TimeValue thisBatchStartTime, BulkRequest request) {
+        if (task.isCancelled()) {
+            finishHim(null);
+            return;
+        }
+        bulkRetry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
+            @Override
+            public void onResponse(BulkResponse response) {
+                onBulkResponse(thisBatchStartTime, response);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                finishHim(e);
+            }
+        });
+    }
+
+    /**
+     * Processes bulk responses, accounting for failures.
+     */
+    void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
+        try {
+            List<Failure> failures = new ArrayList<Failure>();
+            Set<String> destinationIndicesThisBatch = new HashSet<>();
+            for (BulkItemResponse item : response) {
+                if (item.isFailed()) {
+                    recordFailure(item.getFailure(), failures);
+                    continue;
+                }
+                switch (item.getOpType()) {
+                    case CREATE:
+                    case INDEX:
+                        if (item.getResponse().getResult() == DocWriteResponse.Result.CREATED) {
+                            task.countCreated();
+                        } else {
+                            task.countUpdated();
+                        }
+                        break;
+                    case UPDATE:
+                        task.countUpdated();
+                        break;
+                    case DELETE:
+                        task.countDeleted();
+                        break;
+                }
+                // Track the indexes we've seen so we can refresh them if requested
+                destinationIndicesThisBatch.add(item.getIndex());
+            }
+
+            if (task.isCancelled()) {
+                finishHim(null);
+                return;
+            }
+
+            addDestinationIndices(destinationIndicesThisBatch);
+
+            if (false == failures.isEmpty()) {
+                refreshAndFinish(unmodifiableList(failures), emptyList(), false);
+                return;
+            }
+
+            if (mainRequest.getSize() != SIZE_ALL_MATCHES && task.getSuccessfullyProcessed() >= mainRequest.getSize()) {
+                // We've processed all the requested docs.
+                refreshAndFinish(emptyList(), emptyList(), false);
+                return;
+            }
+
+            startNextScroll(thisBatchStartTime, response.getItems().length);
+        } catch (Exception t) {
+            finishHim(t);
+        }
+    }
+
+    /**
+     * Start the next scroll request.
+     *
+     * @param lastBatchSize the number of requests sent in the last batch. This is used to calculate the throttling values which are applied
+     *        when the scroll returns
+     */
+    void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
+        if (task.isCancelled()) {
+            finishHim(null);
+            return;
+        }
+        TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, lastBatchSize);
+        scrollSource.startNextScroll(extraKeepAlive, response -> {
+            onScrollResponse(lastBatchStartTime, lastBatchSize, response);
+        });
+    }
+
+    private void recordFailure(Failure failure, List<Failure> failures) {
+        if (failure.getStatus() == CONFLICT) {
+            task.countVersionConflict();
+            if (false == mainRequest.isAbortOnVersionConflict()) {
+                return;
+            }
+        }
+        failures.add(failure);
+    }
+
+    /**
+     * Start terminating a request that finished non-catastrophically by refreshing the modified indices and then proceeding to
+     * {@link #finishHim(Exception, List, List, boolean)}.
+     */
+    void refreshAndFinish(List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
+        if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) {
+            finishHim(null, indexingFailures, searchFailures, timedOut);
+            return;
+        }
+        RefreshRequest refresh = new RefreshRequest();
+        refresh.indices(destinationIndices.toArray(new String[destinationIndices.size()]));
+        client.admin().indices().refresh(refresh, new ActionListener<RefreshResponse>() {
+            @Override
+            public void onResponse(RefreshResponse response) {
+                finishHim(null, indexingFailures, searchFailures, timedOut);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                finishHim(e);
+            }
+        });
+    }
+
+    /**
+     * Finish the request.
+     *
+     * @param failure if non null then the request failed catastrophically with this exception
+     */
+    protected void finishHim(Exception failure) {
+        finishHim(failure, emptyList(), emptyList(), false);
+    }
+
+    /**
+     * Finish the request.
+     * @param failure if non null then the request failed catastrophically with this exception
+     * @param indexingFailures any indexing failures accumulated during the request
+     * @param searchFailures any search failures accumulated during the request
+     * @param timedOut have any of the sub-requests timed out?
+     */
+    protected void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
+        scrollSource.close();
+        if (failure == null) {
+            listener.onResponse(
+                    buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures, timedOut));
+        } else {
+            listener.onFailure(failure);
+        }
+    }
+
+    /**
+     * Get the backoff policy for use with retries.
+     */
+    BackoffPolicy buildBackoffPolicy() {
+        return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries());
+    }
+
+    /**
+     * Add to the list of indices that were modified by this request. This is the list of indices refreshed at the end of the request if the
+     * request asks for a refresh.
+     */
+    void addDestinationIndices(Collection<String> indices) {
+        destinationIndices.addAll(indices);
+    }
+
+    /**
+     * Set the last returned scrollId. Exists entirely for testing.
+     */
+    void setScroll(String scroll) {
+        scrollSource.setScroll(scroll);
+    }
+
     /**
      * Wrapper for the {@link DocWriteRequest} that are used in this action class.
      */
-    interface RequestWrapper<Self extends DocWriteRequest<Self>> {
+    public interface RequestWrapper<Self extends DocWriteRequest<Self>> {
 
         void setIndex(String index);
 
@@ -273,7 +630,7 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
     /**
      * Wraps a {@link IndexRequest} in a {@link RequestWrapper}
      */
-    static RequestWrapper<IndexRequest> wrap(IndexRequest request) {
+    public static RequestWrapper<IndexRequest> wrap(IndexRequest request) {
         return new IndexRequestWrapper(request);
     }
 
@@ -404,7 +761,7 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
                 return request;
             }
             if (executable == null) {
-                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE, emptyMap());
+                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE);
                 executable = scriptService.executable(compiled, params);
             }
             if (context == null) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequest.java
similarity index 99%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequest.java
index bb546fcbc63..44580875011 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequest.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
@@ -355,7 +355,7 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
     /**
      * Build a new request for a slice of the parent request.
      */
-    abstract Self forSlice(TaskId slicingTask, SearchRequest slice);
+    protected abstract Self forSlice(TaskId slicingTask, SearchRequest slice);
 
     /**
      * Setup a clone of this request with the information needed to process a slice of it.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestBuilder.java
similarity index 95%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestBuilder.java
index 185c271e70f..4131175f9df 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestBuilder.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestBuilder;
@@ -31,11 +31,11 @@ import org.elasticsearch.index.query.QueryBuilder;
 public abstract class AbstractBulkByScrollRequestBuilder<
                 Request extends AbstractBulkByScrollRequest<Request>,
                 Self extends AbstractBulkByScrollRequestBuilder<Request, Self>>
-        extends ActionRequestBuilder<Request, BulkIndexByScrollResponse, Self> {
+        extends ActionRequestBuilder<Request, BulkByScrollResponse, Self> {
     private final SearchRequestBuilder source;
 
     protected AbstractBulkByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder source, Request request) {
+            Action<Request, BulkByScrollResponse, Self> action, SearchRequestBuilder source, Request request) {
         super(client, action, request);
         this.source = source;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AsyncDeleteByQueryAction.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AsyncDeleteByQueryAction.java
new file mode 100644
index 00000000000..cdcfb754fb6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/AsyncDeleteByQueryAction.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk.byscroll;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+
+/**
+ * Implementation of delete-by-query using scrolling and bulk.
+ */
+public class AsyncDeleteByQueryAction extends AbstractAsyncBulkByScrollAction<DeleteByQueryRequest> {
+    public AsyncDeleteByQueryAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+            ThreadPool threadPool, DeleteByQueryRequest request, ScriptService scriptService, ClusterState clusterState,
+            ActionListener<BulkByScrollResponse> listener) {
+        super(task, logger, client, threadPool, request, scriptService, clusterState, listener);
+    }
+
+    @Override
+    protected boolean needsSourceDocumentVersions() {
+        /*
+         * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been
+         * changed.
+         */
+        return true;
+    }
+
+    @Override
+    protected boolean accept(ScrollableHitSource.Hit doc) {
+        // Delete-by-query does not require the source to delete a document
+        // and the default implementation checks for it
+        return true;
+    }
+
+    @Override
+    protected RequestWrapper<DeleteRequest> buildRequest(ScrollableHitSource.Hit doc) {
+        DeleteRequest delete = new DeleteRequest();
+        delete.index(doc.getIndex());
+        delete.type(doc.getType());
+        delete.id(doc.getId());
+        delete.version(doc.getVersion());
+        return wrap(delete);
+    }
+
+    /**
+     * Overrides the parent's implementation is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
+     * don't care for a deletion.
+     */
+    @Override
+    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+        request.setParent(doc.getParent());
+        request.setRouting(doc.getRouting());
+        return request;
+    }
+
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexParallelizationHelper.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelper.java
similarity index 86%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexParallelizationHelper.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelper.java
index b2dbd51f381..f2bd62c2335 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexParallelizationHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelper.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
@@ -32,24 +32,21 @@ import org.elasticsearch.tasks.TaskManager;
 /**
  * Helps parallelize reindex requests using sliced scrolls.
  */
-public class ReindexParallelizationHelper {
-    private ReindexParallelizationHelper() {}
+public class BulkByScrollParallelizationHelper {
+    private BulkByScrollParallelizationHelper() {}
 
     public static <
                 Request extends AbstractBulkByScrollRequest<Request>
-            > void startSlices(Client client, TaskManager taskManager, Action<Request, BulkIndexByScrollResponse, ?> action,
-                    String localNodeId, ParentBulkByScrollTask task, Request request, ActionListener<BulkIndexByScrollResponse> listener) {
+            > void startSlices(Client client, TaskManager taskManager, Action<Request, BulkByScrollResponse, ?> action,
+                    String localNodeId, ParentBulkByScrollTask task, Request request, ActionListener<BulkByScrollResponse> listener) {
         TaskId parentTaskId = new TaskId(localNodeId, task.getId());
         for (final SearchRequest slice : sliceIntoSubRequests(request.getSearchRequest(), UidFieldMapper.NAME, request.getSlices())) {
             // TODO move the request to the correct node. maybe here or somehow do it as part of startup for reindex in general....
             Request requestForSlice = request.forSlice(parentTaskId, slice);
-            ActionListener<BulkIndexByScrollResponse> sliceListener = ActionListener.wrap(
+            ActionListener<BulkByScrollResponse> sliceListener = ActionListener.wrap(
                     r -> task.onSliceResponse(listener, slice.source().slice().getId(), r),
                     e -> task.onSliceFailure(listener, slice.source().slice().getId(), e));
             client.execute(action, requestForSlice, sliceListener);
-            /* Explicitly tell the task manager that we're running child tasks on the local node so it will cancel them when the parent is
-             * cancelled. */
-            taskManager.registerChildTask(task, localNodeId);
         }
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponse.java
similarity index 91%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponse.java
index 1574a167108..20f2cf2ed06 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponse.java
@@ -17,17 +17,17 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -41,17 +41,17 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 /**
  * Response used for actions that index many documents using a scroll request.
  */
-public class BulkIndexByScrollResponse extends ActionResponse implements ToXContent {
+public class BulkByScrollResponse extends ActionResponse implements ToXContent {
     private TimeValue took;
     private BulkByScrollTask.Status status;
     private List<Failure> bulkFailures;
     private List<SearchFailure> searchFailures;
     private boolean timedOut;
 
-    public BulkIndexByScrollResponse() {
+    public BulkByScrollResponse() {
     }
 
-    public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> bulkFailures,
+    public BulkByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> bulkFailures,
                                      List<SearchFailure> searchFailures, boolean timedOut) {
         this.took = took;
         this.status = requireNonNull(status, "Null status not supported");
@@ -60,12 +60,12 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         this.timedOut = timedOut;
     }
 
-    public BulkIndexByScrollResponse(Iterable<BulkIndexByScrollResponse> toMerge, @Nullable String reasonCancelled) {
+    public BulkByScrollResponse(Iterable<BulkByScrollResponse> toMerge, @Nullable String reasonCancelled) {
         long mergedTook = 0;
         List<BulkByScrollTask.StatusOrException> statuses = new ArrayList<>();
         bulkFailures = new ArrayList<>();
         searchFailures = new ArrayList<>();
-        for (BulkIndexByScrollResponse response : toMerge) {
+        for (BulkByScrollResponse response : toMerge) {
             mergedTook = max(mergedTook, response.getTook().nanos());
             statuses.add(new BulkByScrollTask.StatusOrException(response.status));
             bulkFailures.addAll(response.getBulkFailures());
@@ -80,7 +80,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         return took;
     }
 
-    protected BulkByScrollTask.Status getStatus() {
+    public BulkByScrollTask.Status getStatus() {
         return status;
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTask.java
similarity index 97%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTask.java
index 7de4f19339f..7c9124057b3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTask.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -51,15 +51,14 @@ public abstract class BulkByScrollTask extends CancellableTask {
 
     /**
      * The number of sub-slices that are still running. {@link WorkingBulkByScrollTask} will always have 0 and
-     * {@link ParentBulkByScrollTask} will return the number of waiting tasks. Used by {@link TransportRethrottleAction} to decide how to
-     * perform the rethrottling.
+     * {@link ParentBulkByScrollTask} will return the number of waiting tasks. Used to decide how to perform rethrottling.
      */
-    abstract int runningSliceSubTasks();
+    public abstract int runningSliceSubTasks();
 
     /**
      * Apply the {@code newRequestsPerSecond}.
      */
-    abstract void rethrottle(float newRequestsPerSecond);
+    public abstract void rethrottle(float newRequestsPerSecond);
 
     /*
      * Overridden to force children to return compatible status.
@@ -71,6 +70,11 @@ public abstract class BulkByScrollTask extends CancellableTask {
      */
     public abstract TaskInfo getInfoGivenSliceInfo(String localNodeId, List<TaskInfo> sliceInfo);
 
+    @Override
+    public boolean shouldCancelChildrenOnCancellation() {
+        return true;
+    }
+
     public static class Status implements Task.Status, SuccessfullyProcessed {
         public static final String NAME = "bulk-by-scroll";
 
@@ -486,7 +490,7 @@ public abstract class BulkByScrollTask extends CancellableTask {
                 status.toXContent(builder, params);
             } else {
                 builder.startObject();
-                ElasticsearchException.toXContent(builder, params, exception);
+                ElasticsearchException.generateThrowableXContent(builder, params, exception);
                 builder.endObject();
             }
             return builder;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java
similarity index 95%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java
index 7b6be85140f..4f2aefc1011 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ClientScrollableHitSource.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -37,6 +37,8 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.search.SearchHit;
@@ -88,11 +90,7 @@ public class ClientScrollableHitSource extends ScrollableHitSource {
     }
 
     @Override
-    public void clearScroll(String scrollId) {
-        /*
-         * Fire off the clear scroll but don't wait for it it return before
-         * we send the use their response.
-         */
+    public void clearScroll(String scrollId, Runnable onCompletion) {
         ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
         clearScrollRequest.addScrollId(scrollId);
         /*
@@ -103,15 +101,22 @@ public class ClientScrollableHitSource extends ScrollableHitSource {
             @Override
             public void onResponse(ClearScrollResponse response) {
                 logger.debug("Freed [{}] contexts", response.getNumFreed());
+                onCompletion.run();
             }
 
             @Override
             public void onFailure(Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), e);
+                onCompletion.run();
             }
         });
     }
 
+    @Override
+    protected void cleanup() {
+        // Nothing to do
+    }
+
     /**
      * Run a search action and call onResponse when a the response comes in, retrying if the action fails with an exception caused by
      * rejected execution.
@@ -182,7 +187,7 @@ public class ClientScrollableHitSource extends ScrollableHitSource {
         } else {
             failures = new ArrayList<>(response.getShardFailures().length);
             for (ShardSearchFailure failure: response.getShardFailures()) {
-                String nodeId = failure.shard() == null ? null : failure.shard().nodeId();
+                String nodeId = failure.shard() == null ? null : failure.shard().getNodeId();
                 failures.add(new SearchFailure(failure.getCause(), failure.index(), failure.shardId(), nodeId));
             }
         }
@@ -229,6 +234,10 @@ public class ClientScrollableHitSource extends ScrollableHitSource {
             return source;
         }
 
+        @Override
+        public XContentType getXContentType() {
+            return XContentFactory.xContentType(source);
+        }
         @Override
         public long getVersion() {
             return delegate.getVersion();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequest.java
similarity index 97%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequest.java
index cde5fea926c..4c261f471f2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequest.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -79,7 +79,7 @@ public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQu
     }
 
     @Override
-    DeleteByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+    protected DeleteByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
         return doForSlice(new DeleteByQueryRequest(slice, false), slicingTask);
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTask.java
similarity index 84%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTask.java
index 76b2bb76944..dfaf03f4115 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ParentBulkByScrollTask.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTask.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.collect.Tuple;
@@ -35,12 +35,12 @@ import static java.util.Collections.unmodifiableList;
 /**
  * Task for parent bulk by scroll requests that have sub-workers.
  */
-class ParentBulkByScrollTask extends BulkByScrollTask {
+public class ParentBulkByScrollTask extends BulkByScrollTask {
     /**
      * Holds the responses as they come back. This uses {@link Tuple} as an "Either" style holder where only the response or the exception
      * is set.
      */
-    private final AtomicArray<Tuple<BulkIndexByScrollResponse, Exception>> results;
+    private final AtomicArray<Tuple<BulkByScrollResponse, Exception>> results;
     private final AtomicInteger counter;
 
     public ParentBulkByScrollTask(long id, String type, String action, String description, TaskId parentTaskId, int slices) {
@@ -50,7 +50,7 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
     }
 
     @Override
-    void rethrottle(float newRequestsPerSecond) {
+    public void rethrottle(float newRequestsPerSecond) {
         // Nothing to do because all rethrottling is done on slice sub tasks.
     }
 
@@ -63,7 +63,7 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
     }
 
     @Override
-    int runningSliceSubTasks() {
+    public int runningSliceSubTasks() {
         return counter.get();
     }
 
@@ -82,7 +82,7 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
     }
 
     private void addResultsToList(List<StatusOrException> sliceStatuses) {
-        for (AtomicArray.Entry<Tuple<BulkIndexByScrollResponse, Exception>> t : results.asList()) {
+        for (AtomicArray.Entry<Tuple<BulkByScrollResponse, Exception>> t : results.asList()) {
             if (t.value != null) {
                 if (t.value.v1() != null) {
                     sliceStatuses.set(t.index, new StatusOrException(t.value.v1().getStatus()));
@@ -96,7 +96,7 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
     /**
      * Record a response from a slice and respond to the listener if the request is finished.
      */
-    void onSliceResponse(ActionListener<BulkIndexByScrollResponse> listener, int sliceId, BulkIndexByScrollResponse response) {
+    public void onSliceResponse(ActionListener<BulkByScrollResponse> listener, int sliceId, BulkByScrollResponse response) {
         results.setOnce(sliceId, new Tuple<>(response, null));
         /* If the request isn't finished we could automatically rethrottle the sub-requests here but we would only want to do that if we
          * were fairly sure they had a while left to go. */
@@ -106,19 +106,19 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
     /**
      * Record a failure from a slice and respond to the listener if the request is finished.
      */
-    void onSliceFailure(ActionListener<BulkIndexByScrollResponse> listener, int sliceId, Exception e) {
+    void onSliceFailure(ActionListener<BulkByScrollResponse> listener, int sliceId, Exception e) {
         results.setOnce(sliceId, new Tuple<>(null, e));
         recordSliceCompletionAndRespondIfAllDone(listener);
         // TODO cancel when a slice fails?
     }
 
-    private void recordSliceCompletionAndRespondIfAllDone(ActionListener<BulkIndexByScrollResponse> listener) {
+    private void recordSliceCompletionAndRespondIfAllDone(ActionListener<BulkByScrollResponse> listener) {
         if (counter.decrementAndGet() != 0) {
             return;
         }
-        List<BulkIndexByScrollResponse> responses = new ArrayList<>(results.length());
+        List<BulkByScrollResponse> responses = new ArrayList<>(results.length());
         Exception exception = null;
-        for (AtomicArray.Entry<Tuple<BulkIndexByScrollResponse, Exception>> t : results.asList()) {
+        for (AtomicArray.Entry<Tuple<BulkByScrollResponse, Exception>> t : results.asList()) {
             if (t.value.v1() == null) {
                 assert t.value.v2() != null : "exception shouldn't be null if value is null";
                 if (exception == null) {
@@ -132,7 +132,7 @@ class ParentBulkByScrollTask extends BulkByScrollTask {
             }
         }
         if (exception == null) {
-            listener.onResponse(new BulkIndexByScrollResponse(responses, getReasonCancelled()));
+            listener.onResponse(new BulkByScrollResponse(responses, getReasonCancelled()));
         } else {
             listener.onFailure(exception);
         }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java
similarity index 89%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java
index bf13d6d72e2..73aa6536986 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/ScrollableHitSource.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -32,7 +32,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -81,15 +81,26 @@ public abstract class ScrollableHitSource implements Closeable {
         });
     }
     protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse);
-    
+
     @Override
-    public void close() {
+    public final void close() {
         String scrollId = this.scrollId.get();
         if (Strings.hasLength(scrollId)) {
-            clearScroll(scrollId);
+            clearScroll(scrollId, this::cleanup);
+        } else {
+            cleanup();
         }
     }
-    protected abstract void clearScroll(String scrollId);
+    /**
+     * Called to clear a scroll id.
+     * @param scrollId the id to clear
+     * @param onCompletion implementers must call this after completing the clear whether they are successful or not
+     */
+    protected abstract void clearScroll(String scrollId, Runnable onCompletion);
+    /**
+     * Called after the process has been totally finished to clean up any resources the process needed like remote connections.
+     */
+    protected abstract void cleanup();
 
     /**
      * Set the id of the last scroll. Used for debugging.
@@ -179,6 +190,10 @@ public abstract class ScrollableHitSource implements Closeable {
          * all.
          */
         @Nullable BytesReference getSource();
+        /**
+         * The content type of the hit source. Returns null if the source didn't come back from the search.
+         */
+        @Nullable XContentType getXContentType();
         /**
          * The document id of the parent of the hit if there is a parent or null if there isn't.
          */
@@ -190,8 +205,7 @@ public abstract class ScrollableHitSource implements Closeable {
     }
 
     /**
-     * An implementation of {@linkplain Hit} that uses getters and setters. Primarily used for testing and {@link RemoteScrollableHitSource}
-     * .
+     * An implementation of {@linkplain Hit} that uses getters and setters.
      */
     public static class BasicHit implements Hit {
         private final String index;
@@ -200,6 +214,7 @@ public abstract class ScrollableHitSource implements Closeable {
         private final long version;
 
         private BytesReference source;
+        private XContentType xContentType;
         private String parent;
         private String routing;
 
@@ -235,8 +250,14 @@ public abstract class ScrollableHitSource implements Closeable {
             return source;
         }
 
-        public BasicHit setSource(BytesReference source) {
+        @Override
+        public XContentType getXContentType() {
+            return xContentType;
+        }
+
+        public BasicHit setSource(BytesReference source, XContentType xContentType) {
             this.source = source;
+            this.xContentType = xContentType;
             return this;
         }
 
@@ -337,7 +358,7 @@ public abstract class ScrollableHitSource implements Closeable {
             builder.field("reason");
             {
                 builder.startObject();
-                ElasticsearchException.toXContent(builder, params, reason);
+                ElasticsearchException.generateThrowableXContent(builder, params, reason);
                 builder.endObject();
             }
             builder.endObject();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/SuccessfullyProcessed.java
similarity index 96%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/SuccessfullyProcessed.java
index 6547984900e..a0176e35202 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/SuccessfullyProcessed.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/SuccessfullyProcessed.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 /**
  * Implemented by {@link BulkByScrollTask} and {@link BulkByScrollTask.Status} to consistently implement
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTask.java
similarity index 98%
rename from modules/reindex/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java
rename to core/src/main/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTask.java
index 3e09ee2d7fd..1b458caa3d5 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTask.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTask.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
@@ -95,7 +95,7 @@ public class WorkingBulkByScrollTask extends BulkByScrollTask implements Success
     }
 
     @Override
-    int runningSliceSubTasks() {
+    public int runningSliceSubTasks() {
         return 0;
     }
 
@@ -162,7 +162,7 @@ public class WorkingBulkByScrollTask extends BulkByScrollTask implements Success
         bulkRetries.incrementAndGet();
     }
 
-    void countSearchRetry() {
+    public void countSearchRetry() {
         searchRetries.incrementAndGet();
     }
 
@@ -209,7 +209,7 @@ public class WorkingBulkByScrollTask extends BulkByScrollTask implements Success
     }
 
     @Override
-    void rethrottle(float newRequestsPerSecond) {
+    public void rethrottle(float newRequestsPerSecond) {
         synchronized (delayedPrepareBulkRequestReference) {
             if (logger.isDebugEnabled()) {
                 logger.debug("[{}]: Rethrottling to [{}] requests per second", getId(), newRequestsPerSecond);
@@ -306,7 +306,7 @@ public class WorkingBulkByScrollTask extends BulkByScrollTask implements Success
         private final AtomicBoolean hasRun = new AtomicBoolean(false);
         private final AbstractRunnable delegate;
 
-        public RunOnce(AbstractRunnable delegate) {
+        RunOnce(AbstractRunnable delegate) {
             this.delegate = delegate;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/byscroll/package-info.java b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/package-info.java
new file mode 100644
index 00000000000..3a31ea2f654
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/bulk/byscroll/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Infrastructure for actions that modify documents based on the results of a scrolling query.
+ */
+package org.elasticsearch.action.bulk.byscroll;
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
index 280324227cc..72d8c4e5857 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.delete;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.common.Nullable;
@@ -43,7 +44,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * @see org.elasticsearch.client.Client#delete(DeleteRequest)
  * @see org.elasticsearch.client.Requests#deleteRequest(String)
  */
-public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocWriteRequest<DeleteRequest> {
+public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocWriteRequest<DeleteRequest>, CompositeIndicesRequest {
 
     private String type;
     private String id;
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
index 0f4eb897d83..9ab06a941eb 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteResponse.java
@@ -20,12 +20,20 @@
 package org.elasticsearch.action.delete;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * The response of the delete action.
  *
@@ -34,6 +42,8 @@ import java.io.IOException;
  */
 public class DeleteResponse extends DocWriteResponse {
 
+    private static final String FOUND = "found";
+
     public DeleteResponse() {
 
     }
@@ -48,12 +58,36 @@ public class DeleteResponse extends DocWriteResponse {
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("found", result == Result.DELETED);
-        super.toXContent(builder, params);
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(FOUND, result == Result.DELETED);
+        super.innerToXContent(builder, params);
         return builder;
     }
 
+    private static final ConstructingObjectParser<DeleteResponse, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(DeleteResponse.class.getName(),
+            args -> {
+                // index uuid and shard id are unknown and can't be parsed back for now.
+                ShardId shardId = new ShardId(new Index((String) args[0], IndexMetaData.INDEX_UUID_NA_VALUE), -1);
+                String type = (String) args[1];
+                String id = (String) args[2];
+                long version = (long) args[3];
+                ShardInfo shardInfo = (ShardInfo) args[5];
+                long seqNo = (args[6] != null) ? (long) args[6] : SequenceNumbersService.UNASSIGNED_SEQ_NO;
+                boolean found = (boolean) args[7];
+                DeleteResponse deleteResponse = new DeleteResponse(shardId, type, id, seqNo, version, found);
+                deleteResponse.setShardInfo(shardInfo);
+                return deleteResponse;
+            });
+        DocWriteResponse.declareParserFields(PARSER);
+        PARSER.declareBoolean(constructorArg(), new ParseField(FOUND));
+    }
+
+    public static DeleteResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
+
     @Override
     public String toString() {
         StringBuilder builder = new StringBuilder();
diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index 0d0d76c7691..3aaf4a472fa 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -19,147 +19,39 @@
 
 package org.elasticsearch.action.delete;
 
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.RoutingMissingException;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
+import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.action.support.replication.TransportWriteAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 /**
  * Performs the delete operation.
+ *
+ * Deprecated use TransportBulkAction with a single item instead
  */
-public class TransportDeleteAction extends TransportWriteAction<DeleteRequest, DeleteRequest,DeleteResponse> {
-
-    private final AutoCreateIndex autoCreateIndex;
-    private final TransportCreateIndexAction createIndexAction;
+@Deprecated
+public class TransportDeleteAction extends TransportSingleItemBulkWriteAction<DeleteRequest, DeleteResponse> {
 
     @Inject
     public TransportDeleteAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                  IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
-                                 TransportCreateIndexAction createIndexAction, ActionFilters actionFilters,
-                                 IndexNameExpressionResolver indexNameExpressionResolver,
-                                 AutoCreateIndex autoCreateIndex) {
-        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
-                indexNameExpressionResolver, DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX);
-        this.createIndexAction = createIndexAction;
-        this.autoCreateIndex = autoCreateIndex;
-    }
-
-    @Override
-    protected void doExecute(Task task, final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
-        ClusterState state = clusterService.state();
-        if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
-            CreateIndexRequest createIndexRequest = new CreateIndexRequest()
-                    .index(request.index())
-                    .cause("auto(delete api)")
-                    .masterNodeTimeout(request.timeout());
-            createIndexAction.execute(task, createIndexRequest, new ActionListener<CreateIndexResponse>() {
-                @Override
-                public void onResponse(CreateIndexResponse result) {
-                    innerExecute(task, request, listener);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
-                        // we have the index, do it
-                        innerExecute(task, request, listener);
-                    } else {
-                        listener.onFailure(e);
-                    }
-                }
-            });
-        } else {
-            innerExecute(task, request, listener);
-        }
-    }
-
-    @Override
-    protected void resolveRequest(final MetaData metaData, IndexMetaData indexMetaData, DeleteRequest request) {
-        super.resolveRequest(metaData, indexMetaData, request);
-        resolveAndValidateRouting(metaData, indexMetaData.getIndex().getName(), request);
-        ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(),
-            indexMetaData.getIndex().getName(), request.id(), request.routing());
-        request.setShardId(shardId);
-    }
-
-    public static void resolveAndValidateRouting(final MetaData metaData, final String concreteIndex,
-                                                 DeleteRequest request) {
-        request.routing(metaData.resolveIndexRouting(request.parent(), request.routing(), request.index()));
-        // check if routing is required, if so, throw error if routing wasn't specified
-        if (request.routing() == null && metaData.routingRequired(concreteIndex, request.type())) {
-            throw new RoutingMissingException(concreteIndex, request.type(), request.id());
-        }
-    }
-
-    private void innerExecute(Task task, final DeleteRequest request, final ActionListener<DeleteResponse> listener) {
-        super.doExecute(task, request, listener);
+                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                 TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
+        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
+            actionFilters, indexNameExpressionResolver, DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX,
+            bulkAction, shardBulkAction);
     }
 
     @Override
     protected DeleteResponse newResponseInstance() {
         return new DeleteResponse();
     }
-
-    @Override
-    protected WritePrimaryResult shardOperationOnPrimary(DeleteRequest request, IndexShard primary) throws Exception {
-        final Engine.DeleteResult result = executeDeleteRequestOnPrimary(request, primary);
-        final DeleteResponse response;
-        if (result.hasFailure() == false) {
-            // update the request with the version so it will go to the replicas
-            request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
-            request.version(result.getVersion());
-            request.seqNo(result.getSeqNo());
-            assert request.versionType().validateVersionForWrites(request.version());
-            response = new DeleteResponse(
-                primary.shardId(),
-                request.type(),
-                request.id(),
-                result.getSeqNo(),
-                result.getVersion(),
-                result.isFound());
-        } else {
-            response = null;
-        }
-        return new WritePrimaryResult(request, response, result.getTranslogLocation(), result.getFailure(), primary);
-    }
-
-    @Override
-    protected WriteReplicaResult shardOperationOnReplica(DeleteRequest request, IndexShard replica) throws Exception {
-        final Engine.DeleteResult result = executeDeleteRequestOnReplica(request, replica);
-        return new WriteReplicaResult(request, result.getTranslogLocation(), result.getFailure(), replica);
-    }
-
-
-    public static Engine.DeleteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard primary) {
-        final Engine.Delete delete = primary.prepareDeleteOnPrimary(request.type(), request.id(), request.version(), request.versionType());
-        return primary.delete(delete);
-    }
-
-    public static Engine.DeleteResult executeDeleteRequestOnReplica(DeleteRequest request, IndexShard replica) {
-        final Engine.Delete delete = replica.prepareDeleteOnReplica(request.type(), request.id(),
-                request.seqNo(), request.primaryTerm(), request.version(), request.versionType());
-        return replica.delete(delete);
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
index 9cc89095057..6b7a8cd7202 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
@@ -124,6 +124,8 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
                 return "string";
             case 4:
                 return "ip";
+            case 5:
+                return "geo_point";
             default:
                 throw new IllegalArgumentException("Unknown type.");
         }
@@ -276,7 +278,7 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         }
     }
 
-    private void updateMinMax(T min, T max) {
+    protected void updateMinMax(T min, T max) {
         if (compare(minValue, min) > 0) {
             minValue = min;
         }
@@ -327,6 +329,8 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
                 writeMinMax(out);
             }
         } else {
+            assert hasMinMax : "cannot serialize null min/max fieldstats in a mixed-cluster " +
+                    "with pre-" + Version.V_5_2_0_UNRELEASED + " nodes, remote version [" + out.getVersion() + "]";
             writeMinMax(out);
         }
     }
@@ -643,6 +647,55 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         }
     }
 
+    public static class GeoPoint extends FieldStats<org.elasticsearch.common.geo.GeoPoint> {
+        public GeoPoint(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                  boolean isSearchable, boolean isAggregatable) {
+            super((byte) 5, maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                isSearchable, isAggregatable);
+        }
+
+        public GeoPoint(long maxDoc, long docCount, long sumDocFreq, long sumTotalTermFreq,
+                        boolean isSearchable, boolean isAggregatable,
+                        org.elasticsearch.common.geo.GeoPoint minValue, org.elasticsearch.common.geo.GeoPoint maxValue) {
+            super((byte) 5, maxDoc, docCount, sumDocFreq, sumTotalTermFreq, isSearchable, isAggregatable,
+                minValue, maxValue);
+        }
+
+        @Override
+        public org.elasticsearch.common.geo.GeoPoint valueOf(String value, String fmt) {
+            return org.elasticsearch.common.geo.GeoPoint.parseFromLatLon(value);
+        }
+
+        @Override
+        protected void updateMinMax(org.elasticsearch.common.geo.GeoPoint min, org.elasticsearch.common.geo.GeoPoint max) {
+            minValue.reset(Math.min(min.lat(), minValue.lat()), Math.min(min.lon(), minValue.lon()));
+            maxValue.reset(Math.max(max.lat(), maxValue.lat()), Math.max(max.lon(), maxValue.lon()));
+        }
+
+        @Override
+        public int compare(org.elasticsearch.common.geo.GeoPoint p1, org.elasticsearch.common.geo.GeoPoint p2) {
+            throw new IllegalArgumentException("compare is not supported for geo_point field stats");
+        }
+
+        @Override
+        public void writeMinMax(StreamOutput out) throws IOException {
+            out.writeDouble(minValue.lat());
+            out.writeDouble(minValue.lon());
+            out.writeDouble(maxValue.lat());
+            out.writeDouble(maxValue.lon());
+        }
+
+        @Override
+        public String getMinValueAsString() {
+            return minValue.toString();
+        }
+
+        @Override
+        public String getMaxValueAsString() {
+            return maxValue.toString();
+        }
+    }
+
     public static FieldStats readFrom(StreamInput in) throws IOException {
         byte type = in.readByte();
         long maxDoc = in.readLong();
@@ -690,7 +743,7 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
                         isSearchable, isAggregatable);
                 }
 
-            case 4:
+            case 4: {
                 if (hasMinMax == false) {
                     return new Ip(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                         isSearchable, isAggregatable);
@@ -705,7 +758,17 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
                 InetAddress max = InetAddressPoint.decode(b2);
                 return new Ip(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
                     isSearchable, isAggregatable, min, max);
-
+            }
+            case 5: {
+                if (hasMinMax == false) {
+                    return new GeoPoint(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                        isSearchable, isAggregatable);
+                }
+                org.elasticsearch.common.geo.GeoPoint min = new org.elasticsearch.common.geo.GeoPoint(in.readDouble(), in.readDouble());
+                org.elasticsearch.common.geo.GeoPoint max = new org.elasticsearch.common.geo.GeoPoint(in.readDouble(), in.readDouble());
+                return new GeoPoint(maxDoc, docCount, sumDocFreq, sumTotalTermFreq,
+                    isSearchable, isAggregatable, min, max);
+            }
             default:
                 throw new IllegalArgumentException("Unknown type.");
         }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
index d94cfcd2958..133a94e69a4 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.fieldstats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.broadcast.BroadcastShardResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -27,6 +28,7 @@ import org.elasticsearch.index.shard.ShardId;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public class FieldStatsShardResponse extends BroadcastShardResponse {
 
@@ -44,6 +46,12 @@ public class FieldStatsShardResponse extends BroadcastShardResponse {
         return fieldStats;
     }
 
+    Map<String, FieldStats<?> > filterNullMinMax() {
+        return fieldStats.entrySet().stream()
+            .filter((e) -> e.getValue().hasMinMax())
+            .collect(Collectors.toMap(p -> p.getKey(), p -> p.getValue()));
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -59,8 +67,17 @@ public class FieldStatsShardResponse extends BroadcastShardResponse {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeVInt(fieldStats.size());
-        for (Map.Entry<String, FieldStats<?>> entry : fieldStats.entrySet()) {
+        final Map<String, FieldStats<?> > stats;
+        if (out.getVersion().before(Version.V_5_2_0_UNRELEASED)) {
+            /**
+             * FieldStats with null min/max are not (de)serializable in versions prior to {@link Version.V_5_2_0_UNRELEASED}
+             */
+            stats = filterNullMinMax();
+        } else {
+            stats = getFieldStats();
+        }
+        out.writeVInt(stats.size());
+        for (Map.Entry<String, FieldStats<?>> entry : stats.entrySet()) {
             out.writeString(entry.getKey());
             entry.getValue().writeTo(out);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
index e65f6951432..9ee72223a66 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
@@ -36,8 +36,6 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
diff --git a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
index 1b347a2d05d..296fbe6610e 100644
--- a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
@@ -25,14 +25,16 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.get.GetField;
 import org.elasticsearch.index.get.GetResult;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * The response of a get action.
@@ -40,9 +42,9 @@ import java.util.Map;
  * @see GetRequest
  * @see org.elasticsearch.client.Client#get(GetRequest)
  */
-public class GetResponse extends ActionResponse implements Iterable<GetField>, ToXContent {
+public class GetResponse extends ActionResponse implements Iterable<GetField>, ToXContentObject {
 
-    private GetResult getResult;
+    GetResult getResult;
 
     GetResponse() {
     }
@@ -156,6 +158,11 @@ public class GetResponse extends ActionResponse implements Iterable<GetField>, T
         return getResult.toXContent(builder, params);
     }
 
+    public static GetResponse fromXContent(XContentParser parser) throws IOException {
+        GetResult getResult = GetResult.fromXContent(parser);
+        return new GetResponse(getResult);
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -168,8 +175,25 @@ public class GetResponse extends ActionResponse implements Iterable<GetField>, T
         getResult.writeTo(out);
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetResponse getResponse = (GetResponse) o;
+        return Objects.equals(getResult, getResponse.getResult);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getResult);
+    }
+
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
index 5407184ded3..d4627391b11 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java
@@ -379,7 +379,8 @@ public class MultiGetRequest extends ActionRequest implements Iterable<MultiGetR
                     } else if ("_version_type".equals(currentFieldName) || "_versionType".equals(currentFieldName) || "version_type".equals(currentFieldName) || "versionType".equals(currentFieldName)) {
                         versionType = VersionType.fromString(parser.text());
                     } else if ("_source".equals(currentFieldName)) {
-                        if (parser.isBooleanValue()) {
+                        // check lenient to avoid interpreting the value as string but parse strict in order to provoke an error early on.
+                        if (parser.isBooleanValueLenient()) {
                             fetchSourceContext = new FetchSourceContext(parser.booleanValue(), fetchSourceContext.includes(),
                                 fetchSourceContext.excludes());
                         } else if (token == XContentParser.Token.VALUE_STRING) {
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index e1fe435fd10..93e4272bd95 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -24,14 +24,14 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiGetResponse extends ActionResponse implements Iterable<MultiGetItemResponse>, ToXContent {
+public class MultiGetResponse extends ActionResponse implements Iterable<MultiGetItemResponse>, ToXContentObject {
 
     /**
      * Represents a failure.
@@ -128,6 +128,7 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCS);
         for (MultiGetItemResponse response : responses) {
             if (response.isFailed()) {
@@ -136,16 +137,15 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderException(builder, params, failure.getFailure());
+                ElasticsearchException.generateFailureXContent(builder, params, failure.getFailure(), true);
                 builder.endObject();
             } else {
                 GetResponse getResponse = response.getResponse();
-                builder.startObject();
                 getResponse.toXContent(builder, params);
-                builder.endObject();
             }
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
@@ -154,9 +154,6 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
         static final String _INDEX = "_index";
         static final String _TYPE = "_type";
         static final String _ID = "_id";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
-
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 6b9de7ecf64..ee835fa06be 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -71,7 +71,7 @@ public class TransportGetAction extends TransportSingleShardAction<GetRequest, G
         if (request.request().realtime && // if the realtime flag is set
                 request.request().preference() == null && // the preference flag is not already set
                 indexMeta != null && // and we have the index
-                IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings())) { // and the index uses shadow replicas
+                indexMeta.isIndexUsingShadowReplicas()) { // and the index uses shadow replicas
             // set the preference for the request to use "_primary" automatically
             request.request().preference(Preference.PRIMARY.type());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index de80f85b89f..7e3f14368de 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.index;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
@@ -46,6 +47,7 @@ import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -54,10 +56,10 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * created using {@link org.elasticsearch.client.Requests#indexRequest(String)}.
  *
  * The index requires the {@link #index()}, {@link #type(String)}, {@link #id(String)} and
- * {@link #source(byte[])} to be set.
+ * {@link #source(byte[], XContentType)} to be set.
  *
- * The source (content to index) can be set in its bytes form using ({@link #source(byte[])}),
- * its string form ({@link #source(String)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder}
+ * The source (content to index) can be set in its bytes form using ({@link #source(byte[], XContentType)}),
+ * its string form ({@link #source(String, XContentType)}) or using a {@link org.elasticsearch.common.xcontent.XContentBuilder}
  * ({@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}).
  *
  * If the {@link #id(String)} is not set, it will be automatically generated.
@@ -66,7 +68,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * @see org.elasticsearch.client.Requests#indexRequest(String)
  * @see org.elasticsearch.client.Client#index(IndexRequest)
  */
-public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocWriteRequest<IndexRequest> {
+public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocWriteRequest<IndexRequest>, CompositeIndicesRequest {
 
     private String type;
     private String id;
@@ -82,7 +84,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
 
-    private XContentType contentType = Requests.INDEX_CONTENT_TYPE;
+    private XContentType contentType;
 
     private String pipeline;
 
@@ -102,7 +104,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
 
     /**
      * Constructs a new index request against the specific index. The {@link #type(String)}
-     * {@link #source(byte[])} must be set.
+     * {@link #source(byte[], XContentType)} must be set.
      */
     public IndexRequest(String index) {
         this.index = index;
@@ -139,7 +141,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         if (source == null) {
             validationException = addValidationError("source is missing", validationException);
         }
-
+        if (contentType == null) {
+            validationException = addValidationError("content type is missing", validationException);
+        }
         final long resolvedVersion = resolveVersionDefaults();
         if (opType() == OpType.CREATE) {
             if (versionType != VersionType.INTERNAL) {
@@ -178,20 +182,13 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     }
 
     /**
-     * The content type that will be used when generating a document from user provided objects like Maps.
+     * The content type. This will be used when generating a document from user provided objects like Maps and when parsing the
+     * source at index time
      */
     public XContentType getContentType() {
         return contentType;
     }
 
-    /**
-     * Sets the content type that will be used when generating a document from user provided objects (like Map).
-     */
-    public IndexRequest contentType(XContentType contentType) {
-        this.contentType = contentType;
-        return this;
-    }
-
     /**
      * The type of the indexed document.
      */
@@ -283,16 +280,16 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     }
 
     public Map<String, Object> sourceAsMap() {
-        return XContentHelper.convertToMap(source, false).v2();
+        return XContentHelper.convertToMap(source, false, contentType).v2();
     }
 
     /**
-     * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}.
+     * Index the Map in {@link Requests#INDEX_CONTENT_TYPE} format
      *
      * @param source The map to index
      */
     public IndexRequest source(Map source) throws ElasticsearchGenerationException {
-        return source(source, contentType);
+        return source(source, Requests.INDEX_CONTENT_TYPE);
     }
 
     /**
@@ -313,24 +310,32 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
     /**
      * Sets the document source to index.
      *
-     * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}
-     * or using the {@link #source(byte[])}.
+     * @deprecated use {@link #source(String, XContentType)}
      */
+    @Deprecated
     public IndexRequest source(String source) {
-        this.source = new BytesArray(source.getBytes(StandardCharsets.UTF_8));
-        return this;
+        return source(new BytesArray(source), XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Sets the document source to index.
+     *
+     * Note, its preferable to either set it using {@link #source(org.elasticsearch.common.xcontent.XContentBuilder)}
+     * or using the {@link #source(byte[], XContentType)}.
+     */
+    public IndexRequest source(String source, XContentType xContentType) {
+        return source(new BytesArray(source), xContentType);
     }
 
     /**
      * Sets the content source to index.
      */
     public IndexRequest source(XContentBuilder sourceBuilder) {
-        source = sourceBuilder.bytes();
-        return this;
+        return source(sourceBuilder.bytes(), sourceBuilder.contentType());
     }
 
     /**
-     * Sets the content source to index.
+     * Sets the content source to index using the default content type ({@link Requests#INDEX_CONTENT_TYPE})
      * <p>
      * <b>Note: the number of objects passed to this method must be an even
      * number. Also the first argument in each pair (the field name) must have a
@@ -338,6 +343,18 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
      * </p>
      */
     public IndexRequest source(Object... source) {
+        return source(Requests.INDEX_CONTENT_TYPE, source);
+    }
+
+    /**
+     * Sets the content source to index.
+     * <p>
+     * <b>Note: the number of objects passed to this method as varargs must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
+     */
+    public IndexRequest source(XContentType xContentType, Object... source) {
         if (source.length % 2 != 0) {
             throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]");
         }
@@ -345,7 +362,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
             throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag was removed, please just use source(BytesReference)");
         }
         try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
             builder.startObject();
             for (int i = 0; i < source.length; i++) {
                 builder.field(source[i++].toString(), source[i]);
@@ -359,19 +376,53 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
 
     /**
      * Sets the document to index in bytes form.
+     * @deprecated use {@link #source(BytesReference, XContentType)}
      */
+    @Deprecated
     public IndexRequest source(BytesReference source) {
-        this.source = source;
-        return this;
+        return source(source, XContentFactory.xContentType(source));
+
     }
 
     /**
      * Sets the document to index in bytes form.
      */
+    public IndexRequest source(BytesReference source, XContentType xContentType) {
+        this.source = Objects.requireNonNull(source);
+        this.contentType = Objects.requireNonNull(xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the document to index in bytes form.
+     * @deprecated use {@link #source(byte[], XContentType)}
+     */
+    @Deprecated
     public IndexRequest source(byte[] source) {
         return source(source, 0, source.length);
     }
 
+    /**
+     * Sets the document to index in bytes form.
+     */
+    public IndexRequest source(byte[] source, XContentType xContentType) {
+        return source(source, 0, source.length, xContentType);
+    }
+
+    /**
+     * Sets the document to index in bytes form (assumed to be safe to be used from different
+     * threads).
+     *
+     * @param source The source to index
+     * @param offset The offset in the byte array
+     * @param length The length of the data
+     * @deprecated use {@link #source(byte[], int, int, XContentType)}
+     */
+    @Deprecated
+    public IndexRequest source(byte[] source, int offset, int length) {
+        return source(new BytesArray(source, offset, length), XContentFactory.xContentType(source));
+    }
+
     /**
      * Sets the document to index in bytes form (assumed to be safe to be used from different
      * threads).
@@ -380,9 +431,8 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
      * @param offset The offset in the byte array
      * @param length The length of the data
      */
-    public IndexRequest source(byte[] source, int offset, int length) {
-        this.source = new BytesArray(source, offset, length);
-        return this;
+    public IndexRequest source(byte[] source, int offset, int length, XContentType xContentType) {
+        return source(new BytesArray(source, offset, length), xContentType);
     }
 
     /**
@@ -514,6 +564,11 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         pipeline = in.readOptionalString();
         isRetry = in.readBoolean();
         autoGeneratedTimestamp = in.readLong();
+        if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            contentType = in.readOptionalWriteable(XContentType::readFrom);
+        } else {
+            contentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -524,7 +579,10 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         out.writeOptionalString(routing);
         out.writeOptionalString(parent);
         if (out.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
-            out.writeOptionalString(null);
+            // Serialize a fake timestamp. 5.x expect this value to be set by the #process method so we can't use null.
+            // On the other hand, indices created on 5.x do not index the timestamp field.  Therefore passing a 0 (or any value) for
+            // the transport layer OK as it will be ignored.
+            out.writeOptionalString("0");
             out.writeOptionalWriteable(null);
         }
         out.writeBytesReference(source);
@@ -539,6 +597,9 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         out.writeOptionalString(pipeline);
         out.writeBoolean(isRetry);
         out.writeLong(autoGeneratedTimestamp);
+        if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            out.writeOptionalWriteable(contentType);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
index f7df8bffced..7af43ec35ec 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
@@ -82,12 +82,22 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
 
     /**
      * Sets the source.
+     * @deprecated use {@link #setSource(BytesReference, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(BytesReference source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the source.
+     */
+    public IndexRequestBuilder setSource(BytesReference source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Index the Map as a JSON.
      *
@@ -112,13 +122,26 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
      * Sets the document source to index.
      * <p>
      * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)}
-     * or using the {@link #setSource(byte[])}.
+     * or using the {@link #setSource(byte[], XContentType)}.
+     * @deprecated use {@link #setSource(String, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(String source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the document source to index.
+     * <p>
+     * Note, its preferable to either set it using {@link #setSource(org.elasticsearch.common.xcontent.XContentBuilder)}
+     * or using the {@link #setSource(byte[], XContentType)}.
+     */
+    public IndexRequestBuilder setSource(String source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the content source to index.
      */
@@ -129,12 +152,22 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
 
     /**
      * Sets the document to index in bytes form.
+     * @deprecated use {@link #setSource(byte[], XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(byte[] source) {
         request.source(source);
         return this;
     }
 
+    /**
+     * Sets the document to index in bytes form.
+     */
+    public IndexRequestBuilder setSource(byte[] source, XContentType xContentType) {
+        request.source(source, xContentType);
+        return this;
+    }
+
     /**
      * Sets the document to index in bytes form (assumed to be safe to be used from different
      * threads).
@@ -142,12 +175,28 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
      * @param source The source to index
      * @param offset The offset in the byte array
      * @param length The length of the data
+     * @deprecated use {@link #setSource(byte[], int, int, XContentType)}
      */
+    @Deprecated
     public IndexRequestBuilder setSource(byte[] source, int offset, int length) {
         request.source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the document to index in bytes form (assumed to be safe to be used from different
+     * threads).
+     *
+     * @param source The source to index
+     * @param offset The offset in the byte array
+     * @param length The length of the data
+     * @param xContentType The type/format of the source
+     */
+    public IndexRequestBuilder setSource(byte[] source, int offset, int length, XContentType xContentType) {
+        request.source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Constructs a simple document with a field name and value pairs.
      * <p>
@@ -162,10 +211,15 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
     }
 
     /**
-     * The content type that will be used to generate a document from user provided objects (like Map).
+     * Constructs a simple document with a field name and value pairs.
+     * <p>
+     * <b>Note: the number of objects passed as varargs to this method must be an even
+     * number. Also the first argument in each pair (the field name) must have a
+     * valid String representation.</b>
+     * </p>
      */
-    public IndexRequestBuilder setContentType(XContentType contentType) {
-        request.contentType(contentType);
+    public IndexRequestBuilder setSource(XContentType xContentType, Object... source) {
+        request.source(xContentType, source);
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
index b092e7e8e74..d2d367152f4 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexResponse.java
@@ -20,13 +20,21 @@
 package org.elasticsearch.action.index;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+
 /**
  * A response of an index operation,
  *
@@ -35,6 +43,8 @@ import java.io.IOException;
  */
 public class IndexResponse extends DocWriteResponse {
 
+    private static final String CREATED = "created";
+
     public IndexResponse() {
     }
 
@@ -57,14 +67,45 @@ public class IndexResponse extends DocWriteResponse {
         builder.append(",version=").append(getVersion());
         builder.append(",result=").append(getResult().getLowercase());
         builder.append(",seqNo=").append(getSeqNo());
-        builder.append(",shards=").append(Strings.toString(getShardInfo(), true));
+        builder.append(",shards=").append(Strings.toString(getShardInfo()));
         return builder.append("]").toString();
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        super.toXContent(builder, params);
-        builder.field("created", result == Result.CREATED);
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerToXContent(builder, params);
+        builder.field(CREATED, result == Result.CREATED);
         return builder;
     }
+
+    /**
+     * ConstructingObjectParser used to parse the {@link IndexResponse}. We use a ObjectParser here
+     * because most fields are parsed by the parent abstract class {@link DocWriteResponse} and it's
+     * not easy to parse part of the fields in the parent class and other fields in the children class
+     * using the usual streamed parsing method.
+     */
+    private static final ConstructingObjectParser<IndexResponse, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(IndexResponse.class.getName(),
+                args -> {
+                    // index uuid and shard id are unknown and can't be parsed back for now.
+                    ShardId shardId = new ShardId(new Index((String) args[0], IndexMetaData.INDEX_UUID_NA_VALUE), -1);
+                    String type = (String) args[1];
+                    String id = (String) args[2];
+                    long version = (long) args[3];
+                    ShardInfo shardInfo = (ShardInfo) args[5];
+                    long seqNo = (args[6] != null) ? (long) args[6] : SequenceNumbersService.UNASSIGNED_SEQ_NO;
+                    boolean created = (boolean) args[7];
+
+                    IndexResponse indexResponse = new IndexResponse(shardId, type, id, seqNo, version, created);
+                    indexResponse.setShardInfo(shardInfo);
+                    return indexResponse;
+                });
+        DocWriteResponse.declareParserFields(PARSER);
+        PARSER.declareBoolean(constructorArg(), new ParseField(CREATED));
+    }
+
+    public static IndexResponse fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index 70220679752..88a210c7180 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -19,39 +19,16 @@
 
 package org.elasticsearch.action.index;
 
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ResourceAlreadyExistsException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
-import org.elasticsearch.action.ingest.IngestActionForwarder;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
+import org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.action.support.replication.ReplicationOperation;
-import org.elasticsearch.action.support.replication.TransportWriteAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.Mapping;
-import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -64,202 +41,25 @@ import org.elasticsearch.transport.TransportService;
  * Defaults to <tt>true</tt>.
  * <li><b>allowIdGeneration</b>: If the id is set not, should it be generated. Defaults to <tt>true</tt>.
  * </ul>
+ *
+ * Deprecated use TransportBulkAction with a single item instead
  */
-public class TransportIndexAction extends TransportWriteAction<IndexRequest, IndexRequest, IndexResponse> {
-
-    private final AutoCreateIndex autoCreateIndex;
-    private final boolean allowIdGeneration;
-    private final TransportCreateIndexAction createIndexAction;
-
-    private final ClusterService clusterService;
-    private final IngestService ingestService;
-    private final MappingUpdatedAction mappingUpdatedAction;
-    private final IngestActionForwarder ingestForwarder;
+@Deprecated
+public class TransportIndexAction extends TransportSingleItemBulkWriteAction<IndexRequest, IndexResponse> {
 
     @Inject
     public TransportIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                IndicesService indicesService, IngestService ingestService, ThreadPool threadPool,
-                                ShardStateAction shardStateAction, TransportCreateIndexAction createIndexAction,
-                                MappingUpdatedAction mappingUpdatedAction, ActionFilters actionFilters,
-                                IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex) {
+                                IndicesService indicesService,
+                                ThreadPool threadPool, ShardStateAction shardStateAction,
+                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                TransportBulkAction bulkAction, TransportShardBulkAction shardBulkAction) {
         super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX);
-        this.mappingUpdatedAction = mappingUpdatedAction;
-        this.createIndexAction = createIndexAction;
-        this.autoCreateIndex = autoCreateIndex;
-        this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
-        this.clusterService = clusterService;
-        this.ingestService = ingestService;
-        this.ingestForwarder = new IngestActionForwarder(transportService);
-        clusterService.addStateApplier(this.ingestForwarder);
-    }
-
-    @Override
-    protected void doExecute(Task task, final IndexRequest request, final ActionListener<IndexResponse> listener) {
-        if (Strings.hasText(request.getPipeline())) {
-            if (clusterService.localNode().isIngestNode()) {
-                processIngestIndexRequest(task, request, listener);
-            } else {
-                ingestForwarder.forwardIngestRequest(IndexAction.INSTANCE, request, listener);
-            }
-            return;
-        }
-        // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API
-        ClusterState state = clusterService.state();
-        if (shouldAutoCreate(request, state)) {
-            CreateIndexRequest createIndexRequest = new CreateIndexRequest();
-            createIndexRequest.index(request.index());
-            createIndexRequest.cause("auto(index api)");
-            createIndexRequest.masterNodeTimeout(request.timeout());
-            createIndexAction.execute(task, createIndexRequest, new ActionListener<CreateIndexResponse>() {
-                @Override
-                public void onResponse(CreateIndexResponse result) {
-                    innerExecute(task, request, listener);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
-                        // we have the index, do it
-                        try {
-                            innerExecute(task, request, listener);
-                        } catch (Exception inner) {
-                            inner.addSuppressed(e);
-                            listener.onFailure(inner);
-                        }
-                    } else {
-                        listener.onFailure(e);
-                    }
-                }
-            });
-        } else {
-            innerExecute(task, request, listener);
-        }
-    }
-
-    protected boolean shouldAutoCreate(IndexRequest request, ClusterState state) {
-        return autoCreateIndex.shouldAutoCreate(request.index(), state);
-    }
-
-    @Override
-    protected void resolveRequest(MetaData metaData, IndexMetaData indexMetaData, IndexRequest request) {
-        super.resolveRequest(metaData, indexMetaData, request);
-        MappingMetaData mappingMd =indexMetaData.mappingOrDefault(request.type());
-        request.resolveRouting(metaData);
-        request.process(mappingMd, allowIdGeneration, indexMetaData.getIndex().getName());
-        ShardId shardId = clusterService.operationRouting().shardId(clusterService.state(),
-            indexMetaData.getIndex().getName(), request.id(), request.routing());
-        request.setShardId(shardId);
-    }
-
-    protected void innerExecute(Task task, final IndexRequest request, final ActionListener<IndexResponse> listener) {
-        super.doExecute(task, request, listener);
+            actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX,
+            bulkAction, shardBulkAction);
     }
 
     @Override
     protected IndexResponse newResponseInstance() {
         return new IndexResponse();
     }
-
-    @Override
-    protected WritePrimaryResult shardOperationOnPrimary(IndexRequest request, IndexShard primary) throws Exception {
-        final Engine.IndexResult indexResult = executeIndexRequestOnPrimary(request, primary, mappingUpdatedAction);
-        final IndexResponse response;
-        if (indexResult.hasFailure() == false) {
-            // update the version on request so it will happen on the replicas
-            final long version = indexResult.getVersion();
-            request.version(version);
-            request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
-            request.seqNo(indexResult.getSeqNo());
-            assert request.versionType().validateVersionForWrites(request.version());
-            response = new IndexResponse(primary.shardId(), request.type(), request.id(), indexResult.getSeqNo(),
-                    indexResult.getVersion(), indexResult.isCreated());
-        } else {
-            response = null;
-        }
-        return new WritePrimaryResult(request, response, indexResult.getTranslogLocation(), indexResult.getFailure(), primary);
-    }
-
-    @Override
-    protected WriteReplicaResult shardOperationOnReplica(IndexRequest request, IndexShard replica) throws Exception {
-        final Engine.IndexResult indexResult = executeIndexRequestOnReplica(request, replica);
-        return new WriteReplicaResult(request, indexResult.getTranslogLocation(), indexResult.getFailure(), replica);
-    }
-
-    /**
-     * Execute the given {@link IndexRequest} on a replica shard, throwing a
-     * {@link RetryOnReplicaException} if the operation needs to be re-tried.
-     */
-    public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) {
-        final ShardId shardId = replica.shardId();
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source())
-                .routing(request.routing()).parent(request.parent());
-
-        final Engine.Index operation;
-        try {
-            operation = replica.prepareIndexOnReplica(sourceToParse, request.seqNo(), request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
-        } catch (MapperParsingException e) {
-            return new Engine.IndexResult(e, request.version(), request.seqNo());
-        }
-        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
-        if (update != null) {
-            throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update);
-        }
-        return replica.index(operation);
-    }
-
-    /** Utility method to prepare an index operation on primary shards */
-    static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) {
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source())
-            .routing(request.routing()).parent(request.parent());
-        return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry());
-    }
-
-    public static Engine.IndexResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard primary,
-            MappingUpdatedAction mappingUpdatedAction) throws Exception {
-        Engine.Index operation;
-        try {
-            operation = prepareIndexOperationOnPrimary(request, primary);
-        } catch (MapperParsingException | IllegalArgumentException e) {
-            return new Engine.IndexResult(e, request.version(), request.seqNo());
-        }
-        Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
-        final ShardId shardId = primary.shardId();
-        if (update != null) {
-            // can throw timeout exception when updating mappings or ISE for attempting to update default mappings
-            // which are bubbled up
-            try {
-                mappingUpdatedAction.updateMappingOnMaster(shardId.getIndex(), request.type(), update);
-            } catch (IllegalArgumentException e) {
-                // throws IAE on conflicts merging dynamic mappings
-                return new Engine.IndexResult(e, request.version(), request.seqNo());
-            }
-            try {
-                operation = prepareIndexOperationOnPrimary(request, primary);
-            } catch (MapperParsingException | IllegalArgumentException e) {
-                return new Engine.IndexResult(e, request.version(), request.seqNo());
-            }
-            update = operation.parsedDoc().dynamicMappingsUpdate();
-            if (update != null) {
-                throw new ReplicationOperation.RetryOnPrimaryException(shardId,
-                    "Dynamic mappings are not available on the node that holds the primary yet");
-            }
-        }
-
-        return primary.index(operation);
-    }
-
-    private void processIngestIndexRequest(Task task, IndexRequest indexRequest, ActionListener listener) {
-        ingestService.getPipelineExecutionService().executeIndexRequest(indexRequest, t -> {
-            logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to execute pipeline [{}]", indexRequest.getPipeline()), t);
-            listener.onFailure(t);
-        }, success -> {
-            // TransportIndexAction uses IndexRequest and same action name on the node that receives the request and the node that
-            // processes the primary action. This could lead to a pipeline being executed twice for the same
-            // index request, hence we set the pipeline to null once its execution completed.
-            indexRequest.setPipeline(null);
-            doExecute(task, indexRequest, listener);
-        });
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
index 74ce894b053..45cb83634f8 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/DeletePipelineTransportAction.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
index f603a354f4b..30843bdff9b 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.rest.RestStatus;
@@ -31,7 +31,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-public class GetPipelineResponse extends ActionResponse implements StatusToXContent {
+public class GetPipelineResponse extends ActionResponse implements StatusToXContentObject {
 
     private List<PipelineConfiguration> pipelines;
 
@@ -52,7 +52,7 @@ public class GetPipelineResponse extends ActionResponse implements StatusToXCont
         int size = in.readVInt();
         pipelines = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            pipelines.add(PipelineConfiguration.readPipelineConfiguration(in));
+            pipelines.add(PipelineConfiguration.readFrom(in));
         }
     }
 
@@ -76,9 +76,11 @@ public class GetPipelineResponse extends ActionResponse implements StatusToXCont
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         for (PipelineConfiguration pipeline : pipelines) {
             builder.field(pipeline.getId(), pipeline.getConfigAsMap());
         }
+        builder.endObject();
         return builder;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
index 8bac5c7b804..f64b36d47ae 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineTransportAction.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
index 10416146ba8..a28950b24c1 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequest.java
@@ -19,32 +19,40 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Objects;
 
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
 public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest> {
 
     private String id;
     private BytesReference source;
+    private XContentType xContentType;
 
+    /**
+     * Create a new pipeline request
+     * @deprecated use {@link #PutPipelineRequest(String, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     public PutPipelineRequest(String id, BytesReference source) {
-        if (id == null) {
-            throw new IllegalArgumentException("id is missing");
-        }
-        if (source == null) {
-            throw new IllegalArgumentException("source is missing");
-        }
+        this(id, source, XContentFactory.xContentType(source));
+    }
 
-        this.id = id;
-        this.source = source;
+    /**
+     * Create a new pipeline request with the id and source along with the content type of the source
+     */
+    public PutPipelineRequest(String id, BytesReference source, XContentType xContentType) {
+        this.id = Objects.requireNonNull(id);
+        this.source = Objects.requireNonNull(source);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     PutPipelineRequest() {
@@ -63,11 +71,20 @@ public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest>
         return source;
     }
 
+    public XContentType getXContentType() {
+        return xContentType;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         id = in.readString();
         source = in.readBytesReference();
+        if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -75,5 +92,8 @@ public class PutPipelineRequest extends AcknowledgedRequest<PutPipelineRequest>
         super.writeTo(out);
         out.writeString(id);
         out.writeBytesReference(source);
+        if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType.writeTo(out);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
index bd927115fb5..c03b3b84f8b 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineRequestBuilder.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineRequest, WritePipelineResponse, PutPipelineRequestBuilder> {
 
@@ -29,8 +30,13 @@ public class PutPipelineRequestBuilder extends ActionRequestBuilder<PutPipelineR
         super(client, action, new PutPipelineRequest());
     }
 
+    @Deprecated
     public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source) {
         super(client, action, new PutPipelineRequest(id, source));
     }
 
+    public PutPipelineRequestBuilder(ElasticsearchClient client, PutPipelineAction action, String id, BytesReference source,
+                                     XContentType xContentType) {
+        super(client, action, new PutPipelineRequest(id, source, xContentType));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index 82cd8d8eb7b..7dde9818049 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.IngestInfo;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
index 82b39ac8972..c6252feea27 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
@@ -84,7 +84,7 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult
         if (failure == null) {
             ingestDocument.toXContent(builder, params);
         } else {
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index ef7b5e3d5bb..362d2e7ab98 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Pipeline;
@@ -34,6 +37,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.ingest.IngestDocument.MetaData;
 
@@ -42,12 +46,23 @@ public class SimulatePipelineRequest extends ActionRequest {
     private String id;
     private boolean verbose;
     private BytesReference source;
+    private XContentType xContentType;
 
+    /**
+     * Create a new request
+     * @deprecated use {@link #SimulatePipelineRequest(BytesReference, XContentType)} that does not attempt content autodetection
+     */
+    @Deprecated
     public SimulatePipelineRequest(BytesReference source) {
-        if (source == null) {
-            throw new IllegalArgumentException("source is missing");
-        }
-        this.source = source;
+        this(source, XContentFactory.xContentType(source));
+    }
+
+    /**
+     * Creates a new request with the given source and its content type
+     */
+    public SimulatePipelineRequest(BytesReference source, XContentType xContentType) {
+        this.source = Objects.requireNonNull(source);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     SimulatePipelineRequest() {
@@ -78,12 +93,21 @@ public class SimulatePipelineRequest extends ActionRequest {
         return source;
     }
 
+    public XContentType getXContentType() {
+        return xContentType;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         id = in.readOptionalString();
         verbose = in.readBoolean();
         source = in.readBytesReference();
+        if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType = XContentType.readFrom(in);
+        } else {
+            xContentType = XContentFactory.xContentType(source);
+        }
     }
 
     @Override
@@ -92,6 +116,9 @@ public class SimulatePipelineRequest extends ActionRequest {
         out.writeOptionalString(id);
         out.writeBoolean(verbose);
         out.writeBytesReference(source);
+        if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType.writeTo(out);
+        }
     }
 
     public static final class Fields {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
index 4a13fa111e6..bb5d0e4e400 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequestBuilder.java
@@ -22,22 +22,46 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class SimulatePipelineRequestBuilder extends ActionRequestBuilder<SimulatePipelineRequest, SimulatePipelineResponse, SimulatePipelineRequestBuilder> {
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     */
     public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action) {
         super(client, action, new SimulatePipelineRequest());
     }
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     * @deprecated use {@link #SimulatePipelineRequestBuilder(ElasticsearchClient, SimulatePipelineAction, BytesReference, XContentType)} to
+     *             avoid content type auto-detection on the source bytes
+     */
+    @Deprecated
     public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source) {
         super(client, action, new SimulatePipelineRequest(source));
     }
 
+    /**
+     * Create a new builder for {@link SimulatePipelineRequest}s
+     */
+    public SimulatePipelineRequestBuilder(ElasticsearchClient client, SimulatePipelineAction action, BytesReference source,
+                                          XContentType xContentType) {
+        super(client, action, new SimulatePipelineRequest(source, xContentType));
+    }
+
+    /**
+     * Set the id for the pipeline to simulate
+     */
     public SimulatePipelineRequestBuilder setId(String id) {
         request.setId(id);
         return this;
     }
 
+    /**
+     * Enable or disable verbose mode
+     */
     public SimulatePipelineRequestBuilder setVerbose(boolean verbose) {
         request.setVerbose(verbose);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
index 83029a1aab5..e9ea1a77507 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
@@ -30,7 +30,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-public class SimulatePipelineResponse extends ActionResponse implements ToXContent {
+public class SimulatePipelineResponse extends ActionResponse implements ToXContentObject {
     private String pipelineId;
     private boolean verbose;
     private List<SimulateDocumentResult> results;
@@ -88,11 +88,13 @@ public class SimulatePipelineResponse extends ActionResponse implements ToXConte
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCUMENTS);
         for (SimulateDocumentResult response : results) {
             response.toXContent(builder, params);
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
index 4f9a219c8ad..3f67007df69 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -28,7 +27,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -48,7 +47,7 @@ public class SimulatePipelineTransportAction extends HandledTransportAction<Simu
 
     @Override
     protected void doExecute(SimulatePipelineRequest request, ActionListener<SimulatePipelineResponse> listener) {
-        final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false).v2();
+        final Map<String, Object> source = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
 
         final SimulatePipelineRequest.Parsed simulateRequest;
         try {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index c978cc56d9e..3ebcb6cb6f3 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -99,10 +98,10 @@ class SimulateProcessorResult implements Writeable, ToXContent {
 
         if (failure != null && ingestDocument != null) {
             builder.startObject("ignored_error");
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
             builder.endObject();
         } else if (failure != null) {
-            ElasticsearchException.renderException(builder, params, failure);
+            ElasticsearchException.generateFailureXContent(builder, params, failure, true);
         }
 
         if (ingestDocument != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 2479ff86750..32585c62384 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -25,33 +25,41 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.FetchSearchResultProvider;
 import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.QuerySearchResultProvider;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.Transport;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
+import java.util.function.IntConsumer;
 
 
 abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult> extends AbstractAsyncAction {
     private static final float DEFAULT_INDEX_BOOST = 1.0f;
-
     protected final Logger logger;
     protected final SearchTransportService searchTransportService;
     private final Executor executor;
@@ -59,39 +67,41 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
     private final GroupShardsIterator shardsIts;
     protected final SearchRequest request;
     /** Used by subclasses to resolve node ids to DiscoveryNodes. **/
-    protected final Function<String, DiscoveryNode> nodeIdToDiscoveryNode;
+    protected final Function<String, Transport.Connection> nodeIdToConnection;
+    protected final SearchPhaseController searchPhaseController;
     protected final SearchTask task;
-    protected final int expectedSuccessfulOps;
+    private final int expectedSuccessfulOps;
     private final int expectedTotalOps;
-    protected final AtomicInteger successfulOps = new AtomicInteger();
+    private final AtomicInteger successfulOps = new AtomicInteger();
     private final AtomicInteger totalOps = new AtomicInteger();
-    protected final AtomicArray<FirstResult> firstResults;
+    private final AtomicArray<FirstResult> initialResults;
     private final Map<String, AliasFilter> aliasFilter;
     private final Map<String, Float> concreteIndexBoosts;
     private final long clusterStateVersion;
     private volatile AtomicArray<ShardSearchFailure> shardFailures;
     private final Object shardFailuresMutex = new Object();
-    protected volatile ScoreDoc[] sortedShardDocs;
 
     protected AbstractSearchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                        Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
+                                        Function<String, Transport.Connection> nodeIdToConnection,
                                         Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                                        Executor executor, SearchRequest request, ActionListener<SearchResponse> listener,
-                                        GroupShardsIterator shardsIts, long startTime, long clusterStateVersion, SearchTask task) {
+                                        SearchPhaseController searchPhaseController, Executor executor, SearchRequest request,
+                                        ActionListener<SearchResponse> listener, GroupShardsIterator shardsIts, long startTime,
+                                        long clusterStateVersion, SearchTask task) {
         super(startTime);
         this.logger = logger;
+        this.searchPhaseController = searchPhaseController;
         this.searchTransportService = searchTransportService;
         this.executor = executor;
         this.request = request;
         this.task = task;
         this.listener = listener;
-        this.nodeIdToDiscoveryNode = nodeIdToDiscoveryNode;
+        this.nodeIdToConnection = nodeIdToConnection;
         this.clusterStateVersion = clusterStateVersion;
         this.shardsIts = shardsIts;
         expectedSuccessfulOps = shardsIts.size();
         // we need to add 1 for non active partition, since we count it in the total!
         expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
-        firstResults = new AtomicArray<>(shardsIts.size());
+        initialResults = new AtomicArray<>(shardsIts.size());
         this.aliasFilter = aliasFilter;
         this.concreteIndexBoosts = concreteIndexBoosts;
     }
@@ -109,45 +119,49 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
             shardIndex++;
             final ShardRouting shard = shardIt.nextOrNull();
             if (shard != null) {
-                performFirstPhase(shardIndex, shardIt, shard);
+                performInitialPhase(shardIndex, shardIt, shard);
             } else {
                 // really, no shards active in this group
-                onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
+                onInitialPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
             }
         }
     }
 
-    void performFirstPhase(final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
+    void performInitialPhase(final int shardIndex, final ShardIterator shardIt, final ShardRouting shard) {
         if (shard == null) {
+            // TODO upgrade this to an assert...
             // no more active shards... (we should not really get here, but just for safety)
-            onFirstPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
+            onInitialPhaseResult(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
         } else {
-            final DiscoveryNode node = nodeIdToDiscoveryNode.apply(shard.currentNodeId());
-            if (node == null) {
-                onFirstPhaseResult(shardIndex, shard, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
-            } else {
+            try {
+                final Transport.Connection connection = nodeIdToConnection.apply(shard.currentNodeId());
                 AliasFilter filter = this.aliasFilter.get(shard.index().getUUID());
                 assert filter != null;
 
                 float indexBoost = concreteIndexBoosts.getOrDefault(shard.index().getUUID(), DEFAULT_INDEX_BOOST);
                 ShardSearchTransportRequest transportRequest = new ShardSearchTransportRequest(request, shardIt.shardId(), shardsIts.size(),
                     filter, indexBoost, startTime());
-                sendExecuteFirstPhase(node, transportRequest , new ActionListener<FirstResult>() {
-                        @Override
-                        public void onResponse(FirstResult result) {
-                            onFirstPhaseResult(shardIndex, shard.currentNodeId(), result, shardIt);
-                        }
+                sendExecuteFirstPhase(connection, transportRequest, new ActionListener<FirstResult>() {
+                    @Override
+                    public void onResponse(FirstResult result) {
+                        onInitialPhaseResult(shardIndex, shard.currentNodeId(), result, shardIt);
+                    }
 
-                        @Override
-                        public void onFailure(Exception t) {
-                            onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
-                        }
-                    });
+                    @Override
+                    public void onFailure(Exception t) {
+                        onInitialPhaseResult(shardIndex, shard, connection.getNode().getId(), shardIt, t);
+                    }
+                });
+            } catch (ConnectTransportException | IllegalArgumentException ex) {
+                // we are getting the connection early here so we might run into nodes that are not connected. in that case we move on to
+                // the next shard. previously when using discovery nodes here we had a special case for null when a node was not connected
+                // at all which is not not needed anymore.
+                onInitialPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, ex);
             }
         }
     }
 
-    private void onFirstPhaseResult(int shardIndex, String nodeId, FirstResult result, ShardIterator shardIt) {
+    private void onInitialPhaseResult(int shardIndex, String nodeId, FirstResult result, ShardIterator shardIt) {
         result.shardTarget(new SearchShardTarget(nodeId, shardIt.shardId()));
         processFirstPhaseResult(shardIndex, result);
         // we need to increment successful ops first before we compare the exit condition otherwise if we
@@ -158,27 +172,32 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         // and when that happens, we break on total ops, so we must maintain them
         final int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
         if (xTotalOps == expectedTotalOps) {
-            try {
-                innerMoveToSecondPhase();
-            } catch (Exception e) {
-                if (logger.isDebugEnabled()) {
-                    logger.debug(
-                        (Supplier<?>) () -> new ParameterizedMessage(
-                            "{}: Failed to execute [{}] while moving to second phase",
-                            shardIt.shardId(),
-                            request),
-                        e);
-                }
-                raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
-            }
+            executePhase(initialPhaseName(), innerGetNextPhase(), null);
         } else if (xTotalOps > expectedTotalOps) {
             raiseEarlyFailure(new IllegalStateException("unexpected higher total ops [" + xTotalOps + "] compared " +
                 "to expected [" + expectedTotalOps + "]"));
         }
     }
 
-    private void onFirstPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,
-                            final ShardIterator shardIt, Exception e) {
+    protected void executePhase(String phaseName, CheckedRunnable<Exception> phase, Exception suppressedException) {
+        try {
+            phase.run();
+        } catch (Exception e) {
+            if (suppressedException != null) {
+                e.addSuppressed(suppressedException);
+            }
+            if (logger.isDebugEnabled()) {
+                logger.debug(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "Failed to execute [{}] while moving to second phase", request),
+                    e);
+            }
+            raiseEarlyFailure(new ReduceSearchPhaseException(phaseName, "", e, buildShardFailures()));
+        }
+    }
+
+    private void onInitialPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,
+                                      final ShardIterator shardIt, Exception e) {
         // we always add the shard failure for a specific shard instance
         // we do make sure to clean it on a successful response from a shard
         SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId());
@@ -201,18 +220,13 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
             final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
             if (successfulOps.get() == 0) {
                 if (logger.isDebugEnabled()) {
-                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("All shards failed for phase: [{}]", firstPhaseName()), e);
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("All shards failed for phase: [{}]", initialPhaseName()), e);
                 }
 
                 // no successful ops, raise an exception
-                raiseEarlyFailure(new SearchPhaseExecutionException(firstPhaseName(), "all shards failed", e, shardSearchFailures));
+                raiseEarlyFailure(new SearchPhaseExecutionException(initialPhaseName(), "all shards failed", e, shardSearchFailures));
             } else {
-                try {
-                    innerMoveToSecondPhase();
-                } catch (Exception inner) {
-                    inner.addSuppressed(e);
-                    raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
-                }
+                executePhase(initialPhaseName(), innerGetNextPhase(), e);
             }
         } else {
             final ShardRouting nextShard = shardIt.nextOrNull();
@@ -227,10 +241,10 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                 e);
             if (!lastShard) {
                 try {
-                    performFirstPhase(shardIndex, shardIt, nextShard);
+                    performInitialPhase(shardIndex, shardIt, nextShard);
                 } catch (Exception inner) {
                     inner.addSuppressed(e);
-                    onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
+                    onInitialPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
                 }
             } else {
                 // no more shards active, add a failure
@@ -290,10 +304,10 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
     }
 
     private void raiseEarlyFailure(Exception e) {
-        for (AtomicArray.Entry<FirstResult> entry : firstResults.asList()) {
+        for (AtomicArray.Entry<FirstResult> entry : initialResults.asList()) {
             try {
-                DiscoveryNode node = nodeIdToDiscoveryNode.apply(entry.value.shardTarget().nodeId());
-                sendReleaseSearchContext(entry.value.id(), node);
+                Transport.Connection connection = nodeIdToConnection.apply(entry.value.shardTarget().getNodeId());
+                sendReleaseSearchContext(entry.value.id(), connection);
             } catch (Exception inner) {
                 inner.addSuppressed(e);
                 logger.trace("failed to release context", inner);
@@ -302,48 +316,23 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         listener.onFailure(e);
     }
 
-    /**
-     * Releases shard targets that are not used in the docsIdsToLoad.
-     */
-    protected void releaseIrrelevantSearchContexts(AtomicArray<? extends QuerySearchResultProvider> queryResults,
-                                                   AtomicArray<IntArrayList> docIdsToLoad) {
-        if (docIdsToLoad == null) {
-            return;
-        }
-        // we only release search context that we did not fetch from if we are not scrolling
-        if (request.scroll() == null) {
-            for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults.asList()) {
-                QuerySearchResult queryResult = entry.value.queryResult();
-                if (queryResult.hasHits()
-                    && docIdsToLoad.get(entry.index) == null) { // but none of them made it to the global top docs
-                    try {
-                        DiscoveryNode node = nodeIdToDiscoveryNode.apply(entry.value.queryResult().shardTarget().nodeId());
-                        sendReleaseSearchContext(entry.value.queryResult().id(), node);
-                    } catch (Exception e) {
-                        logger.trace("failed to release context", e);
-                    }
-                }
-            }
+    protected void sendReleaseSearchContext(long contextId, Transport.Connection connection) {
+        if (connection != null) {
+            searchTransportService.sendFreeContext(connection, contextId, request);
         }
     }
 
-    protected void sendReleaseSearchContext(long contextId, DiscoveryNode node) {
-        if (node != null) {
-            searchTransportService.sendFreeContext(node, contextId, request);
-        }
-    }
-
-    protected ShardFetchSearchRequest createFetchRequest(QuerySearchResult queryResult, AtomicArray.Entry<IntArrayList> entry,
+    protected ShardFetchSearchRequest createFetchRequest(QuerySearchResult queryResult, int index, IntArrayList entry,
                                                          ScoreDoc[] lastEmittedDocPerShard) {
-        final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[entry.index] : null;
-        return new ShardFetchSearchRequest(request, queryResult.id(), entry.value, lastEmittedDoc);
+        final ScoreDoc lastEmittedDoc = (lastEmittedDocPerShard != null) ? lastEmittedDocPerShard[index] : null;
+        return new ShardFetchSearchRequest(request, queryResult.id(), entry, lastEmittedDoc);
     }
 
-    protected abstract void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
+    protected abstract void sendExecuteFirstPhase(Transport.Connection connection, ShardSearchTransportRequest request,
                                                   ActionListener<FirstResult> listener);
 
     protected final void processFirstPhaseResult(int shardIndex, FirstResult result) {
-        firstResults.set(shardIndex, result);
+        initialResults.set(shardIndex, result);
 
         if (logger.isTraceEnabled()) {
             logger.trace("got first-phase result from {}", result != null ? result.shardTarget() : null);
@@ -358,12 +347,12 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         }
     }
 
-    final void innerMoveToSecondPhase() throws Exception {
+    final CheckedRunnable<Exception> innerGetNextPhase() {
         if (logger.isTraceEnabled()) {
             StringBuilder sb = new StringBuilder();
             boolean hadOne = false;
-            for (int i = 0; i < firstResults.length(); i++) {
-                FirstResult result = firstResults.get(i);
+            for (int i = 0; i < initialResults.length(); i++) {
+                FirstResult result = initialResults.get(i);
                 if (result == null) {
                     continue; // failure
                 }
@@ -377,15 +366,191 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
 
             logger.trace("Moving to second phase, based on results from: {} (cluster state version: {})", sb, clusterStateVersion);
         }
-        moveToSecondPhase();
+        return getNextPhase(initialResults);
     }
 
-    protected abstract void moveToSecondPhase() throws Exception;
+    protected abstract CheckedRunnable<Exception> getNextPhase(AtomicArray<FirstResult> initialResults);
 
-    protected abstract String firstPhaseName();
+    protected abstract String initialPhaseName();
 
     protected Executor getExecutor() {
         return executor;
     }
 
+    // this is a simple base class to simplify fan out to shards and collect
+    final class CountedCollector<R extends SearchPhaseResult> {
+        private final AtomicArray<R> resultArray;
+        private final CountDown counter;
+        private final IntConsumer onFinish;
+
+        CountedCollector(AtomicArray<R> resultArray, int expectedOps, IntConsumer onFinish) {
+            this.resultArray = resultArray;
+            this.counter = new CountDown(expectedOps);
+            this.onFinish = onFinish;
+        }
+
+        void countDown() {
+            if (counter.countDown()) {
+                onFinish.accept(successfulOps.get());
+            }
+        }
+
+        void onResult(int index, R result, SearchShardTarget target) {
+            try {
+                result.shardTarget(target);
+                resultArray.set(index, result);
+            } finally {
+                countDown();
+            }
+        }
+
+        void onFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
+            try {
+                addShardFailure(shardIndex, shardTarget, e);
+            } finally {
+                successfulOps.decrementAndGet();
+                countDown();
+            }
+        }
+
+    }
+
+    /*
+     *  At this point AbstractSearchAsyncAction is just a base-class for the first phase of a search where we have multiple replicas
+     *  for each shardID. If one of them is not available we move to the next one. Yet, once we passed that first stage we have to work with
+     *  the shards we succeeded on the initial phase.
+     *  Unfortunately, subsequent phases are not fully detached from the initial phase since they are all non-static inner classes.
+     *  In future changes this will be changed to detach the inner classes to test them in isolation and to simplify their creation.
+     *  The AbstractSearchAsyncAction should be final and it should just get a factory for the next phase instead of requiring subclasses
+     *  etc.
+     */
+    final class FetchPhase implements CheckedRunnable<Exception> {
+        private final AtomicArray<FetchSearchResult> fetchResults;
+        private final SearchPhaseController searchPhaseController;
+        private final AtomicArray<QuerySearchResultProvider> queryResults;
+
+        FetchPhase(AtomicArray<QuerySearchResultProvider> queryResults,
+                           SearchPhaseController searchPhaseController) {
+            this.fetchResults = new AtomicArray<>(queryResults.length());
+            this.searchPhaseController = searchPhaseController;
+            this.queryResults = queryResults;
+        }
+
+        @Override
+        public void run() throws Exception {
+            final boolean isScrollRequest = request.scroll() != null;
+            ScoreDoc[] sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, queryResults);
+            final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(queryResults.length(), sortedShardDocs);
+            final IntConsumer finishPhase =  successOpts
+                -> sendResponseAsync("fetch", searchPhaseController, sortedShardDocs, queryResults, fetchResults);
+            if (sortedShardDocs.length == 0) { // no docs to fetch -- sidestep everything and return
+                queryResults.asList().stream()
+                    .map(e -> e.value.queryResult())
+                    .forEach(this::releaseIrrelevantSearchContext); // we have to release contexts here to free up resources
+                finishPhase.accept(successfulOps.get());
+            } else {
+                final ScoreDoc[] lastEmittedDocPerShard = isScrollRequest ?
+                    searchPhaseController.getLastEmittedDocPerShard(queryResults.asList(), sortedShardDocs, queryResults.length())
+                    : null;
+                final CountedCollector<FetchSearchResult> counter = new CountedCollector<>(fetchResults,
+                    docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not
+                    finishPhase);
+                for (int i = 0; i < docIdsToLoad.length; i++) {
+                    IntArrayList entry = docIdsToLoad[i];
+                    QuerySearchResultProvider queryResult = queryResults.get(i);
+                    if (entry == null) { // no results for this shard ID
+                        if (queryResult != null) {
+                            // if we got some hits from this shard we have to release the context there
+                            // we do this as we go since it will free up resources and passing on the request on the
+                            // transport layer is cheap.
+                            releaseIrrelevantSearchContext(queryResult.queryResult());
+                        }
+                        // in any case we count down this result since we don't talk to this shard anymore
+                        counter.countDown();
+                    } else {
+                        Transport.Connection connection = nodeIdToConnection.apply(queryResult.shardTarget().getNodeId());
+                        ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult(), i, entry,
+                            lastEmittedDocPerShard);
+                        executeFetch(i, queryResult.shardTarget(), counter, fetchSearchRequest, queryResult.queryResult(),
+                            connection);
+                    }
+                }
+            }
+        }
+
+        private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget,
+                                    final CountedCollector<FetchSearchResult> counter,
+                                    final ShardFetchSearchRequest fetchSearchRequest, final QuerySearchResult querySearchResult,
+                                    final Transport.Connection connection) {
+            searchTransportService.sendExecuteFetch(connection, fetchSearchRequest, task, new ActionListener<FetchSearchResult>() {
+                @Override
+                public void onResponse(FetchSearchResult result) {
+                    counter.onResult(shardIndex, result, shardTarget);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    try {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase",
+                                fetchSearchRequest.id()), e);
+                        }
+                        counter.onFailure(shardIndex, shardTarget, e);
+                    } finally {
+                        // the search context might not be cleared on the node where the fetch was executed for example
+                        // because the action was rejected by the thread pool. in this case we need to send a dedicated
+                        // request to clear the search context.
+                        releaseIrrelevantSearchContext(querySearchResult);
+                    }
+                }
+            });
+        }
+
+        /**
+         * Releases shard targets that are not used in the docsIdsToLoad.
+         */
+        private void releaseIrrelevantSearchContext(QuerySearchResult queryResult) {
+            // we only release search context that we did not fetch from if we are not scrolling
+            // and if it has at lease one hit that didn't make it to the global topDocs
+            if (request.scroll() == null && queryResult.hasHits()) {
+                try {
+                    Transport.Connection connection = nodeIdToConnection.apply(queryResult.shardTarget().getNodeId());
+                    sendReleaseSearchContext(queryResult.id(), connection);
+                } catch (Exception e) {
+                    logger.trace("failed to release context", e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Sends back a result to the user. This method will create the sorted docs if they are null and will build the scrollID for the
+     * response. Note: This method will send the response in a different thread depending on the executor.
+     */
+    final void sendResponseAsync(String phase, SearchPhaseController searchPhaseController, ScoreDoc[] sortedDocs,
+                                  AtomicArray<? extends QuerySearchResultProvider> queryResultsArr,
+                                  AtomicArray<? extends FetchSearchResultProvider> fetchResultsArr) {
+        getExecutor().execute(new ActionRunnable<SearchResponse>(listener) {
+            @Override
+            public void doRun() throws IOException {
+                final boolean isScrollRequest = request.scroll() != null;
+                final ScoreDoc[] theScoreDocs = sortedDocs == null ? searchPhaseController.sortDocs(isScrollRequest, queryResultsArr)
+                    : sortedDocs;
+                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, theScoreDocs, queryResultsArr,
+                    fetchResultsArr);
+                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), queryResultsArr) : null;
+                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
+                    buildTookInMillis(), buildShardFailures()));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                ReduceSearchPhaseException failure = new ReduceSearchPhaseException(phase, "", e, buildShardFailures());
+                if (logger.isDebugEnabled()) {
+                    logger.debug("failed to reduce search", failure);
+                }
+                super.onFailure(failure);
+            }
+        });
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
index ff8314acce5..d1ec790d01a 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ClearScrollResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 
@@ -31,7 +31,7 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
-public class ClearScrollResponse extends ActionResponse implements StatusToXContent {
+public class ClearScrollResponse extends ActionResponse implements StatusToXContentObject {
 
     private boolean succeeded;
     private int numFreed;
@@ -66,8 +66,10 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.field(Fields.SUCCEEDED, succeeded);
         builder.field(Fields.NUMFREED, numFreed);
+        builder.endObject();
         return builder;
     }
 
@@ -89,5 +91,4 @@ public class ClearScrollResponse extends ActionResponse implements StatusToXCont
         static final String SUCCEEDED = "succeeded";
         static final String NUMFREED = "num_freed";
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index 317b775a403..4d42ad334a9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -23,12 +23,12 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -37,7 +37,7 @@ import java.util.Iterator;
 /**
  * A multi search response.
  */
-public class MultiSearchResponse extends ActionResponse implements Iterable<MultiSearchResponse.Item>, ToXContent {
+public class MultiSearchResponse extends ActionResponse implements Iterable<MultiSearchResponse.Item>, ToXContentObject {
 
     /**
      * A search response item, holding the actual search response, or an error message if it failed.
@@ -151,39 +151,31 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.RESPONSES);
         for (Item item : items) {
             builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
+                ElasticsearchException.generateFailureXContent(builder, params, item.getFailure(), true);
                 builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
             } else {
-                item.getResponse().toXContent(builder, params);
+                item.getResponse().innerToXContent(builder, params);
                 builder.field(Fields.STATUS, item.getResponse().status().getStatus());
             }
             builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final String RESPONSES = "responses";
         static final String STATUS = "status";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
     }
 
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
index f2ea5356106..b588827867f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
@@ -31,7 +31,7 @@ class ParsedScrollId {
 
     private final ScrollIdForNode[] context;
 
-    public ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
+    ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
         this.source = source;
         this.type = type;
         this.context = context;
diff --git a/core/src/main/java/org/elasticsearch/action/search/RemoteClusterConnection.java b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterConnection.java
new file mode 100644
index 00000000000..73bc4f2ee7e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterConnection.java
@@ -0,0 +1,508 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CancellableThreads;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportActionProxy;
+import org.elasticsearch.transport.TransportConnectionListener;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
+
+/**
+ * Represents a connection to a single remote cluster. In contrast to a local cluster a remote cluster is not joined such that the
+ * current node is part of the cluster and it won't receive cluster state updates from the remote cluster. Remote clusters are also not
+ * fully connected with the current node. From a connection perspective a local cluster forms a bi-directional star network while in the
+ * remote case we only connect to a subset of the nodes in the cluster in an uni-directional fashion.
+ *
+ * This class also handles the discovery of nodes from the remote cluster. The initial list of seed nodes is only used to discover all nodes
+ * in the remote cluster and connects to all eligible nodes, for details see {@link RemoteClusterService#REMOTE_NODE_ATTRIBUTE}.
+ *
+ * In the case of a disconnection, this class will issue a re-connect task to establish at most
+ * {@link RemoteClusterService#REMOTE_CONNECTIONS_PER_CLUSTER} until either all eligible nodes are exhausted or the maximum number of
+ * connections per cluster has been reached.
+ */
+final class RemoteClusterConnection extends AbstractComponent implements TransportConnectionListener, Closeable {
+
+    private final TransportService transportService;
+    private final ConnectionProfile remoteProfile;
+    private final Set<DiscoveryNode> connectedNodes = Collections.newSetFromMap(new ConcurrentHashMap<>());
+    private final Supplier<DiscoveryNode> nodeSupplier;
+    private final String clusterAlias;
+    private final int maxNumRemoteConnections;
+    private final Predicate<DiscoveryNode> nodePredicate;
+    private volatile List<DiscoveryNode> seedNodes;
+    private final ConnectHandler connectHandler;
+
+    /**
+     * Creates a new {@link RemoteClusterConnection}
+     * @param settings the nodes settings object
+     * @param clusterAlias the configured alias of the cluster to connect to
+     * @param seedNodes a list of seed nodes to discover eligible nodes from
+     * @param transportService the local nodes transport service
+     * @param maxNumRemoteConnections the maximum number of connections to the remote cluster
+     * @param nodePredicate a predicate to filter eligible remote nodes to connect to
+     */
+    RemoteClusterConnection(Settings settings, String clusterAlias, List<DiscoveryNode> seedNodes,
+                            TransportService transportService, int maxNumRemoteConnections, Predicate<DiscoveryNode> nodePredicate) {
+        super(settings);
+        this.transportService = transportService;
+        this.maxNumRemoteConnections = maxNumRemoteConnections;
+        this.nodePredicate = nodePredicate;
+        this.clusterAlias = clusterAlias;
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.setConnectTimeout(TcpTransport.TCP_CONNECT_TIMEOUT.get(settings));
+        builder.setHandshakeTimeout(TcpTransport.TCP_CONNECT_TIMEOUT.get(settings));
+        builder.addConnections(6, TransportRequestOptions.Type.REG, TransportRequestOptions.Type.PING); // TODO make this configurable?
+        builder.addConnections(0, // we don't want this to be used for anything else but search
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.STATE,
+            TransportRequestOptions.Type.RECOVERY);
+        remoteProfile = builder.build();
+        nodeSupplier = new Supplier<DiscoveryNode>() {
+            private volatile Iterator<DiscoveryNode> current;
+            @Override
+            public DiscoveryNode get() {
+                if (current == null || current.hasNext() == false) {
+                    current = connectedNodes.iterator();
+                    if (current.hasNext() == false) {
+                        throw new IllegalStateException("No node available for cluster: " + clusterAlias + " nodes: " + connectedNodes);
+                    }
+                }
+                return current.next();
+            }
+        };
+        this.seedNodes = Collections.unmodifiableList(seedNodes);
+        this.connectHandler = new ConnectHandler();
+        transportService.addConnectionListener(this);
+    }
+
+    /**
+     * Updates the list of seed nodes for this cluster connection
+     */
+    synchronized void updateSeedNodes(List<DiscoveryNode> seedNodes, ActionListener<Void> connectListener) {
+        this.seedNodes = Collections.unmodifiableList(new ArrayList<>(seedNodes));
+        connectHandler.connect(connectListener);
+    }
+
+    @Override
+    public void onNodeDisconnected(DiscoveryNode node) {
+        boolean remove = connectedNodes.remove(node);
+        if (remove && connectedNodes.size() < maxNumRemoteConnections) {
+            // try to reconnect and fill up the slot of the disconnected node
+            connectHandler.forceConnect();
+        }
+    }
+
+    /**
+     * Fetches all shards for the search request from this remote connection. This is used to later run the search on the remote end.
+     */
+    public void fetchSearchShards(SearchRequest searchRequest, final List<String> indices,
+                                  ActionListener<ClusterSearchShardsResponse> listener) {
+        if (connectedNodes.isEmpty()) {
+            // just in case if we are not connected for some reason we try to connect and if we fail we have to notify the listener
+            // this will cause some back pressure on the search end and eventually will cause rejections but that's fine
+            // we can't proceed with a search on a cluster level.
+            // in the future we might want to just skip the remote nodes in such a case but that can already be implemented on the caller
+            // end since they provide the listener.
+            connectHandler.connect(ActionListener.wrap((x) -> fetchShardsInternal(searchRequest, indices, listener), listener::onFailure));
+        } else {
+            fetchShardsInternal(searchRequest, indices, listener);
+        }
+    }
+
+    private void fetchShardsInternal(SearchRequest searchRequest, List<String> indices,
+                                     final ActionListener<ClusterSearchShardsResponse> listener) {
+        final DiscoveryNode node = nodeSupplier.get();
+        ClusterSearchShardsRequest searchShardsRequest = new ClusterSearchShardsRequest(indices.toArray(new String[indices.size()]))
+            .indicesOptions(searchRequest.indicesOptions()).local(true).preference(searchRequest.preference())
+            .routing(searchRequest.routing());
+        transportService.sendRequest(node, ClusterSearchShardsAction.NAME, searchShardsRequest,
+            new TransportResponseHandler<ClusterSearchShardsResponse>() {
+
+                @Override
+                public ClusterSearchShardsResponse newInstance() {
+                    return new ClusterSearchShardsResponse();
+                }
+
+                @Override
+                public void handleResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) {
+                    listener.onResponse(clusterSearchShardsResponse);
+                }
+
+                @Override
+                public void handleException(TransportException e) {
+                    listener.onFailure(e);
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SEARCH;
+                }
+            });
+    }
+
+    /**
+     * Returns a connection to the remote cluster. This connection might be a proxy connection that redirects internally to the
+     * given node.
+     */
+    Transport.Connection getConnection(DiscoveryNode remoteClusterNode) {
+        DiscoveryNode discoveryNode = nodeSupplier.get();
+        Transport.Connection connection = transportService.getConnection(discoveryNode);
+        return new Transport.Connection() {
+            @Override
+            public DiscoveryNode getNode() {
+                return remoteClusterNode;
+            }
+
+            @Override
+            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+                throws IOException, TransportException {
+                connection.sendRequest(requestId, TransportActionProxy.getProxyAction(action),
+                    TransportActionProxy.wrapRequest(remoteClusterNode, request), options);
+            }
+
+            @Override
+            public void close() throws IOException {
+                assert false: "proxy connections must not be closed";
+            }
+        };
+    }
+
+    @Override
+    public void close() throws IOException {
+        connectHandler.close();
+    }
+
+    public boolean isClosed() {
+        return connectHandler.isClosed();
+    }
+
+    /**
+     * The connect handler manages node discovery and the actual connect to the remote cluster.
+     * There is at most one connect job running at any time. If such a connect job is triggered
+     * while another job is running the provided listeners are queued and batched up until the current running job returns.
+     *
+     * The handler has a built-in queue that can hold up to 100 connect attempts and will reject requests once the queue is full.
+     * In a scenario when a remote cluster becomes unavailable we will queue requests up but if we can't connect quick enough
+     * we will just reject the connect trigger which will lead to failing searches.
+     */
+    private class ConnectHandler implements Closeable {
+        private final Semaphore running = new Semaphore(1);
+        private final AtomicBoolean closed = new AtomicBoolean(false);
+        private final BlockingQueue<ActionListener<Void>> queue = new ArrayBlockingQueue<>(100);
+        private final CancellableThreads cancellableThreads = new CancellableThreads();
+
+        /**
+         * Triggers a connect round iff there are pending requests queued up and if there is no
+         * connect round currently running.
+         */
+        void maybeConnect() {
+            connect(null);
+        }
+
+        /**
+         * Triggers a connect round unless there is one running already. If there is a connect round running, the listener will either
+         * be queued or rejected and failed.
+         */
+        void connect(ActionListener<Void> connectListener) {
+            connect(connectListener, false);
+        }
+
+        /**
+         * Triggers a connect round unless there is one already running. In contrast to {@link #maybeConnect()} will this method also
+         * trigger a connect round if there is no listener queued up.
+         */
+        void forceConnect() {
+            connect(null, true);
+        }
+
+        private void connect(ActionListener<Void> connectListener, boolean forceRun) {
+            final boolean runConnect;
+            final Collection<ActionListener<Void>> toNotify;
+            synchronized (queue) {
+                if (connectListener != null && queue.offer(connectListener) == false) {
+                    connectListener.onFailure(new RejectedExecutionException("connect queue is full"));
+                    return;
+                }
+                if (forceRun == false && queue.isEmpty()) {
+                    return;
+                }
+                runConnect = running.tryAcquire();
+                if (runConnect) {
+                    toNotify = new ArrayList<>();
+                    queue.drainTo(toNotify);
+                    if (closed.get()) {
+                        running.release();
+                        ActionListener.onFailure(toNotify, new AlreadyClosedException("connect handler is already closed"));
+                        return;
+                    }
+                } else {
+                    toNotify = Collections.emptyList();
+                }
+            }
+            if (runConnect) {
+                forkConnect(toNotify);
+            }
+        }
+
+        private void forkConnect(final Collection<ActionListener<Void>> toNotify) {
+            ThreadPool threadPool = transportService.getThreadPool();
+            ExecutorService executor = threadPool.executor(ThreadPool.Names.MANAGEMENT);
+            executor.submit(new AbstractRunnable() {
+                @Override
+                public void onFailure(Exception e) {
+                    synchronized (queue) {
+                        running.release();
+                    }
+                    try {
+                        ActionListener.onFailure(toNotify, e);
+                    } finally {
+                        maybeConnect();
+                    }
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    ActionListener<Void> listener = ActionListener.wrap((x) -> {
+                        synchronized (queue) {
+                            running.release();
+                        }
+                        try {
+                            ActionListener.onResponse(toNotify, x);
+                        } finally {
+                            maybeConnect();
+                        }
+
+                    }, (e) -> {
+                        synchronized (queue) {
+                            running.release();
+                        }
+                        try {
+                            ActionListener.onFailure(toNotify, e);
+                        } finally {
+                            maybeConnect();
+                        }
+                    });
+                    collectRemoteNodes(seedNodes.iterator(), transportService, listener);
+                }
+            });
+
+        }
+
+        void collectRemoteNodes(Iterator<DiscoveryNode> seedNodes,
+                                final TransportService transportService, ActionListener<Void> listener) {
+            if (Thread.currentThread().isInterrupted()) {
+                listener.onFailure(new InterruptedException("remote connect thread got interrupted"));
+            }
+            try {
+                if (seedNodes.hasNext()) {
+                    cancellableThreads.executeIO(() -> {
+                        final DiscoveryNode seedNode = seedNodes.next();
+                        final DiscoveryNode handshakeNode;
+                        Transport.Connection connection = transportService.openConnection(seedNode,
+                            ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, null, null));
+                        boolean success = false;
+                        try {
+                            handshakeNode = transportService.handshake(connection, remoteProfile.getHandshakeTimeout().millis(),
+                                (c) -> true);
+                            if (nodePredicate.test(handshakeNode) && connectedNodes.size() < maxNumRemoteConnections) {
+                                transportService.connectToNode(handshakeNode, remoteProfile);
+                                connectedNodes.add(handshakeNode);
+                            }
+                            ClusterStateRequest request = new ClusterStateRequest();
+                            request.clear();
+                            request.nodes(true);
+                            // here we pass on the connection since we can only close it once the sendRequest returns otherwise
+                            // due to the async nature (it will return before it's actually sent) this can cause the request to fail
+                            // due to an already closed connection.
+                            transportService.sendRequest(connection,
+                                ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY,
+                                new SniffClusterStateResponseHandler(transportService, connection, listener, seedNodes,
+                                    cancellableThreads));
+                            success = true;
+                        } finally {
+                            if (success == false) {
+                                connection.close();
+                            }
+                        }
+                    });
+                } else {
+                    listener.onFailure(new IllegalStateException("no seed node left"));
+                }
+            } catch (CancellableThreads.ExecutionCancelledException ex) {
+                listener.onFailure(ex); // we got canceled - fail the listener and step out
+            } catch (ConnectTransportException | IOException | IllegalStateException ex) {
+                // ISE if we fail the handshake with an version incompatible node
+                if (seedNodes.hasNext()) {
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("fetching nodes from external cluster {} failed",
+                        clusterAlias), ex);
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                } else {
+                    listener.onFailure(ex);
+                }
+            }
+        }
+
+        @Override
+        public void close() throws IOException {
+            try {
+                if (closed.compareAndSet(false, true)) {
+                    cancellableThreads.cancel("connect handler is closed");
+                    running.acquire(); // acquire the semaphore to ensure all connections are closed and all thread joined
+                    running.release();
+                    maybeConnect(); // now go and notify pending listeners
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+
+        final boolean isClosed() {
+            return closed.get();
+        }
+
+        /* This class handles the _state response from the remote cluster when sniffing nodes to connect to */
+        private class SniffClusterStateResponseHandler implements TransportResponseHandler<ClusterStateResponse> {
+
+            private final TransportService transportService;
+            private final Transport.Connection connection;
+            private final ActionListener<Void> listener;
+            private final Iterator<DiscoveryNode> seedNodes;
+            private final CancellableThreads cancellableThreads;
+
+            SniffClusterStateResponseHandler(TransportService transportService, Transport.Connection connection,
+                                             ActionListener<Void> listener, Iterator<DiscoveryNode> seedNodes,
+                                             CancellableThreads cancellableThreads) {
+                this.transportService = transportService;
+                this.connection = connection;
+                this.listener = listener;
+                this.seedNodes = seedNodes;
+                this.cancellableThreads = cancellableThreads;
+            }
+
+            @Override
+            public ClusterStateResponse newInstance() {
+                return new ClusterStateResponse();
+            }
+
+            @Override
+            public void handleResponse(ClusterStateResponse response) {
+                try {
+                    try (Closeable theConnection = connection) { // the connection is unused - see comment in #collectRemoteNodes
+                        // we have to close this connection before we notify listeners - this is mainly needed for test correctness
+                        // since if we do it afterwards we might fail assertions that check if all high level connections are closed.
+                        // from a code correctness perspective we could also close it afterwards. This try/with block will
+                        // maintain the possibly exceptions thrown from within the try block and suppress the ones that are possible thrown
+                        // by closing the connection
+                        cancellableThreads.executeIO(() -> {
+                            DiscoveryNodes nodes = response.getState().nodes();
+                            Iterable<DiscoveryNode> nodesIter = nodes.getNodes()::valuesIt;
+                            for (DiscoveryNode node : nodesIter) {
+                                if (nodePredicate.test(node) && connectedNodes.size() < maxNumRemoteConnections) {
+                                    try {
+                                        transportService.connectToNode(node, remoteProfile); // noop if node is connected
+                                        connectedNodes.add(node);
+                                    } catch (ConnectTransportException | IllegalStateException ex) {
+                                        // ISE if we fail the handshake with an version incompatible node
+                                        // fair enough we can't connect just move on
+                                        logger.debug((Supplier<?>)
+                                            () -> new ParameterizedMessage("failed to connect to node {}", node), ex);
+                                    }
+                                }
+                            }
+                        });
+                    }
+                    listener.onResponse(null);
+                } catch (CancellableThreads.ExecutionCancelledException ex) {
+                    listener.onFailure(ex); // we got canceled - fail the listener and step out
+                } catch (Exception ex) {
+                    logger.warn((Supplier<?>)
+                        () -> new ParameterizedMessage("fetching nodes from external cluster {} failed",
+                            clusterAlias), ex);
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                }
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                logger.warn((Supplier<?>)
+                    () -> new ParameterizedMessage("fetching nodes from external cluster {} failed", clusterAlias),
+                    exp);
+                try {
+                    IOUtils.closeWhileHandlingException(connection);
+                } finally {
+                    // once the connection is closed lets try the next node
+                    collectRemoteNodes(seedNodes, transportService, listener);
+                }
+            }
+
+            @Override
+            public String executor() {
+                return ThreadPool.Names.MANAGEMENT;
+            }
+        }
+    }
+
+    boolean assertNoRunningConnections() { // for testing only
+        assert connectHandler.running.availablePermits() == 1;
+        return true;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java
new file mode 100644
index 00000000000..92b4ca5f0b3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/search/RemoteClusterService.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.PlainShardIterator;
+import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Basic service for accessing remote clusters via gateway nodes
+ */
+public final class RemoteClusterService extends AbstractComponent implements Closeable {
+
+    static final String LOCAL_CLUSTER_GROUP_KEY = "";
+
+    /**
+     * A list of initial seed nodes to discover eligible nodes from the remote cluster
+     */
+    public static final Setting.AffixSetting<List<InetSocketAddress>> REMOTE_CLUSTERS_SEEDS = Setting.affixKeySetting("search.remote.",
+        "seeds", (key) -> Setting.listSetting(key, Collections.emptyList(), RemoteClusterService::parseSeedAddress,
+            Setting.Property.NodeScope, Setting.Property.Dynamic));
+    /**
+     * The maximum number of connections that will be established to a remote cluster. For instance if there is only a single
+     * seed node, other nodes will be discovered up to the given number of nodes in this setting. The default is 3.
+     */
+    public static final Setting<Integer> REMOTE_CONNECTIONS_PER_CLUSTER = Setting.intSetting("search.remote.connections_per_cluster",
+        3, 1, Setting.Property.NodeScope);
+
+    /**
+     * The initial connect timeout for remote cluster connections
+     */
+    public static final Setting<TimeValue> REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING =
+        Setting.positiveTimeSetting("search.remote.initial_connect_timeout", TimeValue.timeValueSeconds(30), Setting.Property.NodeScope);
+
+    /**
+     * The name of a node attribute to select nodes that should be connected to in the remote cluster.
+     * For instance a node can be configured with <tt>node.attr.gateway: true</tt> in order to be eligible as a gateway node between
+     * clusters. In that case <tt>search.remote.node.attr: gateway</tt> can be used to filter out other nodes in the remote cluster.
+     * The value of the setting is expected to be a boolean, <tt>true</tt> for nodes that can become gateways, <tt>false</tt> otherwise.
+     */
+    public static final Setting<String> REMOTE_NODE_ATTRIBUTE = Setting.simpleString("search.remote.node.attr",
+        Setting.Property.NodeScope);
+
+    private static final char REMOTE_CLUSTER_INDEX_SEPARATOR = ':';
+
+    private final TransportService transportService;
+    private final int numRemoteConnections;
+    private volatile Map<String, RemoteClusterConnection> remoteClusters = Collections.emptyMap();
+
+    RemoteClusterService(Settings settings, TransportService transportService) {
+        super(settings);
+        this.transportService = transportService;
+        numRemoteConnections = REMOTE_CONNECTIONS_PER_CLUSTER.get(settings);
+    }
+
+    /**
+     * This method updates the list of remote clusters. It's intended to be used as an update consumer on the settings infrastructure
+     * @param seeds a cluster alias to discovery node mapping representing the remote clusters seeds nodes
+     * @param connectionListener a listener invoked once every configured cluster has been connected to
+     */
+    private synchronized void updateRemoteClusters(Map<String, List<DiscoveryNode>> seeds, ActionListener<Void> connectionListener) {
+        if (seeds.containsKey(LOCAL_CLUSTER_GROUP_KEY)) {
+            throw new IllegalArgumentException("remote clusters must not have the empty string as its key");
+        }
+        Map<String, RemoteClusterConnection> remoteClusters = new HashMap<>();
+        if (seeds.isEmpty()) {
+            connectionListener.onResponse(null);
+        } else {
+            CountDown countDown = new CountDown(seeds.size());
+            Predicate<DiscoveryNode> nodePredicate = (node) -> Version.CURRENT.isCompatible(node.getVersion());
+            if (REMOTE_NODE_ATTRIBUTE.exists(settings)) {
+                // nodes can be tagged with node.attr.remote_gateway: true to allow a node to be a gateway node for
+                // cross cluster search
+                String attribute = REMOTE_NODE_ATTRIBUTE.get(settings);
+                nodePredicate = nodePredicate.and((node) -> Boolean.getBoolean(node.getAttributes().getOrDefault(attribute, "false")));
+            }
+            remoteClusters.putAll(this.remoteClusters);
+            for (Map.Entry<String, List<DiscoveryNode>> entry : seeds.entrySet()) {
+                RemoteClusterConnection remote = this.remoteClusters.get(entry.getKey());
+                if (entry.getValue().isEmpty()) { // with no seed nodes we just remove the connection
+                    try {
+                        IOUtils.close(remote);
+                    } catch (IOException e) {
+                        logger.warn("failed to close remote cluster connections for cluster: " + entry.getKey(), e);
+                    }
+                    remoteClusters.remove(entry.getKey());
+                    continue;
+                }
+
+                if (remote == null) { // this is a new cluster we have to add a new representation
+                    remote = new RemoteClusterConnection(settings, entry.getKey(), entry.getValue(), transportService, numRemoteConnections,
+                        nodePredicate);
+                    remoteClusters.put(entry.getKey(), remote);
+                }
+
+                // now update the seed nodes no matter if it's new or already existing
+                RemoteClusterConnection finalRemote = remote;
+                remote.updateSeedNodes(entry.getValue(), ActionListener.wrap(
+                    response -> {
+                        if (countDown.countDown()) {
+                            connectionListener.onResponse(response);
+                        }
+                    },
+                    exception -> {
+                        if (countDown.fastForward()) {
+                            connectionListener.onFailure(exception);
+                        }
+                        if (finalRemote.isClosed() == false) {
+                            logger.warn("failed to update seed list for cluster: " + entry.getKey(), exception);
+                        }
+                    }));
+            }
+        }
+        this.remoteClusters = Collections.unmodifiableMap(remoteClusters);
+    }
+
+    /**
+     * Returns <code>true</code> if at least one remote cluster is configured
+     */
+    boolean isCrossClusterSearchEnabled() {
+        return remoteClusters.isEmpty() == false;
+    }
+
+    /**
+     * Groups indices per cluster by splitting remote cluster-alias, index-name pairs on {@link #REMOTE_CLUSTER_INDEX_SEPARATOR}. All
+     * indices per cluster are collected as a list in the returned map keyed by the cluster alias. Local indices are grouped under
+     * {@link #LOCAL_CLUSTER_GROUP_KEY}. The returned map is mutable.
+     *
+     * @param requestIndices the indices in the search request to filter
+     * @param indexExists a predicate that can test if a certain index or alias exists
+     *
+     * @return a map of grouped remote and local indices
+     */
+    Map<String, List<String>> groupClusterIndices(String[] requestIndices, Predicate<String> indexExists) {
+        Map<String, List<String>> perClusterIndices = new HashMap<>();
+        for (String index : requestIndices) {
+            int i = index.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR);
+            String indexName = index;
+            String clusterName = LOCAL_CLUSTER_GROUP_KEY;
+            if (i >= 0) {
+                String remoteClusterName = index.substring(0, i);
+                if (isRemoteClusterRegistered(remoteClusterName)) {
+                    if (indexExists.test(index)) {
+                        // we use : as a separator for remote clusters. might conflict if there is an index that is actually named
+                        // remote_cluster_alias:index_name - for this case we fail the request. the user can easily change the cluster alias
+                        // if that happens
+                        throw new IllegalArgumentException("Can not filter indices; index " + index +
+                            " exists but there is also a remote cluster named: " + remoteClusterName);
+                    }
+                    indexName = index.substring(i + 1);
+                    clusterName = remoteClusterName;
+                }
+            }
+            perClusterIndices.computeIfAbsent(clusterName, k -> new ArrayList<String>()).add(indexName);
+        }
+        return perClusterIndices;
+}
+
+    /**
+     * Returns <code>true</code> iff the given cluster is configured as a remote cluster. Otherwise <code>false</code>
+     */
+    boolean isRemoteClusterRegistered(String clusterName) {
+        return remoteClusters.containsKey(clusterName);
+    }
+
+    void collectSearchShards(SearchRequest searchRequest, Map<String, List<String>> remoteIndicesByCluster,
+                             ActionListener<Map<String, ClusterSearchShardsResponse>> listener) {
+        final CountDown responsesCountDown = new CountDown(remoteIndicesByCluster.size());
+        final Map<String, ClusterSearchShardsResponse> searchShardsResponses = new ConcurrentHashMap<>();
+        final AtomicReference<TransportException> transportException = new AtomicReference<>();
+        for (Map.Entry<String, List<String>> entry : remoteIndicesByCluster.entrySet()) {
+            final String clusterName = entry.getKey();
+            RemoteClusterConnection remoteClusterConnection = remoteClusters.get(clusterName);
+            if (remoteClusterConnection == null) {
+                throw new IllegalArgumentException("no such remote cluster: " + clusterName);
+            }
+            final List<String> indices = entry.getValue();
+            remoteClusterConnection.fetchSearchShards(searchRequest, indices,
+                new ActionListener<ClusterSearchShardsResponse>() {
+                    @Override
+                    public void onResponse(ClusterSearchShardsResponse clusterSearchShardsResponse) {
+                        searchShardsResponses.put(clusterName, clusterSearchShardsResponse);
+                        if (responsesCountDown.countDown()) {
+                            TransportException exception = transportException.get();
+                            if (exception == null) {
+                                listener.onResponse(searchShardsResponses);
+                            } else {
+                                listener.onFailure(transportException.get());
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        TransportException exception = new TransportException("unable to communicate with remote cluster [" +
+                            clusterName + "]", e);
+                        if (transportException.compareAndSet(null, exception) == false) {
+                            exception = transportException.accumulateAndGet(exception, (previous, current) -> {
+                                current.addSuppressed(previous);
+                                return current;
+                            });
+                        }
+                        if (responsesCountDown.countDown()) {
+                            listener.onFailure(exception);
+                        }
+                    }
+                });
+        }
+    }
+
+
+    Function<String, Transport.Connection> processRemoteShards(Map<String, ClusterSearchShardsResponse> searchShardsResponses,
+                                                                       List<ShardIterator> remoteShardIterators,
+                                                                       Map<String, AliasFilter> aliasFilterMap) {
+        Map<String, Supplier<Transport.Connection>> nodeToCluster = new HashMap<>();
+        for (Map.Entry<String, ClusterSearchShardsResponse> entry : searchShardsResponses.entrySet()) {
+            String clusterName = entry.getKey();
+            ClusterSearchShardsResponse searchShardsResponse = entry.getValue();
+            for (DiscoveryNode remoteNode : searchShardsResponse.getNodes()) {
+                nodeToCluster.put(remoteNode.getId(), () -> getConnection(remoteNode, clusterName));
+            }
+            Map<String, AliasFilter> indicesAndFilters = searchShardsResponse.getIndicesAndFilters();
+            for (ClusterSearchShardsGroup clusterSearchShardsGroup : searchShardsResponse.getGroups()) {
+                //add the cluster name to the remote index names for indices disambiguation
+                //this ends up in the hits returned with the search response
+                ShardId shardId = clusterSearchShardsGroup.getShardId();
+                Index remoteIndex = shardId.getIndex();
+                Index index = new Index(clusterName + REMOTE_CLUSTER_INDEX_SEPARATOR + remoteIndex.getName(), remoteIndex.getUUID());
+                ShardIterator shardIterator = new PlainShardIterator(new ShardId(index, shardId.getId()),
+                    Arrays.asList(clusterSearchShardsGroup.getShards()));
+                remoteShardIterators.add(shardIterator);
+                AliasFilter aliasFilter;
+                if (indicesAndFilters == null) {
+                    aliasFilter = new AliasFilter(null, Strings.EMPTY_ARRAY);
+                } else {
+                    aliasFilter = indicesAndFilters.get(shardId.getIndexName());
+                    assert aliasFilter != null;
+                }
+                // here we have to map the filters to the UUID since from now on we use the uuid for the lookup
+                aliasFilterMap.put(remoteIndex.getUUID(), aliasFilter);
+            }
+        }
+        return (nodeId) -> {
+            Supplier<Transport.Connection> supplier = nodeToCluster.get(nodeId);
+            if (supplier == null) {
+                throw new IllegalArgumentException("unknown remote node: " + nodeId);
+            }
+            return supplier.get();
+        };
+    }
+
+    /**
+     * Returns a connection to the given node on the given remote cluster
+     * @throws IllegalArgumentException if the remote cluster is unknown
+     */
+    private Transport.Connection getConnection(DiscoveryNode node, String cluster) {
+        RemoteClusterConnection connection = remoteClusters.get(cluster);
+        if (connection == null) {
+            throw new IllegalArgumentException("no such remote cluster: " + cluster);
+        }
+        return connection.getConnection(node);
+    }
+
+    void updateRemoteCluster(String clusterAlias, List<InetSocketAddress> addresses) {
+        updateRemoteClusters(Collections.singletonMap(clusterAlias, addresses.stream().map(address -> {
+                TransportAddress transportAddress = new TransportAddress(address);
+                return new DiscoveryNode(clusterAlias + "#" + transportAddress.toString(),
+                    transportAddress,
+                    Version.CURRENT.minimumCompatibilityVersion());
+            }).collect(Collectors.toList())),
+            ActionListener.wrap((x) -> {}, (x) -> {}) );
+    }
+
+    static Map<String, List<DiscoveryNode>> buildRemoteClustersSeeds(Settings settings) {
+        Stream<Setting<List<InetSocketAddress>>> allConcreteSettings = REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings);
+        return allConcreteSettings.collect(
+            Collectors.toMap(REMOTE_CLUSTERS_SEEDS::getNamespace,  concreteSetting -> {
+            String clusterName = REMOTE_CLUSTERS_SEEDS.getNamespace(concreteSetting);
+            List<DiscoveryNode> nodes = new ArrayList<>();
+            for (InetSocketAddress address : concreteSetting.get(settings)) {
+                TransportAddress transportAddress = new TransportAddress(address);
+                DiscoveryNode node = new DiscoveryNode(clusterName + "#" + transportAddress.toString(),
+                    transportAddress,
+                    Version.CURRENT.minimumCompatibilityVersion());
+                nodes.add(node);
+            }
+            return nodes;
+        }));
+    }
+
+    private static InetSocketAddress parseSeedAddress(String remoteHost) {
+        int portSeparator = remoteHost.lastIndexOf(':'); // in case we have a IPv6 address ie. [::1]:9300
+        if (portSeparator == -1 || portSeparator == remoteHost.length()) {
+            throw new IllegalArgumentException("remote hosts need to be configured as [host:port], found [" + remoteHost + "] instead");
+        }
+        String host = remoteHost.substring(0, portSeparator);
+        InetAddress hostAddress;
+        try {
+            hostAddress = InetAddress.getByName(host);
+        } catch (UnknownHostException e) {
+            throw new IllegalArgumentException("unknown host [" + host + "]", e);
+        }
+        try {
+            int port = Integer.valueOf(remoteHost.substring(portSeparator + 1));
+            if (port <= 0) {
+                throw new IllegalArgumentException("port number must be > 0 but was: [" + port + "]");
+            }
+            return new InetSocketAddress(hostAddress, port);
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("port must be a number", e);
+        }
+
+    }
+
+    /**
+     * Connects to all remote clusters in a blocking fashion. This should be called on node startup to establish an initial connection
+     * to all configured seed nodes.
+     */
+    void initializeRemoteClusters() {
+        final TimeValue timeValue = REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING.get(settings);
+        final PlainActionFuture<Void> future = new PlainActionFuture<>();
+        Map<String, List<DiscoveryNode>> seeds = buildRemoteClustersSeeds(settings);
+        updateRemoteClusters(seeds, future);
+        try {
+            future.get(timeValue.millis(), TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } catch (TimeoutException ex) {
+            logger.warn("failed to connect to remote clusters within {}", timeValue.toString());
+        } catch (Exception e) {
+            throw new IllegalStateException("failed to connect to remote clusters", e);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        IOUtils.close(remoteClusters.values());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java b/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
index 488132fdda2..76d4ac11413 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ScrollIdForNode.java
@@ -23,7 +23,7 @@ class ScrollIdForNode {
     private final String node;
     private final long scrollId;
 
-    public ScrollIdForNode(String node, long scrollId) {
+    ScrollIdForNode(String node, long scrollId) {
         this.node = node;
         this.scrollId = scrollId;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
deleted file mode 100644
index 9db3a21c485..00000000000
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.search;
-
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
-import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.QueryFetchSearchResult;
-import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
-import org.elasticsearch.search.query.QuerySearchRequest;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
-
-class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
-
-    private final AtomicArray<QueryFetchSearchResult> queryFetchResults;
-    private final SearchPhaseController searchPhaseController;
-    SearchDfsQueryAndFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                      Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
-                                      Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
-                                      SearchPhaseController searchPhaseController, Executor executor, SearchRequest request,
-                                      ActionListener<SearchResponse> listener,  GroupShardsIterator shardsIts,
-                                      long startTime, long clusterStateVersion, SearchTask task) {
-        super(logger, searchTransportService, nodeIdToDiscoveryNode, aliasFilter, concreteIndexBoosts, executor,
-                request, listener, shardsIts, startTime, clusterStateVersion, task);
-        this.searchPhaseController = searchPhaseController;
-        queryFetchResults = new AtomicArray<>(firstResults.length());
-    }
-
-    @Override
-    protected String firstPhaseName() {
-        return "dfs";
-    }
-
-    @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
-                                         ActionListener<DfsSearchResult> listener) {
-        searchTransportService.sendExecuteDfs(node, request, task, listener);
-    }
-
-    @Override
-    protected void moveToSecondPhase() {
-        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
-        final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
-
-        for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
-            DfsSearchResult dfsResult = entry.value;
-            DiscoveryNode node = nodeIdToDiscoveryNode.apply(dfsResult.shardTarget().nodeId());
-            QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
-            executeSecondPhase(entry.index, dfsResult, counter, node, querySearchRequest);
-        }
-    }
-
-    void executeSecondPhase(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter,
-                            final DiscoveryNode node, final QuerySearchRequest querySearchRequest) {
-        searchTransportService.sendExecuteFetch(node, querySearchRequest, task, new ActionListener<QueryFetchSearchResult>() {
-            @Override
-            public void onResponse(QueryFetchSearchResult result) {
-                result.shardTarget(dfsResult.shardTarget());
-                queryFetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                try {
-                    onSecondPhaseFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
-                } finally {
-                    // the query might not have been executed at all (for example because thread pool rejected execution)
-                    // and the search context that was created in dfs phase might not be released.
-                    // release it again to be in the safe side
-                    sendReleaseSearchContext(querySearchRequest.id(), node);
-                }
-            }
-        });
-    }
-
-    void onSecondPhaseFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
-                              AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", querySearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        getExecutor().execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                sortedShardDocs = searchPhaseController.sortDocs(true, queryFetchResults);
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(true, sortedShardDocs, queryFetchResults,
-                    queryFetchResults);
-                String scrollId = null;
-                if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
-                }
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("query_fetch", "", e, buildShardFailures());
-                if (logger.isDebugEnabled()) {
-                    logger.debug("failed to reduce search", failure);
-                }
-                super.onFailure(e);
-            }
-        });
-
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 3fe24cc9911..9cc686507f5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -19,209 +19,109 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 import org.elasticsearch.search.dfs.DfsSearchResult;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.search.query.QuerySearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
+import org.elasticsearch.search.query.QuerySearchResultProvider;
+import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
-class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
-
-    final AtomicArray<QuerySearchResult> queryResults;
-    final AtomicArray<FetchSearchResult> fetchResults;
-    final AtomicArray<IntArrayList> docIdsToLoad;
-    private final SearchPhaseController searchPhaseController;
+final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSearchResult> {
 
     SearchDfsQueryThenFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                       Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
+                                       Function<String, Transport.Connection> nodeIdToConnection,
                                        Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
                                        SearchPhaseController searchPhaseController, Executor executor, SearchRequest request,
                                        ActionListener<SearchResponse> listener, GroupShardsIterator shardsIts, long startTime,
                                        long clusterStateVersion, SearchTask task) {
-        super(logger, searchTransportService, nodeIdToDiscoveryNode, aliasFilter, concreteIndexBoosts, executor,
+        super(logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, searchPhaseController, executor,
                 request, listener, shardsIts, startTime, clusterStateVersion, task);
-        this.searchPhaseController = searchPhaseController;
-        queryResults = new AtomicArray<>(firstResults.length());
-        fetchResults = new AtomicArray<>(firstResults.length());
-        docIdsToLoad = new AtomicArray<>(firstResults.length());
     }
 
     @Override
-    protected String firstPhaseName() {
+    protected String initialPhaseName() {
         return "dfs";
     }
 
     @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
+    protected void sendExecuteFirstPhase(Transport.Connection connection, ShardSearchTransportRequest request,
                                          ActionListener<DfsSearchResult> listener) {
-        searchTransportService.sendExecuteDfs(node, request, task, listener);
+        searchTransportService.sendExecuteDfs(connection, request, task, listener);
     }
 
     @Override
-    protected void moveToSecondPhase() {
-        final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
-        final AtomicInteger counter = new AtomicInteger(firstResults.asList().size());
-        for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
-            DfsSearchResult dfsResult = entry.value;
-            DiscoveryNode node = nodeIdToDiscoveryNode.apply(dfsResult.shardTarget().nodeId());
-            QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
-            executeQuery(entry.index, dfsResult, counter, querySearchRequest, node);
-        }
+    protected CheckedRunnable<Exception> getNextPhase(AtomicArray<DfsSearchResult> initialResults) {
+        return new DfsQueryPhase(initialResults, searchPhaseController,
+            (queryResults) -> new FetchPhase(queryResults, searchPhaseController));
     }
 
-    void executeQuery(final int shardIndex, final DfsSearchResult dfsResult, final AtomicInteger counter,
-                      final QuerySearchRequest querySearchRequest, final DiscoveryNode node) {
-        searchTransportService.sendExecuteQuery(node, querySearchRequest, task, new ActionListener<QuerySearchResult>() {
-            @Override
-            public void onResponse(QuerySearchResult result) {
-                result.shardTarget(dfsResult.shardTarget());
-                queryResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    executeFetchPhase();
-                }
-            }
+    private final class DfsQueryPhase implements CheckedRunnable<Exception> {
+        private final AtomicArray<QuerySearchResultProvider> queryResult;
+        private final SearchPhaseController searchPhaseController;
+        private final AtomicArray<DfsSearchResult> firstResults;
+        private final Function<AtomicArray<QuerySearchResultProvider>, CheckedRunnable<Exception>> nextPhaseFactory;
 
-            @Override
-            public void onFailure(Exception t) {
-                try {
-                    onQueryFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
-                } finally {
-                    // the query might not have been executed at all (for example because thread pool rejected
-                    // execution) and the search context that was created in dfs phase might not be released.
-                    // release it again to be in the safe side
-                    sendReleaseSearchContext(querySearchRequest.id(), node);
-                }
-            }
-        });
-    }
-
-    void onQueryFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
-                        AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase", querySearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", buildShardFailures()));
-            } else {
-                executeFetchPhase();
-            }
-        }
-    }
-
-    void executeFetchPhase() {
-        try {
-            innerExecuteFetchPhase();
-        } catch (Exception e) {
-            listener.onFailure(new ReduceSearchPhaseException("query", "", e, buildShardFailures()));
-        }
-    }
-
-    void innerExecuteFetchPhase() throws Exception {
-        final boolean isScrollRequest = request.scroll() != null;
-        sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, queryResults);
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
-            finishHim();
-            return;
+        DfsQueryPhase(AtomicArray<DfsSearchResult> firstResults,
+                             SearchPhaseController searchPhaseController,
+                             Function<AtomicArray<QuerySearchResultProvider>, CheckedRunnable<Exception>> nextPhaseFactory) {
+            this.queryResult = new AtomicArray<>(firstResults.length());
+            this.searchPhaseController = searchPhaseController;
+            this.firstResults = firstResults;
+            this.nextPhaseFactory = nextPhaseFactory;
         }
 
-        final ScoreDoc[] lastEmittedDocPerShard = (request.scroll() != null) ?
-            searchPhaseController.getLastEmittedDocPerShard(queryResults.asList(), sortedShardDocs, firstResults.length()) : null;
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (final AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            QuerySearchResult queryResult = queryResults.get(entry.index);
-            DiscoveryNode node = nodeIdToDiscoveryNode.apply(queryResult.shardTarget().nodeId());
-            ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult, entry, lastEmittedDocPerShard);
-            executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
-        }
-    }
-
-    void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
-                      final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
-        searchTransportService.sendExecuteFetch(node, fetchSearchRequest, task, new ActionListener<FetchSearchResult>() {
-            @Override
-            public void onResponse(FetchSearchResult result) {
-                result.shardTarget(shardTarget);
-                fetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                // the search context might not be cleared on the node where the fetch was executed for example
-                // because the action was rejected by the thread pool. in this case we need to send a dedicated
-                // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
-                // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
-                docIdsToLoad.set(shardIndex, null);
-                onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
-            }
-        });
-    }
-
-    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex,
-                        SearchShardTarget shardTarget, AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, shardTarget, e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        getExecutor().execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, queryResults,
-                    fetchResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-                releaseIrrelevantSearchContexts(queryResults, docIdsToLoad);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("failed to reduce search", failure);
+        @Override
+        public void run() throws Exception {
+            final AggregatedDfs dfs = searchPhaseController.aggregateDfs(firstResults);
+            final CountedCollector<QuerySearchResultProvider> counter = new CountedCollector<>(queryResult, firstResults.asList().size(),
+                (successfulOps) -> {
+                    if (successfulOps == 0) {
+                        listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", buildShardFailures()));
+                    } else {
+                        executePhase("fetch", this.nextPhaseFactory.apply(queryResult), null);
                     }
-                    super.onFailure(failure);
-                } finally {
-                    releaseIrrelevantSearchContexts(queryResults, docIdsToLoad);
-                }
+                });
+            for (final AtomicArray.Entry<DfsSearchResult> entry : firstResults.asList()) {
+                DfsSearchResult dfsResult = entry.value;
+                final int shardIndex = entry.index;
+                Transport.Connection connection = nodeIdToConnection.apply(dfsResult.shardTarget().getNodeId());
+                QuerySearchRequest querySearchRequest = new QuerySearchRequest(request, dfsResult.id(), dfs);
+                searchTransportService.sendExecuteQuery(connection, querySearchRequest, task, new ActionListener<QuerySearchResult>() {
+                    @Override
+                    public void onResponse(QuerySearchResult result) {
+                        counter.onResult(shardIndex, result, dfsResult.shardTarget());
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        try {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute query phase",
+                                    querySearchRequest.id()), e);
+                            }
+                            counter.onFailure(shardIndex, dfsResult.shardTarget(), e);
+                        } finally {
+                            // the query might not have been executed at all (for example because thread pool rejected
+                            // execution) and the search context that was created in dfs phase might not be released.
+                            // release it again to be in the safe side
+                            sendReleaseSearchContext(querySearchRequest.id(), connection);
+                        }
+                    }
+                });
             }
-        });
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
index 92270c6fe36..f59d7fe50db 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java
@@ -30,11 +30,16 @@ import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IntsRef;
 import org.elasticsearch.common.collect.HppcMaps;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.ArrayUtils;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.IntArray;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.InternalAggregation;
@@ -63,17 +68,19 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
 public class SearchPhaseController extends AbstractComponent {
 
     private static final Comparator<AtomicArray.Entry<? extends QuerySearchResultProvider>> QUERY_RESULT_ORDERING = (o1, o2) -> {
-        int i = o1.value.shardTarget().index().compareTo(o2.value.shardTarget().index());
+        int i = o1.value.shardTarget().getIndex().compareTo(o2.value.shardTarget().getIndex());
         if (i == 0) {
-            i = o1.value.shardTarget().shardId().id() - o2.value.shardTarget().shardId().id();
+            i = o1.value.shardTarget().getShardId().id() - o2.value.shardTarget().getShardId().id();
         }
         return i;
     };
@@ -82,11 +89,25 @@ public class SearchPhaseController extends AbstractComponent {
 
     private final BigArrays bigArrays;
     private final ScriptService scriptService;
+    private final List<BiConsumer<SearchRequest, SearchResponse> > searchResponseListener;
 
     public SearchPhaseController(Settings settings, BigArrays bigArrays, ScriptService scriptService) {
+        this(settings, bigArrays, scriptService, Collections.emptyList());
+    }
+
+    public SearchPhaseController(Settings settings, BigArrays bigArrays, ScriptService scriptService,
+                                 List<BiConsumer<SearchRequest, SearchResponse> > searchResponseListener) {
         super(settings);
         this.bigArrays = bigArrays;
         this.scriptService = scriptService;
+        this.searchResponseListener = searchResponseListener;
+    }
+
+    /**
+     * Returns the search response listeners registry
+     */
+    public List<BiConsumer<SearchRequest, SearchResponse> > getSearchResponseListener() {
+        return searchResponseListener;
     }
 
     public AggregatedDfs aggregateDfs(AtomicArray<DfsSearchResult> results) {
@@ -236,7 +257,26 @@ public class SearchPhaseController extends AbstractComponent {
         }
 
         final TopDocs mergedTopDocs;
-        if (firstResult.queryResult().topDocs() instanceof TopFieldDocs) {
+        int numShards = resultsArr.length();
+        if (firstResult.queryResult().topDocs() instanceof CollapseTopFieldDocs) {
+            CollapseTopFieldDocs firstTopDocs = (CollapseTopFieldDocs) firstResult.queryResult().topDocs();
+            final Sort sort = new Sort(firstTopDocs.fields);
+
+            final CollapseTopFieldDocs[] shardTopDocs = new CollapseTopFieldDocs[numShards];
+            for (AtomicArray.Entry<? extends QuerySearchResultProvider> sortedResult : sortedResults) {
+                TopDocs topDocs = sortedResult.value.queryResult().topDocs();
+                // the 'index' field is the position in the resultsArr atomic array
+                shardTopDocs[sortedResult.index] = (CollapseTopFieldDocs) topDocs;
+            }
+            // TopDocs#merge can't deal with null shard TopDocs
+            for (int i = 0; i < shardTopDocs.length; ++i) {
+                if (shardTopDocs[i] == null) {
+                    shardTopDocs[i] = new CollapseTopFieldDocs(firstTopDocs.field, 0, new FieldDoc[0],
+                        sort.getSort(), new Object[0], Float.NaN);
+                }
+            }
+            mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs);
+        } else if (firstResult.queryResult().topDocs() instanceof TopFieldDocs) {
             TopFieldDocs firstTopDocs = (TopFieldDocs) firstResult.queryResult().topDocs();
             final Sort sort = new Sort(firstTopDocs.fields);
 
@@ -316,6 +356,8 @@ public class SearchPhaseController extends AbstractComponent {
             }
             // from is always zero as when we use scroll, we ignore from
             long size = Math.min(fetchHits, topN(queryResults));
+            // with collapsing we can have more hits than sorted docs
+            size = Math.min(sortedScoreDocs.length, size);
             for (int sortedDocsIndex = 0; sortedDocsIndex < size; sortedDocsIndex++) {
                 ScoreDoc scoreDoc = sortedScoreDocs[sortedDocsIndex];
                 lastEmittedDocPerShard[scoreDoc.shardIndex] = scoreDoc;
@@ -328,15 +370,16 @@ public class SearchPhaseController extends AbstractComponent {
     /**
      * Builds an array, with potential null elements, with docs to load.
      */
-    public void fillDocIdsToLoad(AtomicArray<IntArrayList> docIdsToLoad, ScoreDoc[] shardDocs) {
+    public IntArrayList[] fillDocIdsToLoad(int numShards, ScoreDoc[] shardDocs) {
+        IntArrayList[] docIdsToLoad = new IntArrayList[numShards];
         for (ScoreDoc shardDoc : shardDocs) {
-            IntArrayList shardDocIdsToLoad = docIdsToLoad.get(shardDoc.shardIndex);
+            IntArrayList shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex];
             if (shardDocIdsToLoad == null) {
-                shardDocIdsToLoad = new IntArrayList(); // can't be shared!, uses unsafe on it later on
-                docIdsToLoad.set(shardDoc.shardIndex, shardDocIdsToLoad);
+                shardDocIdsToLoad = docIdsToLoad[shardDoc.shardIndex] = new IntArrayList();
             }
             shardDocIdsToLoad.add(shardDoc.doc);
         }
+        return docIdsToLoad;
     }
 
     /**
@@ -362,11 +405,16 @@ public class SearchPhaseController extends AbstractComponent {
         boolean sorted = false;
         int sortScoreIndex = -1;
         if (firstResult.topDocs() instanceof TopFieldDocs) {
-            sorted = true;
             TopFieldDocs fieldDocs = (TopFieldDocs) firstResult.queryResult().topDocs();
-            for (int i = 0; i < fieldDocs.fields.length; i++) {
-                if (fieldDocs.fields[i].getType() == SortField.Type.SCORE) {
-                    sortScoreIndex = i;
+            if (fieldDocs instanceof CollapseTopFieldDocs &&
+                fieldDocs.fields.length == 1 && fieldDocs.fields[0].getType() == SortField.Type.SCORE) {
+                sorted = false;
+            } else {
+                sorted = true;
+                for (int i = 0; i < fieldDocs.fields.length; i++) {
+                    if (fieldDocs.fields[i].getType() == SortField.Type.SCORE) {
+                        sortScoreIndex = i;
+                    }
                 }
             }
         }
@@ -405,6 +453,8 @@ public class SearchPhaseController extends AbstractComponent {
         }
         int from = ignoreFrom ? 0 : firstResult.queryResult().from();
         int numSearchHits = (int) Math.min(fetchHits - from, topN(queryResults));
+        // with collapsing we can have more fetch hits than sorted docs
+        numSearchHits = Math.min(sortedDocs.length, numSearchHits);
         // merge hits
         List<InternalSearchHit> hits = new ArrayList<>();
         if (!fetchResults.isEmpty()) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index 515d3204fb6..c6e0b21dffd 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -103,6 +103,7 @@ public class SearchPhaseExecutionException extends ElasticsearchException {
         return shardFailures;
     }
 
+    @Override
     public Throwable getCause() {
         Throwable cause = super.getCause();
         if (cause == null) {
@@ -131,28 +132,34 @@ public class SearchPhaseExecutionException extends ElasticsearchException {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("phase", phaseName);
         final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
         builder.field("grouped", group); // notify that it's grouped
         builder.field("failed_shards");
         builder.startArray();
-        ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ? ExceptionsHelper.groupBy(shardFailures) : shardFailures;
+        ShardOperationFailedException[] failures = params.paramAsBoolean("group_shard_failures", true) ?
+                ExceptionsHelper.groupBy(shardFailures) : shardFailures;
         for (ShardOperationFailedException failure : failures) {
             builder.startObject();
             failure.toXContent(builder, params);
             builder.endObject();
         }
         builder.endArray();
-        super.innerToXContent(builder, params);
     }
 
     @Override
-    protected void causeToXContent(XContentBuilder builder, Params params) throws IOException {
-        if (super.getCause() != null) {
-            // if the cause is null we inject a guessed root cause that will then be rendered twice so wi disable it manually
-            super.causeToXContent(builder, params);
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        Throwable ex = ExceptionsHelper.unwrapCause(this);
+        if (ex != this) {
+            generateThrowableXContent(builder, params, this);
+        } else {
+            // We don't have a cause when all shards failed, but we do have shards failures so we can "guess" a cause
+            // (see {@link #getCause()}). Here, we use super.getCause() because we don't want the guessed exception to
+            // be rendered twice (one in the "cause" field, one in "failed_shards")
+            innerToXContent(builder, params, this, getExceptionName(), getMessage(), getHeaders(), getMetadata(), super.getCause());
         }
+        return builder;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
index f597ede64bc..34779684132 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
@@ -21,69 +21,44 @@ package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.common.CheckedRunnable;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.fetch.QueryFetchSearchResult;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
+import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.function.Function;
 
-class SearchQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<QueryFetchSearchResult> {
-
-    private final SearchPhaseController searchPhaseController;
+final class SearchQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<QueryFetchSearchResult> {
 
     SearchQueryAndFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                   Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
+                                   Function<String, Transport.Connection> nodeIdToConnection,
                                    Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
                                    SearchPhaseController searchPhaseController, Executor executor,
                                    SearchRequest request, ActionListener<SearchResponse> listener,
                                    GroupShardsIterator shardsIts, long startTime, long clusterStateVersion,
                                    SearchTask task) {
-        super(logger, searchTransportService, nodeIdToDiscoveryNode, aliasFilter, concreteIndexBoosts, executor,
+        super(logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, searchPhaseController, executor,
                 request, listener, shardsIts, startTime, clusterStateVersion, task);
-        this.searchPhaseController = searchPhaseController;
-
     }
 
     @Override
-    protected String firstPhaseName() {
+    protected String initialPhaseName() {
         return "query_fetch";
     }
 
     @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
+    protected void sendExecuteFirstPhase(Transport.Connection connection, ShardSearchTransportRequest request,
                                          ActionListener<QueryFetchSearchResult> listener) {
-        searchTransportService.sendExecuteFetch(node, request, task, listener);
+        searchTransportService.sendExecuteFetch(connection, request, task, listener);
     }
 
     @Override
-    protected void moveToSecondPhase() throws Exception {
-        getExecutor().execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, firstResults);
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, firstResults,
-                    firstResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
-                    buildTookInMillis(), buildShardFailures()));
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
-                if (logger.isDebugEnabled()) {
-                    logger.debug("failed to reduce search", failure);
-                }
-                super.onFailure(failure);
-            }
-        });
+    protected CheckedRunnable<Exception> getNextPhase(AtomicArray<QueryFetchSearchResult> initialResults) {
+        return () -> sendResponseAsync("fetch", searchPhaseController, null, initialResults, initialResults);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 7b300063291..a219f183398 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -19,144 +19,46 @@
 
 package org.elasticsearch.action.search;
 
-import com.carrotsearch.hppc.IntArrayList;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.apache.lucene.search.ScoreDoc;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.ShardFetchSearchRequest;
 import org.elasticsearch.search.internal.AliasFilter;
-import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.search.query.QuerySearchResultProvider;
+import org.elasticsearch.transport.Transport;
 
-import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 
-class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySearchResultProvider> {
-
-    final AtomicArray<FetchSearchResult> fetchResults;
-    final AtomicArray<IntArrayList> docIdsToLoad;
-    private final SearchPhaseController searchPhaseController;
+final class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySearchResultProvider> {
 
     SearchQueryThenFetchAsyncAction(Logger logger, SearchTransportService searchTransportService,
-                                    Function<String, DiscoveryNode> nodeIdToDiscoveryNode,
+                                    Function<String, Transport.Connection> nodeIdToConnection,
                                     Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts,
                                     SearchPhaseController searchPhaseController, Executor executor,
                                     SearchRequest request, ActionListener<SearchResponse> listener,
                                     GroupShardsIterator shardsIts, long startTime, long clusterStateVersion,
                                     SearchTask task) {
-        super(logger, searchTransportService, nodeIdToDiscoveryNode, aliasFilter, concreteIndexBoosts, executor, request, listener,
-            shardsIts, startTime, clusterStateVersion, task);
-        this.searchPhaseController = searchPhaseController;
-        fetchResults = new AtomicArray<>(firstResults.length());
-        docIdsToLoad = new AtomicArray<>(firstResults.length());
+        super(logger, searchTransportService, nodeIdToConnection, aliasFilter, concreteIndexBoosts, searchPhaseController, executor,
+            request, listener, shardsIts, startTime, clusterStateVersion, task);
     }
 
     @Override
-    protected String firstPhaseName() {
+    protected String initialPhaseName() {
         return "query";
     }
 
     @Override
-    protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request,
+    protected void sendExecuteFirstPhase(Transport.Connection connection, ShardSearchTransportRequest request,
                                          ActionListener<QuerySearchResultProvider> listener) {
-        searchTransportService.sendExecuteQuery(node, request, task, listener);
+        searchTransportService.sendExecuteQuery(connection, request, task, listener);
     }
 
     @Override
-    protected void moveToSecondPhase() throws Exception {
-        final boolean isScrollRequest = request.scroll() != null;
-        sortedShardDocs = searchPhaseController.sortDocs(isScrollRequest, firstResults);
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
-            finishHim();
-            return;
-        }
-
-        final ScoreDoc[] lastEmittedDocPerShard = isScrollRequest ?
-            searchPhaseController.getLastEmittedDocPerShard(firstResults.asList(), sortedShardDocs, firstResults.length()) : null;
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            QuerySearchResultProvider queryResult = firstResults.get(entry.index);
-            DiscoveryNode node = nodeIdToDiscoveryNode.apply(queryResult.shardTarget().nodeId());
-            ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult(), entry, lastEmittedDocPerShard);
-            executeFetch(entry.index, queryResult.shardTarget(), counter, fetchSearchRequest, node);
-        }
-    }
-
-    void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
-                      final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
-        searchTransportService.sendExecuteFetch(node, fetchSearchRequest, task, new ActionListener<FetchSearchResult>() {
-            @Override
-            public void onResponse(FetchSearchResult result) {
-                result.shardTarget(shardTarget);
-                fetchResults.set(shardIndex, result);
-                if (counter.decrementAndGet() == 0) {
-                    finishHim();
-                }
-            }
-
-            @Override
-            public void onFailure(Exception t) {
-                // the search context might not be cleared on the node where the fetch was executed for example
-                // because the action was rejected by the thread pool. in this case we need to send a dedicated
-                // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
-                // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
-                docIdsToLoad.set(shardIndex, null);
-                onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
-            }
-        });
-    }
-
-    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex, SearchShardTarget shardTarget,
-                        AtomicInteger counter) {
-        if (logger.isDebugEnabled()) {
-            logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e);
-        }
-        this.addShardFailure(shardIndex, shardTarget, e);
-        successfulOps.decrementAndGet();
-        if (counter.decrementAndGet() == 0) {
-            finishHim();
-        }
-    }
-
-    private void finishHim() {
-        getExecutor().execute(new ActionRunnable<SearchResponse>(listener) {
-            @Override
-            public void doRun() throws IOException {
-                final boolean isScrollRequest = request.scroll() != null;
-                final InternalSearchResponse internalResponse = searchPhaseController.merge(isScrollRequest, sortedShardDocs, firstResults,
-                    fetchResults);
-                String scrollId = isScrollRequest ? TransportSearchHelper.buildScrollId(request.searchType(), firstResults) : null;
-                listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps,
-                    successfulOps.get(), buildTookInMillis(), buildShardFailures()));
-                releaseIrrelevantSearchContexts(firstResults, docIdsToLoad);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures());
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("failed to reduce search", failure);
-                    }
-                    super.onFailure(failure);
-                } finally {
-                    releaseIrrelevantSearchContexts(firstResults, docIdsToLoad);
-                }
-            }
-        });
+    protected CheckedRunnable<Exception> getNextPhase(AtomicArray<QuerySearchResultProvider> initialResults) {
+        return new FetchPhase(initialResults, searchPhaseController);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index b21b0c54be3..9c69f1a763f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -199,7 +198,7 @@ public final class SearchRequest extends ActionRequest implements IndicesRequest
      * "query_then_fetch"/"queryThenFetch", and "query_and_fetch"/"queryAndFetch".
      */
     public SearchRequest searchType(String searchType) {
-        return searchType(SearchType.fromString(searchType, ParseFieldMatcher.EMPTY));
+        return searchType(SearchType.fromString(searchType));
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 3c320447fe8..865cf01430f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -26,13 +26,14 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
@@ -503,6 +504,11 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    public SearchRequestBuilder setCollapse(CollapseBuilder collapse) {
+        sourceBuilder().collapse(collapse);
+        return this;
+    }
+
     @Override
     public String toString() {
         if (request.source() != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 3135d2c8f53..4b8ba5e64b6 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
@@ -44,7 +44,7 @@ import static org.elasticsearch.search.internal.InternalSearchResponse.readInter
 /**
  * A response of a search request.
  */
-public class SearchResponse extends ActionResponse implements StatusToXContent {
+public class SearchResponse extends ActionResponse implements StatusToXContentObject {
 
     private InternalSearchResponse internalResponse;
 
@@ -181,6 +181,13 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         if (scrollId != null) {
             builder.field(Fields._SCROLL_ID, scrollId);
         }
@@ -231,6 +238,6 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
 
     @Override
     public String toString() {
-        return Strings.toString(this, true);
+        return Strings.toString(this);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 851e3343bc2..f7c213c6f1f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -36,6 +36,7 @@ import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -168,45 +169,50 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
 
     private void executeFetchPhase() throws Exception {
         sortedShardDocs = searchPhaseController.sortDocs(true, queryResults);
-        AtomicArray<IntArrayList> docIdsToLoad = new AtomicArray<>(queryResults.length());
-        searchPhaseController.fillDocIdsToLoad(docIdsToLoad, sortedShardDocs);
-
-        if (docIdsToLoad.asList().isEmpty()) {
+        if (sortedShardDocs.length == 0) {
             finishHim();
             return;
         }
 
-
+        final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(queryResults.length(), sortedShardDocs);
         final ScoreDoc[] lastEmittedDocPerShard = searchPhaseController.getLastEmittedDocPerShard(queryResults.asList(),
             sortedShardDocs, queryResults.length());
-        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.asList().size());
-        for (final AtomicArray.Entry<IntArrayList> entry : docIdsToLoad.asList()) {
-            IntArrayList docIds = entry.value;
-            final QuerySearchResult querySearchResult = queryResults.get(entry.index);
-            ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[entry.index];
-            ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.id(), docIds, lastEmittedDoc);
-            DiscoveryNode node = nodes.get(querySearchResult.shardTarget().nodeId());
-            searchTransportService.sendExecuteFetchScroll(node, shardFetchRequest, task, new ActionListener<FetchSearchResult>() {
-                @Override
-                public void onResponse(FetchSearchResult result) {
-                    result.shardTarget(querySearchResult.shardTarget());
-                    fetchResults.set(entry.index, result);
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
+        final AtomicInteger counter = new AtomicInteger(docIdsToLoad.length);
+        for (int i = 0; i < docIdsToLoad.length; i++) {
+            final int index = i;
+            final IntArrayList docIds = docIdsToLoad[index];
+            if (docIds != null) {
+                final QuerySearchResult querySearchResult = queryResults.get(index);
+                ScoreDoc lastEmittedDoc = lastEmittedDocPerShard[index];
+                ShardFetchRequest shardFetchRequest = new ShardFetchRequest(querySearchResult.id(), docIds, lastEmittedDoc);
+                DiscoveryNode node = nodes.get(querySearchResult.shardTarget().getNodeId());
+                searchTransportService.sendExecuteFetchScroll(node, shardFetchRequest, task, new ActionListener<FetchSearchResult>() {
+                    @Override
+                    public void onResponse(FetchSearchResult result) {
+                        result.shardTarget(querySearchResult.shardTarget());
+                        fetchResults.set(index, result);
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
+                        }
                     }
-                }
 
-                @Override
-                public void onFailure(Exception t) {
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("Failed to execute fetch phase", t);
-                    }
-                    successfulOps.decrementAndGet();
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
+                    @Override
+                    public void onFailure(Exception t) {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Failed to execute fetch phase", t);
+                        }
+                        successfulOps.decrementAndGet();
+                        if (counter.decrementAndGet() == 0) {
+                            finishHim();
+                        }
                     }
+                });
+            } else {
+                // the counter is set to the total size of docIdsToLoad which can have null values so we have to count them down too
+                if (counter.decrementAndGet() == 0) {
+                    finishHim();
                 }
-            });
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchTask.java b/core/src/main/java/org/elasticsearch/action/search/SearchTask.java
index 24f94a43319..d0a1cdd456f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchTask.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchTask.java
@@ -31,4 +31,9 @@ public class SearchTask extends CancellableTask {
         super(id, type, action, description, parentTaskId);
     }
 
+    @Override
+    public boolean shouldCancelChildrenOnCancellation() {
+        return true;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
index 5b052132566..c62699b6bab 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchTransportService.java
@@ -25,9 +25,10 @@ import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.dfs.DfsSearchResult;
@@ -44,19 +45,23 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportActionProxy;
 import org.elasticsearch.transport.TaskAwareTransportRequestHandler;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.function.Consumer;
 
 /**
  * An encapsulation of {@link org.elasticsearch.search.SearchService} operations exposed through
  * transport.
  */
-public class SearchTransportService extends AbstractComponent {
+public class SearchTransportService extends AbstractLifecycleComponent {
 
     public static final String FREE_CONTEXT_SCROLL_ACTION_NAME = "indices:data/read/search[free_context/scroll]";
     public static final String FREE_CONTEXT_ACTION_NAME = "indices:data/read/search[free_context]";
@@ -66,21 +71,24 @@ public class SearchTransportService extends AbstractComponent {
     public static final String QUERY_ID_ACTION_NAME = "indices:data/read/search[phase/query/id]";
     public static final String QUERY_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query/scroll]";
     public static final String QUERY_FETCH_ACTION_NAME = "indices:data/read/search[phase/query+fetch]";
-    public static final String QUERY_QUERY_FETCH_ACTION_NAME = "indices:data/read/search[phase/query/query+fetch]";
     public static final String QUERY_FETCH_SCROLL_ACTION_NAME = "indices:data/read/search[phase/query+fetch/scroll]";
     public static final String FETCH_ID_SCROLL_ACTION_NAME = "indices:data/read/search[phase/fetch/id/scroll]";
     public static final String FETCH_ID_ACTION_NAME = "indices:data/read/search[phase/fetch/id]";
 
     private final TransportService transportService;
+    private final RemoteClusterService remoteClusterService;
 
-    public SearchTransportService(Settings settings, TransportService transportService) {
+    public SearchTransportService(Settings settings, ClusterSettings clusterSettings, TransportService transportService) {
         super(settings);
         this.transportService = transportService;
+        this.remoteClusterService = new RemoteClusterService(settings, transportService);
+        clusterSettings.addAffixUpdateConsumer(RemoteClusterService.REMOTE_CLUSTERS_SEEDS, remoteClusterService::updateRemoteCluster,
+            (namespace, value) -> {});
     }
 
-    public void sendFreeContext(DiscoveryNode node, final long contextId, SearchRequest request) {
-        transportService.sendRequest(node, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(request, contextId),
-            new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
+    public void sendFreeContext(Transport.Connection connection, final long contextId, SearchRequest request) {
+        transportService.sendRequest(connection, FREE_CONTEXT_ACTION_NAME, new SearchFreeContextRequest(request, contextId),
+            TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(new ActionListener<SearchFreeContextResponse>() {
                 @Override
                 public void onResponse(SearchFreeContextResponse response) {
                     // no need to respond if it was freed or not
@@ -103,64 +111,62 @@ public class SearchTransportService extends AbstractComponent {
             new ActionListenerResponseHandler<>(listener, () -> TransportResponse.Empty.INSTANCE));
     }
 
-    public void sendExecuteDfs(DiscoveryNode node, final ShardSearchTransportRequest request, SearchTask task,
+    public void sendExecuteDfs(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task,
                                final ActionListener<DfsSearchResult> listener) {
-        transportService.sendChildRequest(node, DFS_ACTION_NAME, request, task,
+        transportService.sendChildRequest(connection, DFS_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, DfsSearchResult::new));
     }
 
-    public void sendExecuteQuery(DiscoveryNode node, final ShardSearchTransportRequest request, SearchTask task,
+    public void sendExecuteQuery(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task,
                                  final ActionListener<QuerySearchResultProvider> listener) {
-        transportService.sendChildRequest(node, QUERY_ACTION_NAME, request, task,
+        transportService.sendChildRequest(connection, QUERY_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, QuerySearchResult::new));
     }
 
-    public void sendExecuteQuery(DiscoveryNode node, final QuerySearchRequest request, SearchTask task,
+    public void sendExecuteQuery(Transport.Connection connection, final QuerySearchRequest request, SearchTask task,
                                  final ActionListener<QuerySearchResult> listener) {
-        transportService.sendChildRequest(node, QUERY_ID_ACTION_NAME, request, task,
+        transportService.sendChildRequest(connection, QUERY_ID_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, QuerySearchResult::new));
     }
 
     public void sendExecuteQuery(DiscoveryNode node, final InternalScrollSearchRequest request, SearchTask task,
                                  final ActionListener<ScrollQuerySearchResult> listener) {
-        transportService.sendChildRequest(node, QUERY_SCROLL_ACTION_NAME, request, task,
+        transportService.sendChildRequest(transportService.getConnection(node), QUERY_SCROLL_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, ScrollQuerySearchResult::new));
     }
 
-    public void sendExecuteFetch(DiscoveryNode node, final ShardSearchTransportRequest request, SearchTask task,
+    public void sendExecuteFetch(Transport.Connection connection, final ShardSearchTransportRequest request, SearchTask task,
                                  final ActionListener<QueryFetchSearchResult> listener) {
-        transportService.sendChildRequest(node, QUERY_FETCH_ACTION_NAME, request, task,
-            new ActionListenerResponseHandler<>(listener, QueryFetchSearchResult::new));
-    }
-
-    public void sendExecuteFetch(DiscoveryNode node, final QuerySearchRequest request, SearchTask task,
-                                 final ActionListener<QueryFetchSearchResult> listener) {
-        transportService.sendChildRequest(node, QUERY_QUERY_FETCH_ACTION_NAME, request, task,
+        transportService.sendChildRequest(connection, QUERY_FETCH_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, QueryFetchSearchResult::new));
     }
 
     public void sendExecuteFetch(DiscoveryNode node, final InternalScrollSearchRequest request, SearchTask task,
                                  final ActionListener<ScrollQueryFetchSearchResult> listener) {
-        transportService.sendChildRequest(node, QUERY_FETCH_SCROLL_ACTION_NAME, request, task,
+        transportService.sendChildRequest(transportService.getConnection(node), QUERY_FETCH_SCROLL_ACTION_NAME, request, task,
             new ActionListenerResponseHandler<>(listener, ScrollQueryFetchSearchResult::new));
     }
 
-    public void sendExecuteFetch(DiscoveryNode node, final ShardFetchSearchRequest request, SearchTask task,
+    public void sendExecuteFetch(Transport.Connection connection, final ShardFetchSearchRequest request, SearchTask task,
                                  final ActionListener<FetchSearchResult> listener) {
-        sendExecuteFetch(node, FETCH_ID_ACTION_NAME, request, task, listener);
+        sendExecuteFetch(connection, FETCH_ID_ACTION_NAME, request, task, listener);
     }
 
     public void sendExecuteFetchScroll(DiscoveryNode node, final ShardFetchRequest request, SearchTask task,
                                        final ActionListener<FetchSearchResult> listener) {
-        sendExecuteFetch(node, FETCH_ID_SCROLL_ACTION_NAME, request, task, listener);
+        sendExecuteFetch(transportService.getConnection(node), FETCH_ID_SCROLL_ACTION_NAME, request, task, listener);
     }
 
-    private void sendExecuteFetch(DiscoveryNode node, String action, final ShardFetchRequest request, SearchTask task,
+    private void sendExecuteFetch(Transport.Connection connection, String action, final ShardFetchRequest request, SearchTask task,
                                   final ActionListener<FetchSearchResult> listener) {
-        transportService.sendChildRequest(node, action, request, task,
+        transportService.sendChildRequest(connection, action, request, task,
             new ActionListenerResponseHandler<>(listener, FetchSearchResult::new));
     }
 
+    public RemoteClusterService getRemoteClusterService() {
+        return remoteClusterService;
+    }
+
     static class ScrollFreeContextRequest extends TransportRequest {
         private long id;
 
@@ -191,7 +197,7 @@ public class SearchTransportService extends AbstractComponent {
     static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest {
         private OriginalIndices originalIndices;
 
-        public SearchFreeContextRequest() {
+        SearchFreeContextRequest() {
         }
 
         SearchFreeContextRequest(SearchRequest request, long id) {
@@ -265,6 +271,7 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(new SearchFreeContextResponse(freed));
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_SCROLL_ACTION_NAME, SearchFreeContextResponse::new);
         transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, SearchFreeContextRequest::new, ThreadPool.Names.SAME,
             new TaskAwareTransportRequestHandler<SearchFreeContextRequest>() {
                 @Override
@@ -273,6 +280,7 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(new SearchFreeContextResponse(freed));
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, SearchFreeContextResponse::new);
         transportService.registerRequestHandler(CLEAR_SCROLL_CONTEXTS_ACTION_NAME, () -> TransportRequest.Empty.INSTANCE,
             ThreadPool.Names.SAME,
             new TaskAwareTransportRequestHandler<TransportRequest.Empty>() {
@@ -282,6 +290,9 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(TransportResponse.Empty.INSTANCE);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, CLEAR_SCROLL_CONTEXTS_ACTION_NAME,
+            () -> TransportResponse.Empty.INSTANCE);
+
         transportService.registerRequestHandler(DFS_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
                 @Override
@@ -291,6 +302,8 @@ public class SearchTransportService extends AbstractComponent {
 
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, DFS_ACTION_NAME, DfsSearchResult::new);
+
         transportService.registerRequestHandler(QUERY_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
                 @Override
@@ -299,6 +312,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_ACTION_NAME, QuerySearchResult::new);
+
         transportService.registerRequestHandler(QUERY_ID_ACTION_NAME, QuerySearchRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<QuerySearchRequest>() {
                 @Override
@@ -307,6 +322,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_ID_ACTION_NAME, QuerySearchResult::new);
+
         transportService.registerRequestHandler(QUERY_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<InternalScrollSearchRequest>() {
                 @Override
@@ -315,6 +332,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_SCROLL_ACTION_NAME, ScrollQuerySearchResult::new);
+
         transportService.registerRequestHandler(QUERY_FETCH_ACTION_NAME, ShardSearchTransportRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<ShardSearchTransportRequest>() {
                 @Override
@@ -323,14 +342,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
-        transportService.registerRequestHandler(QUERY_QUERY_FETCH_ACTION_NAME, QuerySearchRequest::new, ThreadPool.Names.SEARCH,
-            new TaskAwareTransportRequestHandler<QuerySearchRequest>() {
-                @Override
-                public void messageReceived(QuerySearchRequest request, TransportChannel channel, Task task) throws Exception {
-                    QueryFetchSearchResult result = searchService.executeFetchPhase(request, (SearchTask)task);
-                    channel.sendResponse(result);
-                }
-            });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_ACTION_NAME, QueryFetchSearchResult::new);
+
         transportService.registerRequestHandler(QUERY_FETCH_SCROLL_ACTION_NAME, InternalScrollSearchRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<InternalScrollSearchRequest>() {
                 @Override
@@ -339,6 +352,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, QUERY_FETCH_SCROLL_ACTION_NAME, ScrollQueryFetchSearchResult::new);
+
         transportService.registerRequestHandler(FETCH_ID_SCROLL_ACTION_NAME, ShardFetchRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<ShardFetchRequest>() {
                 @Override
@@ -347,6 +362,8 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, FETCH_ID_SCROLL_ACTION_NAME, FetchSearchResult::new);
+
         transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH,
             new TaskAwareTransportRequestHandler<ShardFetchSearchRequest>() {
                 @Override
@@ -355,6 +372,24 @@ public class SearchTransportService extends AbstractComponent {
                     channel.sendResponse(result);
                 }
             });
+        TransportActionProxy.registerProxyAction(transportService, FETCH_ID_ACTION_NAME, FetchSearchResult::new);
+    }
 
+    Transport.Connection getConnection(DiscoveryNode node) {
+        return transportService.getConnection(node);
+    }
+
+    @Override
+    protected void doStart() {
+        // here we start to connect to the remote clusters
+        remoteClusterService.initializeRemoteClusters();
+    }
+
+    @Override
+    protected void doStop() {}
+
+    @Override
+    protected void doClose() throws IOException {
+        remoteClusterService.close();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchType.java b/core/src/main/java/org/elasticsearch/action/search/SearchType.java
index 31535736957..8230a9148b8 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchType.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchType.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-
 /**
  * Search type represent the manner at which the search operation is executed.
  *
@@ -39,11 +37,7 @@ public enum SearchType {
      * are fetched. This is very handy when the index has a lot of shards (not replicas, shard id groups).
      */
     QUERY_THEN_FETCH((byte) 1),
-    /**
-     * Same as {@link #QUERY_AND_FETCH}, except for an initial scatter phase which goes and computes the distributed
-     * term frequencies for more accurate scoring.
-     */
-    DFS_QUERY_AND_FETCH((byte) 2),
+    // 2 used to be DFS_QUERY_AND_FETCH
     /**
      * The most naive (and possibly fastest) implementation is to simply execute the query on all relevant shards
      * and return the results. Each shard returns size results. Since each shard already returns size hits, this
@@ -77,8 +71,6 @@ public enum SearchType {
             return DFS_QUERY_THEN_FETCH;
         } else if (id == 1) {
             return QUERY_THEN_FETCH;
-        } else if (id == 2) {
-            return DFS_QUERY_AND_FETCH;
         } else if (id == 3) {
             return QUERY_AND_FETCH;
         } else {
@@ -91,14 +83,12 @@ public enum SearchType {
      * one of "dfs_query_then_fetch"/"dfsQueryThenFetch", "dfs_query_and_fetch"/"dfsQueryAndFetch",
      * "query_then_fetch"/"queryThenFetch" and "query_and_fetch"/"queryAndFetch".
      */
-    public static SearchType fromString(String searchType, ParseFieldMatcher parseFieldMatcher) {
+    public static SearchType fromString(String searchType) {
         if (searchType == null) {
             return SearchType.DEFAULT;
         }
         if ("dfs_query_then_fetch".equals(searchType)) {
             return SearchType.DFS_QUERY_THEN_FETCH;
-        } else if ("dfs_query_and_fetch".equals(searchType)) {
-            return SearchType.DFS_QUERY_AND_FETCH;
         } else if ("query_then_fetch".equals(searchType)) {
             return SearchType.QUERY_THEN_FETCH;
         } else if ("query_and_fetch".equals(searchType)) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 8070081dcd8..2aa0ad3c7be 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -22,21 +22,34 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.SearchShardTarget;
 
 import java.io.IOException;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
+
 /**
  * Represents a failure to search on a specific shard.
  */
 public class ShardSearchFailure implements ShardOperationFailedException {
 
+    private static final String REASON_FIELD = "reason";
+    private static final String NODE_FIELD = "node";
+    private static final String INDEX_FIELD = "index";
+    private static final String SHARD_FIELD = "shard";
+
     public static final ShardSearchFailure[] EMPTY_ARRAY = new ShardSearchFailure[0];
 
     private SearchShardTarget shardTarget;
@@ -68,7 +81,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
         this(reason, shardTarget, RestStatus.INTERNAL_SERVER_ERROR);
     }
 
-    public ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
+    private ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
         this.shardTarget = shardTarget;
         this.reason = reason;
         this.status = status;
@@ -93,7 +106,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
     @Override
     public String index() {
         if (shardTarget != null) {
-            return shardTarget.index();
+            return shardTarget.getIndex();
         }
         return null;
     }
@@ -104,7 +117,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
     @Override
     public int shardId() {
         if (shardTarget != null) {
-            return shardTarget.shardId().id();
+            return shardTarget.getShardId().id();
         }
         return -1;
     }
@@ -153,20 +166,55 @@ public class ShardSearchFailure implements ShardOperationFailedException {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("shard", shardId());
-        builder.field("index", index());
+        builder.field(SHARD_FIELD, shardId());
+        builder.field(INDEX_FIELD, index());
         if (shardTarget != null) {
-            builder.field("node", shardTarget.nodeId());
+            builder.field(NODE_FIELD, shardTarget.getNodeId());
         }
         if (cause != null) {
-            builder.field("reason");
+            builder.field(REASON_FIELD);
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, cause);
+            ElasticsearchException.generateThrowableXContent(builder, params, cause);
             builder.endObject();
         }
         return builder;
     }
 
+    public static ShardSearchFailure fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        String currentFieldName = null;
+        int shardId = -1;
+        String indexName = null;
+        String nodeId = null;
+        ElasticsearchException exception = null;
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (SHARD_FIELD.equals(currentFieldName)) {
+                    shardId  = parser.intValue();
+                } else if (INDEX_FIELD.equals(currentFieldName)) {
+                    indexName  = parser.text();
+                } else if (NODE_FIELD.equals(currentFieldName)) {
+                    nodeId  = parser.text();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (REASON_FIELD.equals(currentFieldName)) {
+                    exception = ElasticsearchException.fromXContent(parser);
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        return new ShardSearchFailure(exception,
+                new SearchShardTarget(nodeId, new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId)));
+    }
+
     @Override
     public Throwable getCause() {
         return cause;
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 48ee5cc288b..8f94c1e3769 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -26,8 +26,11 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
+import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -38,13 +41,17 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Executor;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 
 import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
@@ -58,6 +65,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
 
     private final ClusterService clusterService;
     private final SearchTransportService searchTransportService;
+    private final RemoteClusterService remoteClusterService;
     private final SearchPhaseController searchPhaseController;
     private final SearchService searchService;
 
@@ -69,12 +77,14 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
         super(settings, SearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, SearchRequest::new);
         this.searchPhaseController = searchPhaseController;
         this.searchTransportService = searchTransportService;
+        this.remoteClusterService = searchTransportService.getRemoteClusterService();
         SearchTransportService.registerRequestHandler(transportService, searchService);
         this.clusterService = clusterService;
         this.searchService = searchService;
     }
 
-    private Map<String, AliasFilter> buildPerIndexAliasFilter(SearchRequest request, ClusterState clusterState, Index[] concreteIndices) {
+    private Map<String, AliasFilter> buildPerIndexAliasFilter(SearchRequest request, ClusterState clusterState,
+                                                              Index[] concreteIndices, Map<String, AliasFilter> remoteAliasMap) {
         final Map<String, AliasFilter> aliasFilterMap = new HashMap<>();
         for (Index index : concreteIndices) {
             clusterState.blocks().indexBlockedRaiseException(ClusterBlockLevel.READ, index.getName());
@@ -82,6 +92,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             assert aliasFilter != null;
             aliasFilterMap.put(index.getUUID(), aliasFilter);
         }
+        aliasFilterMap.putAll(remoteAliasMap);
         return aliasFilterMap;
     }
 
@@ -104,7 +115,6 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
                 concreteIndexBoosts.putIfAbsent(concreteIndex.getUUID(), ib.getBoost());
             }
         }
-
         return Collections.unmodifiableMap(concreteIndexBoosts);
     }
 
@@ -112,23 +122,62 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
         // pure paranoia if time goes backwards we are at least positive
         final long startTimeInMillis = Math.max(0, System.currentTimeMillis());
-        ClusterState clusterState = clusterService.state();
-        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
+        final String[] localIndices;
+        final Map<String, List<String>> remoteClusterIndices;
+        final ClusterState clusterState = clusterService.state();
+        if (remoteClusterService.isCrossClusterSearchEnabled()) {
+            remoteClusterIndices = remoteClusterService.groupClusterIndices( searchRequest.indices(), // empty string is not allowed
+                idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState));
+            List<String> remove = remoteClusterIndices.remove(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY);
+            localIndices = remove == null ? Strings.EMPTY_ARRAY : remove.toArray(new String[remove.size()]);
+        } else {
+            remoteClusterIndices = Collections.emptyMap();
+            localIndices = searchRequest.indices();
+        }
 
+        if (remoteClusterIndices.isEmpty()) {
+            executeSearch((SearchTask)task, startTimeInMillis, searchRequest, localIndices, Collections.emptyList(),
+                (nodeId) -> null, clusterState, Collections.emptyMap(), listener);
+        } else {
+            remoteClusterService.collectSearchShards(searchRequest, remoteClusterIndices,
+                ActionListener.wrap((searchShardsResponses) -> {
+                    List<ShardIterator> remoteShardIterators = new ArrayList<>();
+                    Map<String, AliasFilter> remoteAliasFilters = new HashMap<>();
+                    Function<String, Transport.Connection> connectionFunction = remoteClusterService.processRemoteShards(
+                        searchShardsResponses, remoteShardIterators, remoteAliasFilters);
+                    executeSearch((SearchTask)task, startTimeInMillis, searchRequest, localIndices, remoteShardIterators,
+                        connectionFunction, clusterState, remoteAliasFilters, listener);
+                }, listener::onFailure));
+        }
+    }
+
+    private void executeSearch(SearchTask task, long startTimeInMillis, SearchRequest searchRequest, String[] localIndices,
+                               List<ShardIterator> remoteShardIterators, Function<String, Transport.Connection> remoteConnections,
+                               ClusterState clusterState, Map<String, AliasFilter> remoteAliasMap,
+                               ActionListener<SearchResponse> listener) {
+
+        clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
         // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name
         // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead
         // of just for the _search api
-        Index[] indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(),
-            startTimeInMillis, searchRequest.indices());
-        Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices);
+        final Index[] indices;
+        if (localIndices.length == 0 && remoteShardIterators.size() > 0) {
+            indices = Index.EMPTY_ARRAY; // don't search on _all if only remote indices were specified
+        } else {
+            indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(),
+                startTimeInMillis, localIndices);
+        }
+        Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap);
         Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(),
             searchRequest.indices());
         String[] concreteIndices = new String[indices.length];
         for (int i = 0; i < indices.length; i++) {
             concreteIndices[i] = indices[i].getName();
         }
-        GroupShardsIterator shardIterators = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap,
+        GroupShardsIterator localShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap,
             searchRequest.preference());
+        GroupShardsIterator shardIterators = mergeShardsIterators(localShardsIterator, remoteShardIterators);
+
         failIfOverShardCountLimit(clusterService, shardIterators.size());
 
         Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);
@@ -142,7 +191,6 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             // disable request cache if we have only suggest
             searchRequest.requestCache(false);
             switch (searchRequest.searchType()) {
-                case DFS_QUERY_AND_FETCH:
                 case DFS_QUERY_THEN_FETCH:
                     // convert to Q_T_F if we have only suggest
                     searchRequest.searchType(QUERY_THEN_FETCH);
@@ -150,8 +198,52 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             }
         }
 
-        searchAsyncAction((SearchTask)task, searchRequest, shardIterators, startTimeInMillis, clusterState,
-            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, listener).start();
+        final DiscoveryNodes nodes = clusterState.nodes();
+        Function<String, Transport.Connection> connectionLookup = (nodeId) -> {
+            final DiscoveryNode discoveryNode = nodes.get(nodeId);
+            final Transport.Connection connection;
+            if (discoveryNode != null) {
+                connection = searchTransportService.getConnection(discoveryNode);
+            } else  {
+                connection = remoteConnections.apply(nodeId);
+            }
+            if (connection == null) {
+                throw new IllegalStateException("no node found for id: " + nodeId);
+            }
+            return connection;
+        };
+
+        final ActionListener<SearchResponse> wrapper;
+        if (searchPhaseController.getSearchResponseListener().size() > 0) {
+            wrapper = ActionListener.wrap(searchResponse -> {
+                List<BiConsumer<SearchRequest, SearchResponse>> responseListeners =
+                    searchPhaseController.getSearchResponseListener();
+                for (BiConsumer<SearchRequest, SearchResponse> respListener : responseListeners) {
+                    respListener.accept(searchRequest, searchResponse);
+                }
+                listener.onResponse(searchResponse);
+
+            }, listener::onFailure);
+        } else {
+            wrapper = listener;
+        }
+        searchAsyncAction(task, searchRequest, shardIterators, startTimeInMillis, connectionLookup, clusterState.version(),
+            Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, wrapper).start();
+    }
+
+    private static GroupShardsIterator mergeShardsIterators(GroupShardsIterator localShardsIterator,
+                                                            List<ShardIterator> remoteShardIterators) {
+        if (remoteShardIterators.isEmpty()) {
+            return localShardsIterator;
+        }
+        List<ShardIterator> shards = new ArrayList<>();
+        for (ShardIterator shardIterator : remoteShardIterators) {
+            shards.add(shardIterator);
+        }
+        for (ShardIterator shardIterator : localShardsIterator) {
+            shards.add(shardIterator);
+        }
+        return new GroupShardsIterator(shards);
     }
 
     @Override
@@ -160,31 +252,25 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     }
 
     private AbstractSearchAsyncAction searchAsyncAction(SearchTask task, SearchRequest searchRequest, GroupShardsIterator shardIterators,
-                                                        long startTime, ClusterState state,  Map<String, AliasFilter> aliasFilter,
+                                                        long startTime, Function<String, Transport.Connection> connectionLookup,
+                                                        long clusterStateVersion, Map<String, AliasFilter> aliasFilter,
                                                         Map<String, Float> concreteIndexBoosts,
                                                         ActionListener<SearchResponse> listener) {
-        final Function<String, DiscoveryNode> nodesLookup = state.nodes()::get;
-        final long clusterStateVersion = state.version();
         Executor executor = threadPool.executor(ThreadPool.Names.SEARCH);
         AbstractSearchAsyncAction searchAsyncAction;
         switch(searchRequest.searchType()) {
             case DFS_QUERY_THEN_FETCH:
-                searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, nodesLookup,
+                searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                     aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, startTime,
                     clusterStateVersion, task);
                 break;
             case QUERY_THEN_FETCH:
-                searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, nodesLookup,
-                    aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, startTime,
-                    clusterStateVersion, task);
-                break;
-            case DFS_QUERY_AND_FETCH:
-                searchAsyncAction = new SearchDfsQueryAndFetchAsyncAction(logger, searchTransportService, nodesLookup,
+                searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup,
                     aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, startTime,
                     clusterStateVersion, task);
                 break;
             case QUERY_AND_FETCH:
-                searchAsyncAction = new SearchQueryAndFetchAsyncAction(logger, searchTransportService, nodesLookup,
+                searchAsyncAction = new SearchQueryAndFetchAsyncAction(logger, searchTransportService, connectionLookup,
                     aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, startTime,
                     clusterStateVersion, task);
                 break;
@@ -194,7 +280,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
         return searchAsyncAction;
     }
 
-    private void failIfOverShardCountLimit(ClusterService clusterService, int shardCount) {
+    private static void failIfOverShardCountLimit(ClusterService clusterService, int shardCount) {
         final long shardCountLimit = clusterService.getClusterSettings().get(SHARD_COUNT_LIMIT_SETTING);
         if (shardCount > shardCountLimit) {
             throw new IllegalArgumentException("Trying to query " + shardCount + " shards, which is over the limit of "
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index a09a651086b..fdfa3205ea8 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -21,11 +21,9 @@ package org.elasticsearch.action.search;
 
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.RAMOutputStream;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
-import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 
 import java.io.IOException;
 import java.util.Base64;
@@ -39,7 +37,7 @@ final class TransportSearchHelper {
     static String buildScrollId(SearchType searchType, AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
         if (searchType == SearchType.DFS_QUERY_THEN_FETCH || searchType == SearchType.QUERY_THEN_FETCH) {
             return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults);
-        } else if (searchType == SearchType.QUERY_AND_FETCH || searchType == SearchType.DFS_QUERY_AND_FETCH) {
+        } else if (searchType == SearchType.QUERY_AND_FETCH) {
             return buildScrollId(ParsedScrollId.QUERY_AND_FETCH_TYPE, searchPhaseResults);
         } else {
             throw new IllegalStateException("search_type [" + searchType + "] not supported");
@@ -53,7 +51,7 @@ final class TransportSearchHelper {
             for (AtomicArray.Entry<? extends SearchPhaseResult> entry : searchPhaseResults.asList()) {
                 SearchPhaseResult searchPhaseResult = entry.value;
                 out.writeLong(searchPhaseResult.id());
-                out.writeString(searchPhaseResult.shardTarget().nodeId());
+                out.writeString(searchPhaseResult.shardTarget().getNodeId());
             }
             byte[] bytes = new byte[(int) out.getFilePointer()];
             out.writeTo(bytes, 0);
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
index 880d173b2fe..3e12d0cc842 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
@@ -47,7 +47,7 @@ public interface ActionFilter {
      * filter chain. This base class should serve any action filter implementations that doesn't require
      * to apply async filtering logic.
      */
-    public abstract static class Simple extends AbstractComponent implements ActionFilter {
+    abstract class Simple extends AbstractComponent implements ActionFilter {
 
         protected Simple(Settings settings) {
             super(settings);
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java b/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
index 56ba070b1aa..97e0c535bff 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActionFilterChain.java
@@ -33,5 +33,5 @@ public interface ActionFilterChain<Request extends ActionRequest, Response exten
      * Continue processing the request. Should only be called if a response has not been sent through
      * the given {@link ActionListener listener}
      */
-    void proceed(Task task, final String action, final Request request, final ActionListener<Response> listener);
+    void proceed(Task task, String action, Request request, ActionListener<Response> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java b/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
index ee2bc9a4b36..280ba6ac94d 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActiveShardsObserver.java
@@ -69,9 +69,10 @@ public class ActiveShardsObserver extends AbstractComponent {
             return;
         }
 
-        final ClusterStateObserver observer = new ClusterStateObserver(clusterService, logger, threadPool.getThreadContext());
-        if (activeShardCount.enoughShardsActive(observer.observedState(), indexName)) {
-                onResult.accept(true);
+        final ClusterState state = clusterService.state();
+        final ClusterStateObserver observer = new ClusterStateObserver(state, clusterService, null, logger, threadPool.getThreadContext());
+        if (activeShardCount.enoughShardsActive(state, indexName)) {
+            onResult.accept(true);
         } else {
             final Predicate<ClusterState> shardsAllocatedPredicate = newState -> activeShardCount.enoughShardsActive(newState, indexName);
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
index a9a5afed9f3..2e442e2cc14 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.util.ArrayList;
@@ -63,18 +64,20 @@ public final class AutoCreateIndex {
 
     /**
      * Should the index be auto created?
+     * @throws IndexNotFoundException if the the index doesn't exist and shouldn't be auto created
      */
     public boolean shouldAutoCreate(String index, ClusterState state) {
+        if (resolver.hasIndexOrAlias(index, state)) {
+            return false;
+        }
         // One volatile read, so that all checks are done against the same instance:
         final AutoCreate autoCreate = this.autoCreate;
         if (autoCreate.autoCreateIndex == false) {
-            return false;
+            throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] is [false]", index);
         }
         if (dynamicMappingDisabled) {
-            return false;
-        }
-        if (resolver.hasIndexOrAlias(index, state)) {
-            return false;
+            throw new IndexNotFoundException("no such index and [" + MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey() + "] is [false]",
+                    index);
         }
         // matches not set, default value of "true"
         if (autoCreate.expressions.isEmpty()) {
@@ -84,10 +87,15 @@ public final class AutoCreateIndex {
             String indexExpression = expression.v1();
             boolean include = expression.v2();
             if (Regex.simpleMatch(indexExpression, index)) {
-                return include;
+                if (include) {
+                    return true;
+                }
+                throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] contains [-"
+                        + indexExpression + "] which forbids automatic creation of the index", index);
             }
         }
-        return false;
+        throw new IndexNotFoundException("no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey() + "] ([" + autoCreate
+                + "]) doesn't match", index);
     }
 
     AutoCreate getAutoCreate() {
@@ -101,29 +109,33 @@ public final class AutoCreateIndex {
     static class AutoCreate {
         private final boolean autoCreateIndex;
         private final List<Tuple<String, Boolean>> expressions;
+        private final String string;
 
         private AutoCreate(String value) {
             boolean autoCreateIndex;
             List<Tuple<String, Boolean>> expressions = new ArrayList<>();
             try {
-                autoCreateIndex = Booleans.parseBooleanExact(value);
+                autoCreateIndex = Booleans.parseBoolean(value);
             } catch (IllegalArgumentException ex) {
                 try {
                     String[] patterns = Strings.commaDelimitedListToStringArray(value);
                     for (String pattern : patterns) {
                         if (pattern == null || pattern.trim().length() == 0) {
-                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must be either [true, false, or a comma separated list of index patterns]");
+                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must "
+                                    + "be either [true, false, or a comma separated list of index patterns]");
                         }
                         pattern = pattern.trim();
                         Tuple<String, Boolean> expression;
                         if (pattern.startsWith("-")) {
                             if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must contain an index name after [-]");
+                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [-]");
                             }
                             expression = new Tuple<>(pattern.substring(1), false);
                         } else if(pattern.startsWith("+")) {
                             if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must contain an index name after [+]");
+                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [+]");
                             }
                             expression = new Tuple<>(pattern.substring(1), true);
                         } else {
@@ -139,6 +151,7 @@ public final class AutoCreateIndex {
             }
             this.expressions = expressions;
             this.autoCreateIndex = autoCreateIndex;
+            this.string = value;
         }
 
         boolean isAutoCreateIndex() {
@@ -148,5 +161,10 @@ public final class AutoCreateIndex {
         List<Tuple<String, Boolean>> getExpressions() {
             return expressions;
         }
+
+        @Override
+        public String toString() {
+            return string;
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java b/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java
new file mode 100644
index 00000000000..a4293bfb485
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/ContextPreservingActionListener.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.function.Supplier;
+
+/**
+ * Restores the given {@link org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext}
+ * once the listener is invoked
+ */
+public final class ContextPreservingActionListener<R> implements ActionListener<R> {
+
+    private final ActionListener<R> delegate;
+    private final Supplier<ThreadContext.StoredContext> context;
+
+    public ContextPreservingActionListener(Supplier<ThreadContext.StoredContext> contextSupplier, ActionListener<R> delegate) {
+        this.delegate = delegate;
+        this.context = contextSupplier;
+    }
+
+    @Override
+    public void onResponse(R r) {
+        try (ThreadContext.StoredContext ignore = context.get()) {
+            delegate.onResponse(r);
+        }
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        try (ThreadContext.StoredContext ignore = context.get()) {
+            delegate.onFailure(e);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index 3f7df803e24..2ced9145674 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -125,7 +125,7 @@ public class DefaultShardOperationFailedException implements ShardOperationFaile
         if (reason != null) {
             builder.field("reason");
             builder.startObject();
-            ElasticsearchException.toXContent(builder, params, reason);
+            ElasticsearchException.generateThrowableXContent(builder, params, reason);
             builder.endObject();
         }
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java b/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
index 2bc49f7e9f8..b82bfcc7170 100644
--- a/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
+++ b/core/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
@@ -26,7 +26,7 @@ import org.elasticsearch.rest.RestRequest;
 import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
 
 /**
@@ -195,8 +195,8 @@ public class IndicesOptions {
 
         //note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use)
         return fromOptions(
-                lenientNodeBooleanValue(ignoreUnavailableString, defaultSettings.ignoreUnavailable()),
-                lenientNodeBooleanValue(allowNoIndicesString, defaultSettings.allowNoIndices()),
+                nodeBooleanValue(ignoreUnavailableString, "ignore_unavailable", defaultSettings.ignoreUnavailable()),
+                nodeBooleanValue(allowNoIndicesString, "allow_no_indices", defaultSettings.allowNoIndices()),
                 expandWildcardsOpen,
                 expandWildcardsClosed,
                 defaultSettings.allowAliasesToMultipleIndices(),
@@ -279,7 +279,7 @@ public class IndicesOptions {
                 ", allow_no_indices=" + allowNoIndices() +
                 ", expand_wildcards_open=" + expandWildcardsOpen() +
                 ", expand_wildcards_closed=" + expandWildcardsClosed() +
-                ", allow_alisases_to_multiple_indices=" + allowAliasesToMultipleIndices() +
+                ", allow_aliases_to_multiple_indices=" + allowAliasesToMultipleIndices() +
                 ", forbid_closed_indices=" + forbidClosedIndices() +
                 ']';
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
index e8f4d943e95..22edbfca2dc 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
@@ -43,7 +42,6 @@ public abstract class TransportAction<Request extends ActionRequest, Response ex
     protected final ThreadPool threadPool;
     protected final String actionName;
     private final ActionFilter[] filters;
-    protected final ParseFieldMatcher parseFieldMatcher;
     protected final IndexNameExpressionResolver indexNameExpressionResolver;
     protected final TaskManager taskManager;
 
@@ -53,7 +51,6 @@ public abstract class TransportAction<Request extends ActionRequest, Response ex
         this.threadPool = threadPool;
         this.actionName = actionName;
         this.filters = actionFilters.filters();
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.taskManager = taskManager;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index c48fa1e8122..0408b04cc83 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -175,7 +175,6 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
                         // no node connected, act as failure
                         onOperation(shard, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
                     } else {
-                        taskManager.registerChildTask(task, node.getId());
                         transportService.sendRequest(node, transportShardAction, shardRequest, new TransportResponseHandler<ShardResponse>() {
                             @Override
                             public ShardResponse newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 9f11b9b5a70..951c5dbc3ec 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -318,7 +318,6 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                 NodeRequest nodeRequest = new NodeRequest(node.getId(), request, shards);
                 if (task != null) {
                     nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                    taskManager.registerChildTask(task, node.getId());
                 }
                 transportService.sendRequest(node, transportNodeBroadcastAction, nodeRequest, new TransportResponseHandler<NodeResponse>() {
                     @Override
@@ -439,7 +438,6 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             } catch (Exception e) {
                 BroadcastShardOperationFailedException failure =
                     new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", e);
-                failure.setIndex(shardRouting.getIndexName());
                 failure.setShard(shardRouting.shardId());
                 shardResults[shardIndex] = failure;
                 if (TransportActions.isShardNotAvailableException(e)) {
@@ -524,10 +522,10 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
         protected List<BroadcastShardOperationFailedException> exceptions;
         protected List<ShardOperationResult> results;
 
-        public NodeResponse() {
+        NodeResponse() {
         }
 
-        public NodeResponse(String nodeId,
+        NodeResponse(String nodeId,
                             int totalShards,
                             List<ShardOperationResult> results,
                             List<BroadcastShardOperationFailedException> exceptions) {
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index ecb03b5c222..f2bc4da423d 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -121,12 +121,12 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
         }
 
         public void start() {
-            this.observer = new ClusterStateObserver(clusterService, request.masterNodeTimeout(), logger, threadPool.getThreadContext());
-            doStart();
+            ClusterState state = clusterService.state();
+            this.observer = new ClusterStateObserver(state, clusterService, request.masterNodeTimeout(), logger, threadPool.getThreadContext());
+            doStart(state);
         }
 
-        protected void doStart() {
-            final ClusterState clusterState = observer.observedState();
+        protected void doStart(ClusterState clusterState) {
             final Predicate<ClusterState> masterChangePredicate = MasterNodeChangePredicate.build(clusterState);
             final DiscoveryNodes nodes = clusterState.nodes();
             if (nodes.isLocalNodeElectedMaster() || localExecute(request)) {
@@ -160,7 +160,6 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
                             }
                         }
                     };
-                    taskManager.registerChildTask(task, nodes.getLocalNodeId());
                     threadPool.executor(executor).execute(new ActionRunnable(delegate) {
                         @Override
                         protected void doRun() throws Exception {
@@ -173,7 +172,6 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
                     logger.debug("no known master node, scheduling a retry");
                     retry(null, masterChangePredicate);
                 } else {
-                    taskManager.registerChildTask(task, nodes.getMasterNode().getId());
                     transportService.sendRequest(nodes.getMasterNode(), actionName, request, new ActionListenerResponseHandler<Response>(listener, TransportMasterNodeAction.this::newResponse) {
                         @Override
                         public void handleException(final TransportException exp) {
@@ -197,7 +195,7 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
                 new ClusterStateObserver.Listener() {
                     @Override
                     public void onNewClusterState(ClusterState state) {
-                        doStart();
+                        doStart(state);
                     }
 
                     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
index 59b1997b356..9bf7356a9fe 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/info/TransportClusterInfoAction.java
@@ -52,5 +52,5 @@ public abstract class TransportClusterInfoAction<Request extends ClusterInfoRequ
         doMasterOperation(request, concreteIndices, state, listener);
     }
 
-    protected abstract void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, final ActionListener<Response> listener);
+    protected abstract void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, ActionListener<Response> listener);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 6cc063d5af1..d8010f4381f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -199,7 +199,6 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                         TransportRequest nodeRequest = newNodeRequest(nodeId, request);
                         if (task != null) {
                             nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                            taskManager.registerChildTask(task, node.getId());
                         }
 
                         transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java
index f431c67b290..b4731d19e29 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/BasicReplicationRequest.java
@@ -37,4 +37,9 @@ public class BasicReplicationRequest extends ReplicationRequest<BasicReplication
     public BasicReplicationRequest(ShardId shardId) {
         super(shardId);
     }
+
+    @Override
+    public String toString() {
+        return "BasicReplicationRequest{" + shardId + "}";
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
index fa02dac9e1e..107c791a069 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.action.support.replication;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
@@ -36,6 +38,8 @@ import java.io.IOException;
 public abstract class ReplicatedWriteRequest<R extends ReplicatedWriteRequest<R>> extends ReplicationRequest<R> implements WriteRequest<R> {
     private RefreshPolicy refreshPolicy = RefreshPolicy.NONE;
 
+    private long seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+
     /**
      * Constructor for deserialization.
      */
@@ -62,11 +66,32 @@ public abstract class ReplicatedWriteRequest<R extends ReplicatedWriteRequest<R>
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         refreshPolicy = RefreshPolicy.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))  {
+            seqNo = in.readZLong();
+        } else {
+            seqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         refreshPolicy.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            out.writeZLong(seqNo);
+        }
+    }
+
+    /**
+     * Returns the sequence number for this operation. The sequence number is assigned while the operation
+     * is performed on the primary shard.
+     */
+    public long getSeqNo() {
+        return seqNo;
+    }
+
+    /** sets the sequence number for this operation. should only be called on the primary shard */
+    public void setSeqNo(long seqNo) {
+        this.seqNo = seqNo;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 47284789850..6a3d217fcf6 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -202,7 +202,7 @@ public class ReplicationOperation<
                         shard,
                         replicaRequest),
                     replicaException);
-                if (ignoreReplicaException(replicaException)) {
+                if (TransportActions.isShardNotAvailableException(replicaException)) {
                     decPendingAndFinishIfNeeded();
                 } else {
                     RestStatus restStatus = ExceptionsHelper.status(replicaException);
@@ -283,7 +283,7 @@ public class ReplicationOperation<
     }
 
     private void decPendingAndFinishIfNeeded() {
-        assert pendingActions.get() > 0;
+        assert pendingActions.get() > 0 : "pending action count goes below 0 for request [" + request + "]";
         if (pendingActions.decrementAndGet() == 0) {
             finish();
         }
@@ -314,30 +314,6 @@ public class ReplicationOperation<
         }
     }
 
-
-    /**
-     * Should an exception be ignored when the operation is performed on the replica.
-     */
-    public static boolean ignoreReplicaException(Exception e) {
-        if (TransportActions.isShardNotAvailableException(e)) {
-            return true;
-        }
-        // on version conflict or document missing, it means
-        // that a new change has crept into the replica, and it's fine
-        if (isConflictException(e)) {
-            return true;
-        }
-        return false;
-    }
-
-    public static boolean isConflictException(Throwable t) {
-        final Throwable cause = ExceptionsHelper.unwrapCause(t);
-        // on version conflict or document missing, it means
-        // that a new change has crept into the replica, and it's fine
-        return cause instanceof VersionConflictEngineException;
-    }
-
-
     public interface Primary<
                 Request extends ReplicationRequest<Request>,
                 ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index d520b3d4e70..091f96c408f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -55,7 +55,6 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
      */
     protected ShardId shardId;
 
-    long seqNo;
     long primaryTerm;
 
     protected TimeValue timeout = DEFAULT_TIMEOUT;
@@ -171,19 +170,6 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         return routedBasedOnClusterVersion;
     }
 
-    /**
-     * Returns the sequence number for this operation. The sequence number is assigned while the operation
-     * is performed on the primary shard.
-     */
-    public long seqNo() {
-        return seqNo;
-    }
-
-    /** sets the sequence number for this operation. should only be called on the primary shard */
-    public void seqNo(long seqNo) {
-        this.seqNo = seqNo;
-    }
-
     /** returns the primary term active at the time the operation was performed on the primary shard */
     public long primaryTerm() {
         return primaryTerm;
@@ -215,7 +201,6 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         timeout = new TimeValue(in);
         index = in.readString();
         routedBasedOnClusterVersion = in.readVLong();
-        seqNo = in.readVLong();
         primaryTerm = in.readVLong();
     }
 
@@ -232,7 +217,6 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
         timeout.writeTo(out);
         out.writeString(index);
         out.writeVLong(routedBasedOnClusterVersion);
-        out.writeVLong(seqNo);
         out.writeVLong(primaryTerm);
     }
 
@@ -252,13 +236,7 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
     }
 
     @Override
-    public String toString() {
-        if (shardId != null) {
-            return shardId.toString();
-        } else {
-            return index;
-        }
-    }
+    public abstract String toString(); // force a proper to string to ease debugging
 
     @Override
     public String getDescription() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index 6e44c4f158c..97a696a961f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -23,19 +23,26 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+
 /**
  * Base class for write action responses.
  */
@@ -65,7 +72,7 @@ public class ReplicationResponse extends ActionResponse {
         this.shardInfo = shardInfo;
     }
 
-    public static class ShardInfo implements Streamable, ToXContent {
+    public static class ShardInfo implements Streamable, ToXContentObject {
 
         private static final String _SHARDS = "_shards";
         private static final String TOTAL = "total";
@@ -171,7 +178,7 @@ public class ReplicationResponse extends ActionResponse {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(_SHARDS);
+            builder.startObject();
             builder.field(TOTAL, total);
             builder.field(SUCCESSFUL, successful);
             builder.field(FAILED, getFailed());
@@ -186,6 +193,42 @@ public class ReplicationResponse extends ActionResponse {
             return builder;
         }
 
+        public static ShardInfo fromXContent(XContentParser parser) throws IOException {
+            XContentParser.Token token = parser.currentToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+            int total = 0, successful = 0;
+            List<Failure> failuresList = null;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if (TOTAL.equals(currentFieldName)) {
+                        total = parser.intValue();
+                    } else if (SUCCESSFUL.equals(currentFieldName)) {
+                        successful = parser.intValue();
+                    } else if (FAILED.equals(currentFieldName) == false) {
+                        throwUnknownField(currentFieldName, parser.getTokenLocation());
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY) {
+                    if (FAILURES.equals(currentFieldName)) {
+                        failuresList = new ArrayList<>();
+                        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            failuresList.add(Failure.fromXContent(parser));
+                        }
+                    } else {
+                        throwUnknownField(currentFieldName, parser.getTokenLocation());
+                    }
+                }
+            }
+            Failure[] failures = EMPTY;
+            if (failuresList != null) {
+                failures = failuresList.toArray(new Failure[failuresList.size()]);
+            }
+            return new ShardInfo(total, successful, failures);
+        }
+
         @Override
         public String toString() {
             return "ShardInfo{" +
@@ -201,7 +244,7 @@ public class ReplicationResponse extends ActionResponse {
             return shardInfo;
         }
 
-        public static class Failure implements ShardOperationFailedException, ToXContent {
+        public static class Failure implements ShardOperationFailedException, ToXContentObject {
 
             private static final String _INDEX = "_index";
             private static final String _SHARD = "_shard";
@@ -331,13 +374,52 @@ public class ReplicationResponse extends ActionResponse {
                 builder.field(_NODE, nodeId);
                 builder.field(REASON);
                 builder.startObject();
-                ElasticsearchException.toXContent(builder, params, cause);
+                ElasticsearchException.generateThrowableXContent(builder, params, cause);
                 builder.endObject();
                 builder.field(STATUS, status);
                 builder.field(PRIMARY, primary);
                 builder.endObject();
                 return builder;
             }
+
+            public static Failure fromXContent(XContentParser parser) throws IOException {
+                XContentParser.Token token = parser.currentToken();
+                ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+                String shardIndex = null, nodeId = null;
+                int shardId = -1;
+                boolean primary = false;
+                RestStatus status = null;
+                ElasticsearchException reason = null;
+
+                String currentFieldName = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else if (token.isValue()) {
+                        if (_INDEX.equals(currentFieldName)) {
+                            shardIndex = parser.text();
+                        } else if (_SHARD.equals(currentFieldName)) {
+                            shardId = parser.intValue();
+                        } else if (_NODE.equals(currentFieldName)) {
+                            nodeId = parser.text();
+                        } else if (STATUS.equals(currentFieldName)) {
+                            status = RestStatus.valueOf(parser.text());
+                        } else if (PRIMARY.equals(currentFieldName)) {
+                            primary = parser.booleanValue();
+                        } else {
+                            throwUnknownField(currentFieldName, parser.getTokenLocation());
+                        }
+                    } else if (token == XContentParser.Token.START_OBJECT) {
+                        if (REASON.equals(currentFieldName)) {
+                            reason = ElasticsearchException.fromXContent(parser);
+                        } else {
+                            throwUnknownField(currentFieldName, parser.getTokenLocation());
+                        }
+                    }
+                }
+                return new Failure(new ShardId(shardIndex, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), nodeId, reason, status, primary);
+            }
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index e33d10eaa25..8193cf77ceb 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -119,7 +119,6 @@ public abstract class TransportBroadcastReplicationAction<Request extends Broadc
     protected void shardExecute(Task task, Request request, ShardId shardId, ActionListener<ShardResponse> shardActionListener) {
         ShardRequest shardRequest = newShardRequest(request, shardId);
         shardRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-        taskManager.registerChildTask(task, clusterService.localNode().getId());
         replicatedBroadcastShardAction.execute(shardRequest, shardActionListener);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 8cff657e3dd..32f5b8f0f2c 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.support.replication;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.ActionResponse;
@@ -43,6 +44,7 @@ import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasable;
@@ -57,6 +59,7 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
@@ -170,7 +173,8 @@ public abstract class TransportReplicationAction<
      * @param shardRequest the request to the primary shard
      * @param primary      the primary shard to perform the operation on
      */
-    protected abstract PrimaryResult shardOperationOnPrimary(Request shardRequest, IndexShard primary) throws Exception;
+    protected abstract PrimaryResult<ReplicaRequest, Response> shardOperationOnPrimary(
+            Request shardRequest, IndexShard primary) throws Exception;
 
     /**
      * Synchronous replica operation on nodes with replica copies. This is done under the lock form
@@ -182,17 +186,19 @@ public abstract class TransportReplicationAction<
     protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest, IndexShard replica) throws Exception;
 
     /**
-     * Cluster level block to check before request execution
+     * Cluster level block to check before request execution. Returning null means that no blocks need to be checked.
      */
+    @Nullable
     protected ClusterBlockLevel globalBlockLevel() {
-        return ClusterBlockLevel.WRITE;
+        return null;
     }
 
     /**
-     * Index level block to check before request execution
+     * Index level block to check before request execution. Returning null means that no blocks need to be checked.
      */
+    @Nullable
     protected ClusterBlockLevel indexBlockLevel() {
-        return ClusterBlockLevel.WRITE;
+        return null;
     }
 
     /**
@@ -309,7 +315,7 @@ public abstract class TransportReplicationAction<
                 } else {
                     setPhase(replicationTask, "primary");
                     final IndexMetaData indexMetaData = clusterService.state().getMetaData().index(request.shardId().getIndex());
-                    final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings());
+                    final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData);
                     final ActionListener<Response> listener = createResponseListener(primaryShardReference);
                     createReplicatedOperation(request,
                             ActionListener.wrap(result -> result.respond(listener), listener::onFailure),
@@ -359,8 +365,8 @@ public abstract class TransportReplicationAction<
             };
         }
 
-        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult> createReplicatedOperation(
-            Request request, ActionListener<PrimaryResult> listener,
+        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult<ReplicaRequest, Response>> createReplicatedOperation(
+            Request request, ActionListener<PrimaryResult<ReplicaRequest, Response>> listener,
             PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
             return new ReplicationOperation<>(request, primaryShardReference, listener,
                 executeOnReplicas, replicasProxy, clusterService::state, logger, actionName
@@ -368,10 +374,12 @@ public abstract class TransportReplicationAction<
         }
     }
 
-    protected class PrimaryResult implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
+    protected static class PrimaryResult<ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
+            Response extends ReplicationResponse>
+            implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
         final ReplicaRequest replicaRequest;
-        final Response finalResponseIfSuccessful;
-        final Exception finalFailure;
+        public final Response finalResponseIfSuccessful;
+        public final Exception finalFailure;
 
         /**
          * Result of executing a primary operation
@@ -411,7 +419,7 @@ public abstract class TransportReplicationAction<
         }
     }
 
-    protected class ReplicaResult {
+    protected static class ReplicaResult {
         final Exception finalFailure;
 
         public ReplicaResult(Exception finalFailure) {
@@ -506,16 +514,15 @@ public abstract class TransportReplicationAction<
                             request),
                     e);
                 request.onRetry();
-                final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext();
                 observer.waitForNextChange(new ClusterStateObserver.Listener() {
                     @Override
                     public void onNewClusterState(ClusterState state) {
-                        context.close();
                         // Forking a thread on local node via transport service so that custom transport service have an
                         // opportunity to execute custom logic before the replica operation begins
                         String extraMessage = "action [" + transportReplicaAction + "], request[" + request + "]";
                         TransportChannelResponseHandler<TransportResponse.Empty> handler =
-                            new TransportChannelResponseHandler<>(logger, channel, extraMessage, () -> TransportResponse.Empty.INSTANCE);
+                            new TransportChannelResponseHandler<>(logger, channel, extraMessage,
+                                () -> TransportResponse.Empty.INSTANCE);
                         transportService.sendRequest(clusterService.localNode(), transportReplicaAction,
                             new ConcreteShardRequest<>(request, targetAllocationID),
                             handler);
@@ -568,7 +575,7 @@ public abstract class TransportReplicationAction<
         private class ResponseListener implements ActionListener<TransportResponse.Empty> {
             private final ReplicaResponse replicaResponse;
 
-            public ResponseListener(ReplicaResponse replicaResponse) {
+            ResponseListener(ReplicaResponse replicaResponse) {
                 this.replicaResponse = replicaResponse;
             }
 
@@ -630,7 +637,7 @@ public abstract class TransportReplicationAction<
         @Override
         protected void doRun() {
             setPhase(task, "routing");
-            final ClusterState state = observer.observedState();
+            final ClusterState state = observer.setAndGetObservedState();
             if (handleBlockExceptions(state)) {
                 return;
             }
@@ -642,6 +649,9 @@ public abstract class TransportReplicationAction<
                 retry(new IndexNotFoundException(concreteIndex));
                 return;
             }
+            if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
+                throw new IndexClosedException(indexMetaData.getIndex());
+            }
 
             // resolve all derived request fields, so we can route and apply it
             resolveRequest(state.metaData(), indexMetaData, request);
@@ -653,7 +663,6 @@ public abstract class TransportReplicationAction<
                 return;
             }
             final DiscoveryNode node = state.nodes().get(primary.currentNodeId());
-            taskManager.registerChildTask(task, node.getId());
             if (primary.currentNodeId().equals(state.nodes().getLocalNodeId())) {
                 performLocalAction(state, primary, node);
             } else {
@@ -664,7 +673,7 @@ public abstract class TransportReplicationAction<
         private void performLocalAction(ClusterState state, ShardRouting primary, DiscoveryNode node) {
             setPhase(task, "waiting_on_primary");
             if (logger.isTraceEnabled()) {
-                logger.trace("send action [{}] on primary [{}] for request [{}] with cluster state version [{}] to [{}] ",
+                logger.trace("send action [{}] to local primary [{}] for request [{}] with cluster state version [{}] to [{}] ",
                     transportPrimaryAction, request.shardId(), request, state.version(), primary.currentNodeId());
             }
             performAction(node, transportPrimaryAction, true, new ConcreteShardRequest<>(request, primary.allocationId().getId()));
@@ -718,15 +727,21 @@ public abstract class TransportReplicationAction<
         }
 
         private boolean handleBlockExceptions(ClusterState state) {
-            ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel());
-            if (blockException != null) {
-                handleBlockException(blockException);
-                return true;
+            ClusterBlockLevel globalBlockLevel = globalBlockLevel();
+            if (globalBlockLevel != null) {
+                ClusterBlockException blockException = state.blocks().globalBlockedException(globalBlockLevel);
+                if (blockException != null) {
+                    handleBlockException(blockException);
+                    return true;
+                }
             }
-            blockException = state.blocks().indexBlockedException(indexBlockLevel(), concreteIndex(state));
-            if (blockException != null) {
-                handleBlockException(blockException);
-                return true;
+            ClusterBlockLevel indexBlockLevel = indexBlockLevel();
+            if (indexBlockLevel != null) {
+                ClusterBlockException blockException = state.blocks().indexBlockedException(indexBlockLevel, concreteIndex(state));
+                if (blockException != null) {
+                    handleBlockException(blockException);
+                    return true;
+                }
             }
             return false;
         }
@@ -793,11 +808,9 @@ public abstract class TransportReplicationAction<
             }
             setPhase(task, "waiting_for_retry");
             request.onRetry();
-            final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext();
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    context.close();
                     run();
                 }
 
@@ -808,7 +821,6 @@ public abstract class TransportReplicationAction<
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
-                    context.close();
                     // Try one more time...
                     run();
                 }
@@ -898,8 +910,8 @@ public abstract class TransportReplicationAction<
      * Indicated whether this operation should be replicated to shadow replicas or not. If this method returns true the replication phase
      * will be skipped. For example writes such as index and delete don't need to be replicated on shadow replicas but refresh and flush do.
      */
-    protected boolean shouldExecuteReplication(Settings settings) {
-        return IndexMetaData.isIndexUsingShadowReplicas(settings) == false;
+    protected boolean shouldExecuteReplication(IndexMetaData indexMetaData) {
+        return indexMetaData.isIndexUsingShadowReplicas() == false;
     }
 
     class ShardReference implements Releasable {
@@ -927,7 +939,8 @@ public abstract class TransportReplicationAction<
 
     }
 
-    class PrimaryShardReference extends ShardReference implements ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult> {
+    class PrimaryShardReference extends ShardReference
+            implements ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult<ReplicaRequest, Response>> {
 
         PrimaryShardReference(IndexShard indexShard, Releasable operationLock) {
             super(indexShard, operationLock);
@@ -950,6 +963,8 @@ public abstract class TransportReplicationAction<
         public PrimaryResult perform(Request request) throws Exception {
             PrimaryResult result = shardOperationOnPrimary(request, indexShard);
             if (result.replicaRequest() != null) {
+                assert result.finalFailure == null : "a replica request [" + result.replicaRequest()
+                    + "] with a primary failure [" + result.finalFailure + "]";
                 result.replicaRequest().primaryTerm(indexShard.getPrimaryTerm());
             }
             return result;
@@ -983,16 +998,25 @@ public abstract class TransportReplicationAction<
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            localCheckpoint = in.readZLong();
-            allocationId = in.readString();
+            if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                super.readFrom(in);
+                localCheckpoint = in.readZLong();
+                allocationId = in.readString();
+            } else {
+                // 5.x used to read empty responses, which don't really read anything off the stream, so just do nothing.
+            }
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeZLong(localCheckpoint);
-            out.writeString(allocationId);
+            if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                super.writeTo(out);
+                out.writeZLong(localCheckpoint);
+                out.writeString(allocationId);
+            } else {
+                // we use to write empty responses
+                Empty.INSTANCE.writeTo(out);
+            }
         }
 
         @Override
@@ -1016,10 +1040,9 @@ public abstract class TransportReplicationAction<
                 listener.onFailure(new NoNodeAvailableException("unknown node [" + nodeId + "]"));
                 return;
             }
-            transportService.sendRequest(node, transportReplicaAction,
-                new ConcreteShardRequest<>(request, replica.allocationId().getId()), transportOptions,
-                // Eclipse can't handle when this is <> so we specify the type here.
-                new ActionListenerResponseHandler<ReplicaResponse>(listener, ReplicaResponse::new));
+            final ConcreteShardRequest<ReplicaRequest> concreteShardRequest =
+                new ConcreteShardRequest<>(request, replica.allocationId().getId());
+            sendReplicaRequest(concreteShardRequest, node, listener);
         }
 
         @Override
@@ -1060,6 +1083,14 @@ public abstract class TransportReplicationAction<
         }
     }
 
+    /** sends the given replica request to the supplied nodes */
+    protected void sendReplicaRequest(ConcreteShardRequest<ReplicaRequest> concreteShardRequest, DiscoveryNode node,
+                                      ActionListener<ReplicationOperation.ReplicaResponse> listener) {
+        transportService.sendRequest(node, transportReplicaAction, concreteShardRequest, transportOptions,
+            // Eclipse can't handle when this is <> so we specify the type here.
+            new ActionListenerResponseHandler<ReplicaResponse>(listener, ReplicaResponse::new));
+    }
+
     /** a wrapper class to encapsulate a request when being sent to a specific allocation id **/
     public static final class ConcreteShardRequest<R extends TransportRequest> extends TransportRequest {
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
index 15f269c46f5..8569b28257f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -21,10 +21,13 @@ package org.elasticsearch.action.support.replication;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.WriteResponse;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
@@ -67,7 +70,8 @@ public abstract class TransportWriteAction<
      * async refresh is performed on the <code>primary</code> shard according to the <code>Request</code> refresh policy
      */
     @Override
-    protected abstract WritePrimaryResult shardOperationOnPrimary(Request request, IndexShard primary) throws Exception;
+    protected abstract WritePrimaryResult<ReplicaRequest, Response> shardOperationOnPrimary(
+            Request request, IndexShard primary) throws Exception;
 
     /**
      * Called once per replica with a reference to the replica {@linkplain IndexShard} to modify.
@@ -76,19 +80,24 @@ public abstract class TransportWriteAction<
      * async refresh is performed on the <code>replica</code> shard according to the <code>ReplicaRequest</code> refresh policy
      */
     @Override
-    protected abstract WriteReplicaResult shardOperationOnReplica(ReplicaRequest request, IndexShard replica) throws Exception;
+    protected abstract WriteReplicaResult<ReplicaRequest> shardOperationOnReplica(
+            ReplicaRequest request, IndexShard replica) throws Exception;
 
     /**
      * Result of taking the action on the primary.
      */
-    protected class WritePrimaryResult extends PrimaryResult implements RespondingWriteResult {
+    protected static class WritePrimaryResult<ReplicaRequest extends ReplicatedWriteRequest<ReplicaRequest>,
+            Response extends ReplicationResponse & WriteResponse> extends PrimaryResult<ReplicaRequest, Response>
+            implements RespondingWriteResult {
         boolean finishedAsyncActions;
+        public final Location location;
         ActionListener<Response> listener = null;
 
         public WritePrimaryResult(ReplicaRequest request, @Nullable Response finalResponse,
                                   @Nullable Location location, @Nullable Exception operationFailure,
-                                  IndexShard primary) {
+                                  IndexShard primary, Logger logger) {
             super(request, finalResponse, operationFailure);
+            this.location = location;
             assert location == null || operationFailure == null
                     : "expected either failure to be null or translog location to be null, " +
                     "but found: [" + location + "] translog location and [" + operationFailure + "] failure";
@@ -138,13 +147,16 @@ public abstract class TransportWriteAction<
     /**
      * Result of taking the action on the replica.
      */
-    protected class WriteReplicaResult extends ReplicaResult implements RespondingWriteResult {
+    protected static class WriteReplicaResult<ReplicaRequest extends ReplicatedWriteRequest<ReplicaRequest>>
+            extends ReplicaResult implements RespondingWriteResult {
+        public final Location location;
         boolean finishedAsyncActions;
         private ActionListener<TransportResponse.Empty> listener;
 
         public WriteReplicaResult(ReplicaRequest request, @Nullable Location location,
-                                  @Nullable Exception operationFailure, IndexShard replica) {
+                                  @Nullable Exception operationFailure, IndexShard replica, Logger logger) {
             super(operationFailure);
+            this.location = location;
             if (operationFailure != null) {
                 this.finishedAsyncActions = true;
             } else {
@@ -184,6 +196,16 @@ public abstract class TransportWriteAction<
         }
     }
 
+    @Override
+    protected ClusterBlockLevel globalBlockLevel() {
+        return ClusterBlockLevel.WRITE;
+    }
+
+    @Override
+    protected ClusterBlockLevel indexBlockLevel() {
+        return ClusterBlockLevel.WRITE;
+    }
+
     /**
      * callback used by {@link AsyncAfterWriteAction} to notify that all post
      * process actions have been executed
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 7670eab102a..421e2458b0d 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -111,7 +111,6 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
         private final Request request;
         private volatile ClusterStateObserver observer;
         private ShardIterator shardIt;
-        private DiscoveryNodes nodes;
 
         AsyncSingleAction(Request request, ActionListener<Response> listener) {
             this.request = request;
@@ -119,13 +118,12 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
         }
 
         public void start() {
-            this.observer = new ClusterStateObserver(clusterService, request.timeout(), logger, threadPool.getThreadContext());
-            doStart();
+            ClusterState state = clusterService.state();
+            this.observer = new ClusterStateObserver(state, clusterService, request.timeout(), logger, threadPool.getThreadContext());
+            doStart(state);
         }
 
-        protected void doStart() {
-            final ClusterState clusterState = observer.observedState();
-            nodes = clusterState.nodes();
+        protected void doStart(ClusterState clusterState) {
             try {
                 ClusterBlockException blockException = checkGlobalBlock(clusterState);
                 if (blockException != null) {
@@ -171,7 +169,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
             }
 
             request.shardId = shardIt.shardId();
-            DiscoveryNode node = nodes.get(shard.currentNodeId());
+            DiscoveryNode node = clusterState.nodes().get(shard.currentNodeId());
             transportService.sendRequest(node, shardActionName, request, transportOptions(), new TransportResponseHandler<Response>() {
 
                 @Override
@@ -221,18 +219,18 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    doStart();
+                    doStart(state);
                 }
 
                 @Override
                 public void onClusterServiceClose() {
-                    listener.onFailure(new NodeClosedException(nodes.getLocalNode()));
+                    listener.onFailure(new NodeClosedException(clusterService.localNode()));
                 }
 
                 @Override
                 public void onTimeout(TimeValue timeout) {
                     // just to be on the safe side, see if we can start it now?
-                    doStart();
+                    doStart(observer.setAndGetObservedState());
                 }
             }, request.timeout());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index ee384b819b0..9d1cf5e37e7 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -278,7 +278,6 @@ public abstract class TransportTasksAction<
                         } else {
                             NodeTaskRequest nodeRequest = new NodeTaskRequest(request);
                             nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
-                            taskManager.registerChildTask(task, node.getId());
                             transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
                                 new TransportResponseHandler<NodeTasksResponse>() {
                                     @Override
@@ -403,10 +402,10 @@ public abstract class TransportTasksAction<
         protected List<TaskOperationFailure> exceptions;
         protected List<TaskResponse> results;
 
-        public NodeTasksResponse() {
+        NodeTasksResponse() {
         }
 
-        public NodeTasksResponse(String nodeId,
+        NodeTasksResponse(String nodeId,
                                  List<TaskResponse> results,
                                  List<TaskOperationFailure> exceptions) {
             this.nodeId = nodeId;
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 233d4b0c638..8508c834a9f 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -24,14 +24,14 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiTermVectorsResponse extends ActionResponse implements Iterable<MultiTermVectorsItemResponse>, ToXContent {
+public class MultiTermVectorsResponse extends ActionResponse implements Iterable<MultiTermVectorsItemResponse>, ToXContentObject {
 
     /**
      * Represents a failure.
@@ -124,6 +124,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.DOCS);
         for (MultiTermVectorsItemResponse response : responses) {
             if (response.isFailed()) {
@@ -132,16 +133,15 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderException(builder, params, failure.getCause());
+                ElasticsearchException.generateFailureXContent(builder, params, failure.getCause(), true);
                 builder.endObject();
             } else {
                 TermVectorsResponse getResponse = response.getResponse();
-                builder.startObject();
                 getResponse.toXContent(builder, params);
-                builder.endObject();
             }
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
index 534ef4164e2..088691a5c9c 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
@@ -200,7 +200,7 @@ public final class TermVectorsFields extends Fields {
         private long sumDocFreq;
         private int docCount;
 
-        public TermVector(BytesReference termVectors, long readOffset) throws IOException {
+        TermVector(BytesReference termVectors, long readOffset) throws IOException {
             this.perFieldTermVectorInput = termVectors.streamInput();
             this.readOffset = readOffset;
             reset();
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index b83713e3a6a..cba14fecf9c 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.termvectors;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.RealtimeRequest;
 import org.elasticsearch.action.ValidateActions;
@@ -33,7 +34,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 
 import java.io.IOException;
@@ -63,6 +66,8 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
 
     private BytesReference doc;
 
+    private XContentType xContentType;
+
     private String routing;
 
     private String parent;
@@ -156,8 +161,9 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         super(other.index());
         this.id = other.id();
         this.type = other.type();
-        if (this.doc != null) {
+        if (other.doc != null) {
             this.doc = new BytesArray(other.doc().toBytesRef(), true);
+            this.xContentType = other.xContentType;
         }
         this.flagsEnum = other.getFlags().clone();
         this.preference = other.preference();
@@ -225,22 +231,36 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         return doc;
     }
 
-    /**
-     * Sets an artificial document from which term vectors are requested for.
-     */
-    public TermVectorsRequest doc(XContentBuilder documentBuilder) {
-        return this.doc(documentBuilder.bytes(), true);
+    public XContentType xContentType() {
+        return xContentType;
     }
 
     /**
      * Sets an artificial document from which term vectors are requested for.
      */
+    public TermVectorsRequest doc(XContentBuilder documentBuilder) {
+        return this.doc(documentBuilder.bytes(), true, documentBuilder.contentType());
+    }
+
+    /**
+     * Sets an artificial document from which term vectors are requested for.
+     * @deprecated use {@link #doc(BytesReference, boolean, XContentType)} to avoid content auto detection
+     */
+    @Deprecated
     public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId) {
+        return this.doc(doc, generateRandomId, XContentFactory.xContentType(doc));
+    }
+
+    /**
+     * Sets an artificial document from which term vectors are requested for.
+     */
+    public TermVectorsRequest doc(BytesReference doc, boolean generateRandomId, XContentType xContentType) {
         // assign a random id to this artificial document, for routing
         if (generateRandomId) {
             this.id(String.valueOf(randomInt.getAndAdd(1)));
         }
         this.doc = doc;
+        this.xContentType = xContentType;
         return this;
     }
 
@@ -479,6 +499,11 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
 
         if (in.readBoolean()) {
             doc = in.readBytesReference();
+            if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+                xContentType = XContentType.readFrom(in);
+            } else {
+                xContentType = XContentFactory.xContentType(doc);
+            }
         }
         routing = in.readOptionalString();
         parent = in.readOptionalString();
@@ -519,6 +544,9 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         out.writeBoolean(doc != null);
         if (doc != null) {
             out.writeBytesReference(doc);
+            if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+                xContentType.writeTo(out);
+            }
         }
         out.writeOptionalString(routing);
         out.writeOptionalString(parent);
@@ -549,7 +577,7 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         out.writeLong(version);
     }
 
-    public static enum Flag {
+    public enum Flag {
         // Do not change the order of these flags we use
         // the ordinal for encoding! Only append to the end!
         Positions, Offsets, Payloads, FieldStatistics, TermStatistics
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index 964aa00b5c3..c63400be7e9 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.dfs.AggregatedDfs;
 
@@ -46,7 +46,7 @@ import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.Set;
 
-public class TermVectorsResponse extends ActionResponse implements ToXContent {
+public class TermVectorsResponse extends ActionResponse implements ToXContentObject {
 
     private static class FieldStrings {
         // term statistics strings
@@ -174,6 +174,7 @@ public class TermVectorsResponse extends ActionResponse implements ToXContent {
         assert index != null;
         assert type != null;
         assert id != null;
+        builder.startObject();
         builder.field(FieldStrings._INDEX, index);
         builder.field(FieldStrings._TYPE, type);
         if (!isArtificial()) {
@@ -182,15 +183,15 @@ public class TermVectorsResponse extends ActionResponse implements ToXContent {
         builder.field(FieldStrings._VERSION, docVersion);
         builder.field(FieldStrings.FOUND, isExists());
         builder.field(FieldStrings.TOOK, tookInMillis);
-        if (!isExists()) {
-            return builder;
-        }
-        builder.startObject(FieldStrings.TERM_VECTORS);
-        final CharsRefBuilder spare = new CharsRefBuilder();
-        Fields theFields = getFields();
-        Iterator<String> fieldIter = theFields.iterator();
-        while (fieldIter.hasNext()) {
-            buildField(builder, spare, theFields, fieldIter);
+        if (isExists()) {
+            builder.startObject(FieldStrings.TERM_VECTORS);
+            final CharsRefBuilder spare = new CharsRefBuilder();
+            Fields theFields = getFields();
+            Iterator<String> fieldIter = theFields.iterator();
+            while (fieldIter.hasNext()) {
+                buildField(builder, spare, theFields, fieldIter);
+            }
+            builder.endObject();
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 45e4aa8afdd..0235dd95a4b 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -19,19 +19,17 @@
 
 package org.elasticsearch.action.update;
 
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.bulk.TransportBulkAction;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.TransportActions;
@@ -63,11 +61,12 @@ import java.util.Collections;
 import java.util.Map;
 
 import static org.elasticsearch.ExceptionsHelper.unwrapCause;
+import static org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction.toSingleItemBulkRequest;
+import static org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction.wrapBulkResponse;
 
 public class TransportUpdateAction extends TransportInstanceSingleOperationAction<UpdateRequest, UpdateResponse> {
 
-    private final TransportDeleteAction deleteAction;
-    private final TransportIndexAction indexAction;
+    private final TransportBulkAction bulkAction;
     private final AutoCreateIndex autoCreateIndex;
     private final TransportCreateIndexAction createIndexAction;
     private final UpdateHelper updateHelper;
@@ -75,12 +74,10 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
 
     @Inject
     public TransportUpdateAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
-                                 TransportIndexAction indexAction, TransportDeleteAction deleteAction, TransportCreateIndexAction createIndexAction,
-                                 UpdateHelper updateHelper, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                 IndicesService indicesService, AutoCreateIndex autoCreateIndex) {
+                                 TransportBulkAction bulkAction, TransportCreateIndexAction createIndexAction, UpdateHelper updateHelper, ActionFilters actionFilters,
+                                 IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService, AutoCreateIndex autoCreateIndex) {
         super(settings, UpdateAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, UpdateRequest::new);
-        this.indexAction = indexAction;
-        this.deleteAction = deleteAction;
+        this.bulkAction = bulkAction;
         this.createIndexAction = createIndexAction;
         this.updateHelper = updateHelper;
         this.indicesService = indicesService;
@@ -162,7 +159,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 return new PlainShardIterator(shardIterator.shardId(), Collections.singletonList(shard));
             }
         }
-        return new PlainShardIterator(shardIterator.shardId(), Collections.<ShardRouting>emptyList());
+        return new PlainShardIterator(shardIterator.shardId(), Collections.emptyList());
     }
 
     @Override
@@ -180,100 +177,46 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 IndexRequest upsertRequest = result.action();
                 // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request
                 final BytesReference upsertSourceBytes = upsertRequest.source();
-                indexAction.execute(upsertRequest, new ActionListener<IndexResponse>() {
-                    @Override
-                    public void onResponse(IndexResponse response) {
-                        UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
-                        if ((request.fetchSource() != null && request.fetchSource().fetchSource()) ||
-                            (request.fields() != null && request.fields().length > 0)) {
-                            Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(upsertSourceBytes, true);
-                            update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes));
-                        } else {
-                            update.setGetResult(null);
-                        }
-                        update.setForcedRefresh(response.forcedRefresh());
-                        listener.onResponse(update);
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        final Throwable cause = ExceptionsHelper.unwrapCause(e);
-                        if (cause instanceof VersionConflictEngineException) {
-                            if (retryCount < request.retryOnConflict()) {
-                                logger.trace("Retry attempt [{}] of [{}] on version conflict on [{}][{}][{}]",
-                                        retryCount + 1, request.retryOnConflict(), request.index(), request.getShardId(), request.id());
-                                threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
-                                    @Override
-                                    protected void doRun() {
-                                        shardOperation(request, listener, retryCount + 1);
-                                    }
-                                });
-                                return;
+                bulkAction.execute(toSingleItemBulkRequest(upsertRequest), wrapBulkResponse(
+                        ActionListener.<IndexResponse>wrap(response -> {
+                            UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
+                            if ((request.fetchSource() != null && request.fetchSource().fetchSource()) ||
+                                    (request.fields() != null && request.fields().length > 0)) {
+                                Tuple<XContentType, Map<String, Object>> sourceAndContent =
+                                        XContentHelper.convertToMap(upsertSourceBytes, true, upsertRequest.getContentType());
+                                update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), upsertSourceBytes));
+                            } else {
+                                update.setGetResult(null);
                             }
-                        }
-                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
-                    }
-                });
+                            update.setForcedRefresh(response.forcedRefresh());
+                            listener.onResponse(update);
+                        }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount)))
+                );
+
                 break;
             case UPDATED:
                 IndexRequest indexRequest = result.action();
                 // we fetch it from the index request so we don't generate the bytes twice, its already done in the index request
                 final BytesReference indexSourceBytes = indexRequest.source();
-                indexAction.execute(indexRequest, new ActionListener<IndexResponse>() {
-                    @Override
-                    public void onResponse(IndexResponse response) {
-                        UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
-                        update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), indexSourceBytes));
-                        update.setForcedRefresh(response.forcedRefresh());
-                        listener.onResponse(update);
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        final Throwable cause = unwrapCause(e);
-                        if (cause instanceof VersionConflictEngineException) {
-                            if (retryCount < request.retryOnConflict()) {
-                                threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
-                                    @Override
-                                    protected void doRun() {
-                                        shardOperation(request, listener, retryCount + 1);
-                                    }
-                                });
-                                return;
-                            }
-                        }
-                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
-                    }
-                });
+                bulkAction.execute(toSingleItemBulkRequest(indexRequest), wrapBulkResponse(
+                        ActionListener.<IndexResponse>wrap(response -> {
+                            UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
+                            update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), indexSourceBytes));
+                            update.setForcedRefresh(response.forcedRefresh());
+                            listener.onResponse(update);
+                        }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount)))
+                );
                 break;
             case DELETED:
                 DeleteRequest deleteRequest = result.action();
-                deleteAction.execute(deleteRequest, new ActionListener<DeleteResponse>() {
-                    @Override
-                    public void onResponse(DeleteResponse response) {
-                        UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
-                        update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), null));
-                        update.setForcedRefresh(response.forcedRefresh());
-                        listener.onResponse(update);
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        final Throwable cause = unwrapCause(e);
-                        if (cause instanceof VersionConflictEngineException) {
-                            if (retryCount < request.retryOnConflict()) {
-                                threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
-                                    @Override
-                                    protected void doRun() {
-                                        shardOperation(request, listener, retryCount + 1);
-                                    }
-                                });
-                                return;
-                            }
-                        }
-                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
-                    }
-                });
+                bulkAction.execute(toSingleItemBulkRequest(deleteRequest), wrapBulkResponse(
+                        ActionListener.<DeleteResponse>wrap(response -> {
+                            UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getSeqNo(), response.getVersion(), response.getResult());
+                            update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), null));
+                            update.setForcedRefresh(response.forcedRefresh());
+                            listener.onResponse(update);
+                        }, exception -> handleUpdateFailureWithRetry(listener, request, exception, retryCount)))
+                );
                 break;
             case NOOP:
                 UpdateResponse update = result.action();
@@ -290,4 +233,23 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 throw new IllegalStateException("Illegal result " + result.getResponseResult());
         }
     }
+
+    private void handleUpdateFailureWithRetry(final ActionListener<UpdateResponse> listener, final UpdateRequest request,
+                                              final Exception failure, int retryCount) {
+        final Throwable cause = unwrapCause(failure);
+        if (cause instanceof VersionConflictEngineException) {
+            if (retryCount < request.retryOnConflict()) {
+                logger.trace("Retry attempt [{}] of [{}] on version conflict on [{}][{}][{}]",
+                        retryCount + 1, request.retryOnConflict(), request.index(), request.getShardId(), request.id());
+                threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
+                    @Override
+                    protected void doRun() {
+                        shardOperation(request, listener, retryCount + 1);
+                    }
+                });
+                return;
+            }
+        }
+        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index a812cb22eb6..504a297627d 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -27,11 +27,8 @@ import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -53,8 +50,6 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
         implements DocWriteRequest<UpdateRequest>, WriteRequest<UpdateRequest> {
-    private static final DeprecationLogger DEPRECATION_LOGGER =
-        new DeprecationLogger(Loggers.getLogger(UpdateRequest.class));
 
     private String type;
     private String id;
@@ -560,7 +555,9 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
 
     /**
      * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequest doc(String source) {
         safeDoc().source(source);
         return this;
@@ -569,6 +566,16 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequest doc(String source, XContentType xContentType) {
+        safeDoc().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequest doc(byte[] source) {
         safeDoc().source(source);
         return this;
@@ -577,11 +584,29 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequest doc(byte[] source, XContentType xContentType) {
+        safeDoc().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #doc(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequest doc(byte[] source, int offset, int length) {
         safeDoc().source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     */
+    public UpdateRequest doc(byte[] source, int offset, int length, XContentType xContentType) {
+        safeDoc().source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the doc to use for updates when a script is not specified, the doc provided
      * is a field and value pairs.
@@ -592,10 +617,11 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     }
 
     /**
-     * Sets the doc to use for updates when a script is not specified.
+     * Sets the doc to use for updates when a script is not specified, the doc provided
+     * is a field and value pairs.
      */
-    public UpdateRequest doc(String field, Object value) {
-        safeDoc().source(field, value);
+    public UpdateRequest doc(XContentType xContentType, Object... source) {
+        safeDoc().source(xContentType, source);
         return this;
     }
 
@@ -645,7 +671,9 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
 
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequest upsert(String source) {
         safeUpsertRequest().source(source);
         return this;
@@ -654,6 +682,16 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequest upsert(String source, XContentType xContentType) {
+        safeUpsertRequest().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequest upsert(byte[] source) {
         safeUpsertRequest().source(source);
         return this;
@@ -662,11 +700,29 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequest upsert(byte[] source, XContentType xContentType) {
+        safeUpsertRequest().source(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #upsert(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequest upsert(byte[] source, int offset, int length) {
         safeUpsertRequest().source(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     */
+    public UpdateRequest upsert(byte[] source, int offset, int length, XContentType xContentType) {
+        safeUpsertRequest().source(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the doc source of the update request to be used when the document does not exists. The doc
      * includes field and value pairs.
@@ -676,6 +732,15 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists. The doc
+     * includes field and value pairs.
+     */
+    public UpdateRequest upsert(XContentType xContentType, Object... source) {
+        safeUpsertRequest().source(xContentType, source);
+        return this;
+    }
+
     public IndexRequest upsertRequest() {
         return this.upsertRequest;
     }
@@ -714,7 +779,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if ("script".equals(currentFieldName)) {
-                script = Script.parse(parser, ParseFieldMatcher.EMPTY);
+                script = Script.parse(parser);
             } else if ("scripted_upsert".equals(currentFieldName)) {
                 scriptedUpsert = parser.booleanValue();
             } else if ("upsert".equals(currentFieldName)) {
@@ -740,7 +805,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
                     fields(fields.toArray(new String[fields.size()]));
                 }
             } else if ("_source".equals(currentFieldName)) {
-                fetchSourceContext = FetchSourceContext.parse(parser);
+                fetchSourceContext = FetchSourceContext.fromXContent(parser);
             }
         }
         if (script != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
index 50d84a24129..1a4d4077b10 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
@@ -223,7 +223,9 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
 
     /**
      * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequestBuilder setDoc(String source) {
         request.doc(source);
         return this;
@@ -232,6 +234,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequestBuilder setDoc(String source, XContentType xContentType) {
+        request.doc(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setDoc(byte[] source) {
         request.doc(source);
         return this;
@@ -240,6 +252,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
+    public UpdateRequestBuilder setDoc(byte[] source, XContentType xContentType) {
+        request.doc(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc to use for updates when a script is not specified.
+     * @deprecated use {@link #setDoc(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setDoc(byte[] source, int offset, int length) {
         request.doc(source, offset, length);
         return this;
@@ -248,8 +270,8 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
     /**
      * Sets the doc to use for updates when a script is not specified.
      */
-    public UpdateRequestBuilder setDoc(String field, Object value) {
-        request.doc(field, value);
+    public UpdateRequestBuilder setDoc(byte[] source, int offset, int length, XContentType xContentType) {
+        request.doc(source, offset, length, xContentType);
         return this;
     }
 
@@ -262,6 +284,15 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
         return this;
     }
 
+    /**
+     * Sets the doc to use for updates when a script is not specified, the doc provided
+     * is a field and value pairs.
+     */
+    public UpdateRequestBuilder setDoc(XContentType xContentType, Object... source) {
+        request.doc(xContentType, source);
+        return this;
+    }
+
     /**
      * Sets the index request to be used if the document does not exists. Otherwise, a {@link org.elasticsearch.index.engine.DocumentMissingException}
      * is thrown.
@@ -297,7 +328,9 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
 
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setUpsert(String, XContentType)}
      */
+    @Deprecated
     public UpdateRequestBuilder setUpsert(String source) {
         request.upsert(source);
         return this;
@@ -306,6 +339,16 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequestBuilder setUpsert(String source, XContentType xContentType) {
+        request.upsert(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setDoc(byte[], XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setUpsert(byte[] source) {
         request.upsert(source);
         return this;
@@ -314,11 +357,29 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
     /**
      * Sets the doc source of the update request to be used when the document does not exists.
      */
+    public UpdateRequestBuilder setUpsert(byte[] source, XContentType xContentType) {
+        request.upsert(source, xContentType);
+        return this;
+    }
+
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     * @deprecated use {@link #setUpsert(byte[], int, int, XContentType)}
+     */
+    @Deprecated
     public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length) {
         request.upsert(source, offset, length);
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists.
+     */
+    public UpdateRequestBuilder setUpsert(byte[] source, int offset, int length, XContentType xContentType) {
+        request.upsert(source, offset, length, xContentType);
+        return this;
+    }
+
     /**
      * Sets the doc source of the update request to be used when the document does not exists. The doc
      * includes field and value pairs.
@@ -328,6 +389,15 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
         return this;
     }
 
+    /**
+     * Sets the doc source of the update request to be used when the document does not exists. The doc
+     * includes field and value pairs.
+     */
+    public UpdateRequestBuilder setUpsert(XContentType xContentType, Object... source) {
+        request.upsert(xContentType, source);
+        return this;
+    }
+
     /**
      * Sets whether the specified doc parameter should be used as upsert document.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java b/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
index 63c6a4e4747..e952763bbf2 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateResponse.java
@@ -20,18 +20,26 @@
 package org.elasticsearch.action.update;
 
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.function.BiConsumer;
 
 public class UpdateResponse extends DocWriteResponse {
 
+    private static final String GET = "get";
+
     private GetResult getResult;
 
     public UpdateResponse() {
@@ -82,15 +90,11 @@ public class UpdateResponse extends DocWriteResponse {
         }
     }
 
-    static final class Fields {
-        static final String GET = "get";
-    }
-
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        super.toXContent(builder, params);
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerToXContent(builder, params);
         if (getGetResult() != null) {
-            builder.startObject(Fields.GET);
+            builder.startObject(GET);
             getGetResult().toXContentEmbedded(builder, params);
             builder.endObject();
         }
@@ -109,4 +113,45 @@ public class UpdateResponse extends DocWriteResponse {
         builder.append(",shards=").append(getShardInfo());
         return builder.append("]").toString();
     }
+
+    private static final ConstructingObjectParser<UpdateResponse, Void> PARSER;
+    static {
+        PARSER = new ConstructingObjectParser<>(UpdateResponse.class.getName(),
+                args -> {
+                    // index uuid and shard id are unknown and can't be parsed back for now.
+                    String index = (String) args[0];
+                    ShardId shardId = new ShardId(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), -1);
+                    String type = (String) args[1];
+                    String id = (String) args[2];
+                    long version = (long) args[3];
+                    ShardInfo shardInfo = (ShardInfo) args[5];
+                    Long seqNo = (Long) args[6];
+
+                    Result result = null;
+                    for (Result r : Result.values()) {
+                        if (r.getLowercase().equals(args[4])) {
+                            result = r;
+                            break;
+                        }
+                    }
+
+                    UpdateResponse updateResponse = null;
+                    if (shardInfo != null && seqNo != null) {
+                        updateResponse = new UpdateResponse(shardInfo, shardId, type, id, seqNo, version, result);
+                    } else {
+                        updateResponse = new UpdateResponse(shardId, type, id, version, result);
+                    }
+                    return updateResponse;
+                });
+
+        DocWriteResponse.declareParserFields(PARSER);
+        BiConsumer<UpdateResponse, GetResult> setGetResult = (update, get) ->
+            update.setGetResult(new GetResult(update.getIndex(), update.getType(), update.getId(), update.getVersion(),
+                    get.isExists(), get.internalSourceRef(), get.getFields()));
+        PARSER.declareObject(setGetResult, (parser, context) -> GetResult.fromXContentEmbedded(parser), new ParseField(GET));
+    }
+
+    public static UpdateResponse fromXContent(XContentParser parser) {
+        return PARSER.apply(parser, null);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 7cac8415e6e..d3766cc958c 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -30,16 +30,17 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.PidFile;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.inject.CreationException;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.network.IfConfig;
+import org.elasticsearch.common.settings.KeyStoreWrapper;
+import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.env.Environment;
@@ -48,7 +49,7 @@ import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.process.ProcessProbe;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.InternalSettingsPreparer;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -58,9 +59,7 @@ import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
+import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 
 /**
@@ -209,6 +208,9 @@ final class Bootstrap {
             throw new BootstrapException(e);
         }
 
+        // Log ifconfig output before SecurityManager is installed
+        IfConfig.logIfNecessary();
+
         // install SM after natives, shutdown hooks, etc.
         try {
             Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
@@ -226,13 +228,38 @@ final class Bootstrap {
         };
     }
 
-    private static Environment initialEnvironment(boolean foreground, Path pidFile, Map<String, String> esSettings) {
+    private static SecureSettings loadSecureSettings(Environment initialEnv) throws BootstrapException {
+        final KeyStoreWrapper keystore;
+        try {
+            keystore = KeyStoreWrapper.load(initialEnv.configFile());
+        } catch (IOException e) {
+            throw new BootstrapException(e);
+        }
+        if (keystore == null) {
+            return null; // no keystore
+        }
+
+        try {
+            keystore.decrypt(new char[0] /* TODO: read password from stdin */);
+        } catch (Exception e) {
+            throw new BootstrapException(e);
+        }
+        return keystore;
+    }
+
+
+    private static Environment createEnvironment(boolean foreground, Path pidFile,
+                                                 SecureSettings secureSettings, Settings initialSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
         if (pidFile != null) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
-        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, esSettings);
+        builder.put(initialSettings);
+        if (secureSettings != null) {
+            builder.setSecureSettings(secureSettings);
+        }
+        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, Collections.emptyMap());
     }
 
     private void start() throws NodeValidationException {
@@ -262,7 +289,7 @@ final class Bootstrap {
             final boolean foreground,
             final Path pidFile,
             final boolean quiet,
-            final Map<String, String> esSettings) throws BootstrapException, NodeValidationException, UserException {
+            final Environment initialEnv) throws BootstrapException, NodeValidationException, UserException {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
@@ -272,7 +299,8 @@ final class Bootstrap {
 
         INSTANCE = new Bootstrap();
 
-        Environment environment = initialEnvironment(foreground, pidFile, esSettings);
+        final SecureSettings keystore = loadSecureSettings(initialEnv);
+        Environment environment = createEnvironment(foreground, pidFile, keystore, initialEnv.settings());
         try {
             LogConfigurator.configure(environment);
         } catch (IOException e) {
@@ -310,6 +338,14 @@ final class Bootstrap {
 
             INSTANCE.setup(true, environment);
 
+            /* TODO: close this once s3 repository doesn't try to read during repository construction
+            try {
+                // any secure settings must be read during node construction
+                IOUtils.close(keystore);
+            } catch (IOException e) {
+                throw new BootstrapException(e);
+            }*/
+
             INSTANCE.start();
 
             if (closeStandardStreams) {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
index cb1b93aef39..231b9ac6d03 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java
@@ -207,7 +207,7 @@ final class BootstrapChecks {
 
     static class OsXFileDescriptorCheck extends FileDescriptorCheck {
 
-        public OsXFileDescriptorCheck() {
+        OsXFileDescriptorCheck() {
             // see constant OPEN_MAX defined in
             // /usr/include/sys/syslimits.h on OS X and its use in JVM
             // initialization in int os:init_2(void) defined in the JVM
@@ -258,7 +258,7 @@ final class BootstrapChecks {
 
         private final boolean mlockallSet;
 
-        public MlockallCheck(final boolean mlockAllSet) {
+        MlockallCheck(final boolean mlockAllSet) {
             this.mlockallSet = mlockAllSet;
         }
 
@@ -360,7 +360,7 @@ final class BootstrapChecks {
         // visible for testing
         long getMaxMapCount(Logger logger) {
             final Path path = getProcSysVmMaxMapCountPath();
-            try (final BufferedReader bufferedReader = getBufferedReader(path)) {
+            try (BufferedReader bufferedReader = getBufferedReader(path)) {
                 final String rawProcSysVmMaxMapCount = readProcSysVmMaxMapCount(bufferedReader);
                 if (rawProcSysVmMaxMapCount != null) {
                     try {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java
index 540a732dfae..635afaf9599 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapException.java
@@ -20,14 +20,14 @@
 package org.elasticsearch.bootstrap;
 
 import java.nio.file.Path;
-import java.util.Map;
 
 /**
  * Wrapper exception for checked exceptions thrown during the bootstrap process. Methods invoked
  * during bootstrap should explicitly declare the checked exceptions that they can throw, rather
  * than declaring the top-level checked exception {@link Exception}. This exception exists to wrap
- * these checked exceptions so that {@link Bootstrap#init(boolean, Path, boolean, Map)} does not have to
- * declare all of these checked exceptions.
+ * these checked exceptions so that
+ * {@link Bootstrap#init(boolean, Path, boolean, org.elasticsearch.env.Environment)}
+ * does not have to declare all of these checked exceptions.
  */
 class BootstrapException extends Exception {
 
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
index 795d4d899fb..e8538daec56 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
@@ -48,7 +48,7 @@ final class ESPolicy extends Policy {
     final PermissionCollection dynamic;
     final Map<String,Policy> plugins;
 
-    public ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
+    ESPolicy(PermissionCollection dynamic, Map<String,Policy> plugins, boolean filterBadDefaults) {
         this.template = Security.readPolicy(getClass().getResource(POLICY_RESOURCE), JarHell.parseClassPath());
         this.untrusted = Security.readPolicy(getClass().getResource(UNTRUSTED_RESOURCE), new URL[0]);
         if (filterBadDefaults) {
@@ -150,7 +150,7 @@ final class ESPolicy extends Policy {
          * @param preImplies           a test that is applied to a desired permission before checking if the bad default permission that
          *                             this instance wraps implies the desired permission
          */
-        public BadDefaultPermission(final Permission badDefaultPermission, final Predicate<Permission> preImplies) {
+        BadDefaultPermission(final Permission badDefaultPermission, final Predicate<Permission> preImplies) {
             super(badDefaultPermission.getName());
             this.badDefaultPermission = badDefaultPermission;
             this.preImplies = preImplies;
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index 23a24d88f64..fda6e6cfdec 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -25,9 +25,11 @@ import joptsimple.OptionSpecBuilder;
 import joptsimple.util.PathConverter;
 import org.elasticsearch.Build;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.NodeValidationException;
 
@@ -40,7 +42,7 @@ import java.util.Map;
 /**
  * This class starts elasticsearch.
  */
-class Elasticsearch extends SettingCommand {
+class Elasticsearch extends EnvironmentAwareCommand {
 
     private final OptionSpecBuilder versionOption;
     private final OptionSpecBuilder daemonizeOption;
@@ -90,7 +92,7 @@ class Elasticsearch extends SettingCommand {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws UserException {
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws UserException {
         if (options.nonOptionArguments().isEmpty() == false) {
             throw new UserException(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
         }
@@ -109,16 +111,16 @@ class Elasticsearch extends SettingCommand {
         final boolean quiet = options.has(quietOption);
 
         try {
-            init(daemonize, pidFile, quiet, settings);
+            init(daemonize, pidFile, quiet, env);
         } catch (NodeValidationException e) {
             throw new UserException(ExitCodes.CONFIG, e.getMessage());
         }
     }
 
-    void init(final boolean daemonize, final Path pidFile, final boolean quiet, final Map<String, String> esSettings)
+    void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv)
         throws NodeValidationException, UserException {
         try {
-            Bootstrap.init(!daemonize, pidFile, quiet, esSettings);
+            Bootstrap.init(!daemonize, pidFile, quiet, initialEnv);
         } catch (BootstrapException | RuntimeException e) {
             // format exceptions to the console in a special way
             // to avoid 2MB stacktraces from guice, etc.
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
index 27d4ff69e54..d41d1879152 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
@@ -109,7 +109,7 @@ final class JNAKernel32Library {
 
         private final ConsoleCtrlHandler handler;
 
-        public NativeHandlerCallback(ConsoleCtrlHandler handler) {
+        NativeHandlerCallback(ConsoleCtrlHandler handler) {
             this.handler = handler;
         }
 
@@ -155,11 +155,11 @@ final class JNAKernel32Library {
 
     public static class SizeT extends IntegerType {
 
-        public SizeT() {
+        SizeT() {
             this(0);
         }
 
-        public SizeT(long value) {
+        SizeT(long value) {
             super(Native.SIZE_T_SIZE, value);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index f8bc436136e..1d778fdf42a 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -85,12 +85,6 @@ import java.util.Map;
  * cleanups to the scripting apis). But still it can provide some defense for users
  * that enable dynamic scripting without being fully aware of the consequences.
  * <br>
- * <h1>Disabling Security</h1>
- * SecurityManager can be disabled completely with this setting:
- * <pre>
- * es.security.manager.enabled = false
- * </pre>
- * <br>
  * <h1>Debugging Security</h1>
  * A good place to start when there is a problem is to turn on security debugging:
  * <pre>
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java
index e6c5b2e6dd1..d9ca9698717 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/SystemCallFilter.java
@@ -154,7 +154,7 @@ final class SystemCallFilter {
         public short   len;           // number of filters
         public Pointer filter;        // filters
 
-        public SockFProg(SockFilter filters[]) {
+        SockFProg(SockFilter filters[]) {
             len = (short) filters.length;
             // serialize struct sock_filter * explicitly, its less confusing than the JNA magic we would need
             Memory filter = new Memory(len * 8);
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
index 73687393dcd..99a9a7e43d9 100644
--- a/core/src/main/java/org/elasticsearch/cli/Command.java
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -65,8 +65,8 @@ public abstract class Command implements Closeable {
                     this.close();
                 } catch (final IOException e) {
                     try (
-                        final StringWriter sw = new StringWriter();
-                        final PrintWriter pw = new PrintWriter(sw)) {
+                        StringWriter sw = new StringWriter();
+                        PrintWriter pw = new PrintWriter(sw)) {
                         e.printStackTrace(pw);
                         terminal.println(sw.toString());
                     } catch (final IOException impossible) {
diff --git a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java
similarity index 75%
rename from core/src/main/java/org/elasticsearch/cli/SettingCommand.java
rename to core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java
index 17f7c9e5204..8372a6b8ab8 100644
--- a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
+++ b/core/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java
@@ -22,16 +22,20 @@ package org.elasticsearch.cli;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import joptsimple.util.KeyValuePair;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.InternalSettingsPreparer;
 
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
-public abstract class SettingCommand extends Command {
+/** A cli command which requires an {@link org.elasticsearch.env.Environment} to use current paths and settings. */
+public abstract class EnvironmentAwareCommand extends Command {
 
     private final OptionSpec<KeyValuePair> settingOption;
 
-    public SettingCommand(String description) {
+    public EnvironmentAwareCommand(String description) {
         super(description);
         this.settingOption = parser.accepts("E", "Configure a setting").withRequiredArg().ofType(KeyValuePair.class);
     }
@@ -51,9 +55,15 @@ public abstract class SettingCommand extends Command {
         putSystemPropertyIfSettingIsMissing(settings, "path.home", "es.path.home");
         putSystemPropertyIfSettingIsMissing(settings, "path.logs", "es.path.logs");
 
-        execute(terminal, options, settings);
+        execute(terminal, options, createEnv(terminal, settings));
     }
 
+    /** Create an {@link Environment} for the command to use. Overrideable for tests. */
+    protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+        return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+    }
+
+    /** Ensure the given setting exists, reading it from system properties if not already set. */
     protected static void putSystemPropertyIfSettingIsMissing(final Map<String, String> settings, final String setting, final String key) {
         final String value = System.getProperty(key);
         if (value != null) {
@@ -72,6 +82,7 @@ public abstract class SettingCommand extends Command {
         }
     }
 
-    protected abstract void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception;
+    /** Execute the command with the initialized {@link Environment}. */
+    protected abstract void execute(Terminal terminal, OptionSet options, Environment env) throws Exception;
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cli/Terminal.java b/core/src/main/java/org/elasticsearch/cli/Terminal.java
index 58eb5012d07..cd7fc76e681 100644
--- a/core/src/main/java/org/elasticsearch/cli/Terminal.java
+++ b/core/src/main/java/org/elasticsearch/cli/Terminal.java
@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
+import java.util.Locale;
 
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
@@ -92,6 +93,26 @@ public abstract class Terminal {
         }
     }
 
+    /**
+     * Prompt for a yes or no answer from the user. This method will loop until 'y' or 'n'
+     * (or the default empty value) is entered.
+     */
+    public final boolean promptYesNo(String prompt, boolean defaultYes) {
+        String answerPrompt = defaultYes ? " [Y/n]" : " [y/N]";
+        while (true) {
+            String answer = readText(prompt + answerPrompt).toLowerCase(Locale.ROOT);
+            if (answer.isEmpty()) {
+                return defaultYes;
+            }
+            boolean answerYes = answer.equals("y");
+            if (answerYes == false && answer.equals("n") == false) {
+                println("Did not understand answer '" + answer + "'");
+                continue;
+            }
+            return answerYes;
+        }
+    }
+
     private static class ConsoleTerminal extends Terminal {
 
         private static final Console CONSOLE = System.console();
diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 14abc77513a..3f705a215ea 100644
--- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -112,6 +112,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.TaskId;
 
 /**
@@ -545,9 +546,16 @@ public interface ClusterAdminClient extends ElasticsearchClient {
 
     /**
      * Stores an ingest pipeline
+     * @deprecated use {@link #preparePutPipeline(String, BytesReference, XContentType)}
      */
+    @Deprecated
     PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source);
 
+    /**
+     * Stores an ingest pipeline
+     */
+    PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType);
+
     /**
      * Deletes a stored ingest pipeline
      */
@@ -596,8 +604,14 @@ public interface ClusterAdminClient extends ElasticsearchClient {
     /**
      * Simulates an ingest pipeline
      */
+    @Deprecated
     SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source);
 
+    /**
+     * Simulates an ingest pipeline
+     */
+    SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType);
+
     /**
      * Explain the allocation of a shard
      */
diff --git a/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java b/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
index 84438ff6d1a..c6aa4991a82 100644
--- a/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ElasticsearchClient.java
@@ -41,7 +41,7 @@ public interface ElasticsearchClient {
      * @return A future allowing to get back the response.
      */
     <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(
-            final Action<Request, Response, RequestBuilder> action, final Request request);
+            Action<Request, Response, RequestBuilder> action, Request request);
 
     /**
      * Executes a generic action, denoted by an {@link Action}.
@@ -54,7 +54,7 @@ public interface ElasticsearchClient {
      * @param <RequestBuilder> The request builder type.
      */
     <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(
-            final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener);
+            Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener);
 
     /**
      * Prepares a request builder to execute, specified by {@link Action}.
@@ -66,7 +66,7 @@ public interface ElasticsearchClient {
      * @return The request builder, that can, at a later stage, execute the request.
      */
     <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(
-            final Action<Request, Response, RequestBuilder> action);
+            Action<Request, Response, RequestBuilder> action);
 
     /**
      * Returns the threadpool used to execute requests on this client
diff --git a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
index 6c3aa071ba3..e4f26b15702 100644
--- a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -28,12 +28,14 @@ import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.support.AbstractClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskListener;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Map;
+import java.util.function.Supplier;
 
 /**
  * Client that executes actions on the local node.
@@ -41,13 +43,19 @@ import java.util.Map;
 public class NodeClient extends AbstractClient {
 
     private Map<GenericAction, TransportAction> actions;
+    /**
+     * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
+     * {@link #executeLocally(GenericAction, ActionRequest, TaskListener)}.
+     */
+    private Supplier<String> localNodeId;
 
     public NodeClient(Settings settings, ThreadPool threadPool) {
         super(settings, threadPool);
     }
 
-    public void initialize(Map<GenericAction, TransportAction> actions) {
+    public void initialize(Map<GenericAction, TransportAction> actions, Supplier<String> localNodeId) {
         this.actions = actions;
+        this.localNodeId = localNodeId;
     }
 
     @Override
@@ -85,6 +93,14 @@ public class NodeClient extends AbstractClient {
         return transportAction(action).execute(request, listener);
     }
 
+    /**
+     * The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
+     * {@link #executeLocally(GenericAction, ActionRequest, TaskListener)}.
+     */
+    public String getLocalNodeId() {
+        return localNodeId.get();
+    }
+
     /**
      * Get the {@link TransportAction} for an {@link Action}, throwing exceptions if the action isn't available.
      */
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 075fbf1fad6..e0ebcfe70a9 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -343,6 +343,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -400,7 +401,7 @@ public abstract class AbstractClient extends AbstractComponent implements Client
         doExecute(action, request, listener);
     }
 
-    protected abstract <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener);
+    protected abstract <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener);
 
     @Override
     public ActionFuture<IndexResponse> index(final IndexRequest request) {
@@ -671,7 +672,7 @@ public abstract class AbstractClient extends AbstractComponent implements Client
         private final ClusterAdmin clusterAdmin;
         private final IndicesAdmin indicesAdmin;
 
-        public Admin(ElasticsearchClient client) {
+        Admin(ElasticsearchClient client) {
             this.clusterAdmin = new ClusterAdmin(client);
             this.indicesAdmin = new IndicesAdmin(client);
         }
@@ -691,7 +692,7 @@ public abstract class AbstractClient extends AbstractComponent implements Client
 
         private final ElasticsearchClient client;
 
-        public ClusterAdmin(ElasticsearchClient client) {
+        ClusterAdmin(ElasticsearchClient client) {
             this.client = client;
         }
 
@@ -1081,6 +1082,11 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source);
         }
 
+        @Override
+        public PutPipelineRequestBuilder preparePutPipeline(String id, BytesReference source, XContentType xContentType) {
+            return new PutPipelineRequestBuilder(this, PutPipelineAction.INSTANCE, id, source, xContentType);
+        }
+
         @Override
         public void deletePipeline(DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
             execute(DeletePipelineAction.INSTANCE, request, listener);
@@ -1131,6 +1137,11 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source);
         }
 
+        @Override
+        public SimulatePipelineRequestBuilder prepareSimulatePipeline(BytesReference source, XContentType xContentType) {
+            return new SimulatePipelineRequestBuilder(this, SimulatePipelineAction.INSTANCE, source, xContentType);
+        }
+
         @Override
         public void allocationExplain(ClusterAllocationExplainRequest request, ActionListener<ClusterAllocationExplainResponse> listener) {
             execute(ClusterAllocationExplainAction.INSTANCE, request, listener);
@@ -1199,7 +1210,7 @@ public abstract class AbstractClient extends AbstractComponent implements Client
 
         @Override
         public DeleteStoredScriptRequestBuilder prepareDeleteStoredScript(@Nullable String scriptLang, String id){
-            return prepareDeleteStoredScript().setScriptLang(scriptLang).setId(id);
+            return prepareDeleteStoredScript().setLang(scriptLang).setId(id);
         }
     }
 
@@ -1207,7 +1218,7 @@ public abstract class AbstractClient extends AbstractComponent implements Client
 
         private final ElasticsearchClient client;
 
-        public IndicesAdmin(ElasticsearchClient client) {
+        IndicesAdmin(ElasticsearchClient client) {
             this.client = client;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 4b9e4b1eac6..01fdf98ceb4 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -27,7 +27,9 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.support.AbstractClient;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.Module;
@@ -41,9 +43,10 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -63,8 +66,11 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
@@ -136,10 +142,16 @@ public abstract class TransportClient extends AbstractClient {
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
             entries.addAll(NetworkModule.getNamedWriteables());
             entries.addAll(searchModule.getNamedWriteables());
+            entries.addAll(ClusterModule.getNamedWriteables());
             entries.addAll(pluginsService.filterPlugins(Plugin.class).stream()
                                          .flatMap(p -> p.getNamedWriteables().stream())
                                          .collect(Collectors.toList()));
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+            NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
+                    searchModule.getNamedXContents().stream(),
+                    pluginsService.filterPlugins(Plugin.class).stream()
+                            .flatMap(p -> p.getNamedXContent().stream())
+                    ).flatMap(Function.identity()).collect(toList()));
 
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
@@ -147,8 +159,9 @@ public abstract class TransportClient extends AbstractClient {
                 modules.add(pluginModule);
             }
             modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
-            ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getClusterSettings(),
-                threadPool, pluginsService.filterPlugins(ActionPlugin.class));
+            ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(),
+                    settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool,
+                    pluginsService.filterPlugins(ActionPlugin.class), null, null);
             modules.add(actionModule);
 
             CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
@@ -158,10 +171,12 @@ public abstract class TransportClient extends AbstractClient {
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
             NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
-                bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
+                bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = new TransportService(settings, transport, threadPool,
-                networkModule.getTransportInterceptor(), null);
+                networkModule.getTransportInterceptor(),
+                boundTransportAddress -> DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 0),
+                    UUIDs.randomBase64UUID()), null);
             modules.add((b -> {
                 b.bind(BigArrays.class).toInstance(bigArrays);
                 b.bind(PluginsService.class).toInstance(pluginsService);
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 350a35b6e49..ea2906dab67 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -101,6 +101,21 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
 
     private final TransportClient.HostFailureListener hostFailureListener;
 
+    // TODO: migrate this to use low level connections and single type channels
+    /** {@link ConnectionProfile} to use when to connecting to the listed nodes and doing a liveness check */
+    private static final ConnectionProfile LISTED_NODES_PROFILE;
+
+    static {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+        LISTED_NODES_PROFILE = builder.build();
+    }
+
     TransportClientNodesService(Settings settings, TransportService transportService,
                                        ThreadPool threadPool, TransportClient.HostFailureListener hostFailureListener) {
         super(settings);
@@ -249,7 +264,7 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
 
         private volatile int i;
 
-        public RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
+        RetryListener(NodeListenerCallback<Response> callback, ActionListener<Response> listener,
                              List<DiscoveryNode> nodes, int index, TransportClient.HostFailureListener hostFailureListener) {
             this.callback = callback;
             this.listener = listener;
@@ -389,8 +404,8 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
                 if (!transportService.nodeConnected(listedNode)) {
                     try {
                         // its a listed node, light connect to it...
-                        logger.trace("connecting to listed node (light) [{}]", listedNode);
-                        transportService.connectToNode(listedNode, ConnectionProfile.LIGHT_PROFILE);
+                        logger.trace("connecting to listed node [{}]", listedNode);
+                        transportService.connectToNode(listedNode, LISTED_NODES_PROFILE);
                     } catch (Exception e) {
                         logger.info(
                             (Supplier<?>)
@@ -470,7 +485,7 @@ final class TransportClientNodesService extends AbstractComponent implements Clo
                                     } else {
                                         // its a listed node, light connect to it...
                                         logger.trace("connecting to listed node (light) [{}]", listedNode);
-                                        transportService.connectToNode(listedNode, ConnectionProfile.LIGHT_PROFILE);
+                                        transportService.connectToNode(listedNode, LISTED_NODES_PROFILE);
                                     }
                                 } catch (Exception e) {
                                     logger.debug(
diff --git a/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java b/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
index f9d5f33cad6..8e63bc2b9d7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/AbstractDiffable.java
@@ -40,12 +40,7 @@ public abstract class AbstractDiffable<T extends Diffable<T>> implements Diffabl
         }
     }
 
-    @Override
-    public Diff<T> readDiffFrom(StreamInput in) throws IOException {
-        return new CompleteDiff<>(this, in);
-    }
-
-    public static <T extends Diffable<T>> Diff<T> readDiffFrom(T reader, StreamInput in) throws IOException {
+    public static <T extends Diffable<T>> Diff<T> readDiffFrom(Reader<T> reader, StreamInput in) throws IOException {
         return new CompleteDiff<T>(reader, in);
     }
 
@@ -57,23 +52,23 @@ public abstract class AbstractDiffable<T extends Diffable<T>> implements Diffabl
         /**
          * Creates simple diff with changes
          */
-        public CompleteDiff(T part) {
+        CompleteDiff(T part) {
             this.part = part;
         }
 
         /**
          * Creates simple diff without changes
          */
-        public CompleteDiff() {
+        CompleteDiff() {
             this.part = null;
         }
 
         /**
          * Read simple diff from the stream
          */
-        public CompleteDiff(Diffable<T> reader, StreamInput in) throws IOException {
+        CompleteDiff(Reader<T> reader, StreamInput in) throws IOException {
             if (in.readBoolean()) {
-                this.part = reader.readFrom(in);
+                this.part = reader.read(in);
             } else {
                 this.part = null;
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java b/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
new file mode 100644
index 00000000000..fb253a1a5df
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/AbstractNamedDiffable.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * Abstract diffable object with simple diffs implementation that sends the entire object if object has changed or
+ * nothing is object remained the same. Comparing to AbstractDiffable, this class also works with NamedWriteables
+ */
+public abstract class AbstractNamedDiffable<T extends NamedDiffable<T>> implements Diffable<T>, NamedWriteable {
+
+    @Override
+    public Diff<T> diff(T previousState) {
+        if (this.get().equals(previousState)) {
+            return new CompleteNamedDiff<>(previousState.getWriteableName(), previousState.getMinimalSupportedVersion());
+        } else {
+            return new CompleteNamedDiff<>(get());
+        }
+    }
+
+    public static <T extends NamedDiffable<T>> NamedDiff<T> readDiffFrom(Class<? extends T> tClass, String name, StreamInput in)
+        throws IOException {
+        return new CompleteNamedDiff<>(tClass, name, in);
+    }
+
+    private static class CompleteNamedDiff<T extends NamedDiffable<T>> implements NamedDiff<T> {
+
+        @Nullable
+        private final T part;
+
+        private final String name;
+
+        /**
+         * A non-null value is only required for write operation, if the diff was just read from the stream the version
+         * is unnecessary.
+         */
+        @Nullable
+        private final Version minimalSupportedVersion;
+
+        /**
+         * Creates simple diff with changes
+         */
+        CompleteNamedDiff(T part) {
+            this.part = part;
+            this.name = part.getWriteableName();
+            this.minimalSupportedVersion = part.getMinimalSupportedVersion();
+        }
+
+        /**
+         * Creates simple diff without changes
+         */
+        CompleteNamedDiff(String name, Version minimalSupportedVersion) {
+            this.part = null;
+            this.name = name;
+            this.minimalSupportedVersion = minimalSupportedVersion;
+        }
+
+        /**
+         * Read simple diff from the stream
+         */
+        CompleteNamedDiff(Class<? extends T> tClass, String name, StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                this.part = in.readNamedWriteable(tClass, name);
+                this.minimalSupportedVersion = part.getMinimalSupportedVersion();
+            } else {
+                this.part = null;
+                this.minimalSupportedVersion = null; // We just read this diff, so it's not going to be written
+            }
+            this.name = name;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            assert minimalSupportedVersion != null : "shouldn't be called on diff that was de-serialized from the stream";
+            if (part != null) {
+                out.writeBoolean(true);
+                part.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
+
+        @Override
+        public T apply(T part) {
+            if (this.part != null) {
+                return this.part;
+            } else {
+                return part;
+            }
+        }
+
+        @Override
+        public String getWriteableName() {
+            return name;
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            assert minimalSupportedVersion != null : "shouldn't be called on the diff that was de-serialized from the stream";
+            return minimalSupportedVersion;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public T get() {
+        return (T) this;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 930991c443b..8fe8942662a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -22,7 +22,9 @@ package org.elasticsearch.cluster;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
@@ -30,6 +32,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -52,15 +55,25 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.plugins.ClusterPlugin;
+import org.elasticsearch.script.ScriptMetaData;
 import org.elasticsearch.tasks.TaskResultsService;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -94,6 +107,52 @@ public class ClusterModule extends AbstractModule {
         indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
     }
 
+
+    public static List<Entry> getNamedWriteables() {
+        List<Entry> entries = new ArrayList<>();
+        // Cluster State
+        registerClusterCustom(entries, SnapshotsInProgress.TYPE, SnapshotsInProgress::new, SnapshotsInProgress::readDiffFrom);
+        registerClusterCustom(entries, RestoreInProgress.TYPE, RestoreInProgress::new, RestoreInProgress::readDiffFrom);
+        registerClusterCustom(entries, SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress::new,
+            SnapshotDeletionsInProgress::readDiffFrom);
+        // Metadata
+        registerMetaDataCustom(entries, RepositoriesMetaData.TYPE, RepositoriesMetaData::new, RepositoriesMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, IngestMetadata.TYPE, IngestMetadata::new, IngestMetadata::readDiffFrom);
+        registerMetaDataCustom(entries, ScriptMetaData.TYPE, ScriptMetaData::new, ScriptMetaData::readDiffFrom);
+        registerMetaDataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
+        return entries;
+    }
+
+    public static List<NamedXContentRegistry.Entry> getNamedXWriteables() {
+        List<NamedXContentRegistry.Entry> entries = new ArrayList<>();
+        // Metadata
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(RepositoriesMetaData.TYPE),
+            RepositoriesMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IngestMetadata.TYPE),
+            IngestMetadata::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(ScriptMetaData.TYPE),
+            ScriptMetaData::fromXContent));
+        entries.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(IndexGraveyard.TYPE),
+            IndexGraveyard::fromXContent));
+        return entries;
+    }
+
+    private static <T extends ClusterState.Custom> void registerClusterCustom(List<Entry> entries, String name, Reader<? extends T> reader,
+                                                                       Reader<NamedDiff> diffReader) {
+        registerCustom(entries, ClusterState.Custom.class, name, reader, diffReader);
+    }
+
+    private static <T extends MetaData.Custom> void registerMetaDataCustom(List<Entry> entries, String name, Reader<? extends T> reader,
+                                                                       Reader<NamedDiff> diffReader) {
+        registerCustom(entries, MetaData.Custom.class, name, reader, diffReader);
+    }
+
+    private static <T extends NamedWriteable> void registerCustom(List<Entry> entries, Class<T> category, String name,
+                                                                  Reader<? extends T> reader, Reader<NamedDiff> diffReader) {
+        entries.add(new Entry(category, name, reader));
+        entries.add(new Entry(NamedDiff.class, name, diffReader));
+    }
+
     public IndexNameExpressionResolver getIndexNameExpressionResolver() {
         return indexNameExpressionResolver;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 7b6f2b55aa9..c9ef935224c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -37,20 +38,21 @@ import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.zen.PublishClusterStateAction;
 
@@ -86,36 +88,12 @@ import java.util.Set;
  */
 public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
-    public static final ClusterState PROTO = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
+    public static final ClusterState EMPTY_STATE = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
 
-    public interface Custom extends Diffable<Custom>, ToXContent {
-
-        String type();
+    public interface Custom extends NamedDiffable<Custom>, ToXContent {
     }
 
-    private static final Map<String, Custom> customPrototypes = new HashMap<>();
-
-    /**
-     * Register a custom index meta data factory. Make sure to call it from a static block.
-     */
-    public static void registerPrototype(String type, Custom proto) {
-        customPrototypes.put(type, proto);
-    }
-
-    static {
-        // register non plugin custom parts
-        registerPrototype(SnapshotsInProgress.TYPE, SnapshotsInProgress.PROTO);
-        registerPrototype(RestoreInProgress.TYPE, RestoreInProgress.PROTO);
-    }
-
-    public static <T extends Custom> T lookupPrototype(String type) {
-        @SuppressWarnings("unchecked")
-        T proto = (T) customPrototypes.get(type);
-        if (proto == null) {
-            throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "], node likely missing plugins");
-        }
-        return proto;
-    }
+    private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
 
     public static final String UNKNOWN_UUID = "_na_";
 
@@ -403,11 +381,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor1 : templateMetaData.mappings()) {
-                    byte[] mappingSource = cursor1.value.uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(cursor1.value.uncompressed()), false).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor1.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor1.key);
@@ -435,11 +409,8 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
-                    byte[] mappingSource = cursor.value.source().uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper
+                            .convertToMap(new BytesArray(cursor.value.source().uncompressed()), false).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor.key);
@@ -665,53 +636,39 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
          * @param data      input bytes
          * @param localNode used to set the local node in the cluster state.
          */
-        public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException {
-            return readFrom(StreamInput.wrap(data), localNode);
+        public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode, NamedWriteableRegistry registry) throws IOException {
+            StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(data), registry);
+            return readFrom(in, localNode);
 
         }
-
-        /**
-         * @param in        input stream
-         * @param localNode used to set the local node in the cluster state. can be null.
-         */
-        public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
-            return PROTO.readFrom(in, localNode);
-        }
     }
 
     @Override
-    public Diff diff(ClusterState previousState) {
+    public Diff<ClusterState> diff(ClusterState previousState) {
         return new ClusterStateDiff(previousState, this);
     }
 
-    @Override
-    public Diff<ClusterState> readDiffFrom(StreamInput in) throws IOException {
-        return new ClusterStateDiff(in, this);
+    public static Diff<ClusterState> readDiffFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+        return new ClusterStateDiff(in, localNode);
     }
 
-    public ClusterState readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+    public static ClusterState readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
         ClusterName clusterName = new ClusterName(in);
         Builder builder = new Builder(clusterName);
         builder.version = in.readLong();
         builder.uuid = in.readString();
-        builder.metaData = MetaData.Builder.readFrom(in);
-        builder.routingTable = RoutingTable.Builder.readFrom(in);
-        builder.nodes = DiscoveryNodes.Builder.readFrom(in, localNode);
-        builder.blocks = ClusterBlocks.Builder.readClusterBlocks(in);
+        builder.metaData = MetaData.readFrom(in);
+        builder.routingTable = RoutingTable.readFrom(in);
+        builder.nodes = DiscoveryNodes.readFrom(in, localNode);
+        builder.blocks = new ClusterBlocks(in);
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            Custom customIndexMetaData = lookupPrototype(type).readFrom(in);
-            builder.putCustom(type, customIndexMetaData);
+            Custom customIndexMetaData = in.readNamedWriteable(Custom.class);
+            builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData);
         }
         return builder.build();
     }
 
-    @Override
-    public ClusterState readFrom(StreamInput in) throws IOException {
-        return readFrom(in, nodes.getLocalNode());
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         clusterName.writeTo(out);
@@ -721,10 +678,18 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         routingTable.writeTo(out);
         nodes.writeTo(out);
         blocks.writeTo(out);
-        out.writeVInt(customs.size());
-        for (ObjectObjectCursor<String, Custom> cursor : customs) {
-            out.writeString(cursor.key);
-            cursor.value.writeTo(out);
+        // filter out custom states not supported by the other node
+        int numberOfCustoms = 0;
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                numberOfCustoms++;
+            }
+        }
+        out.writeVInt(numberOfCustoms);
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                out.writeNamedWriteable(cursor.value);
+            }
         }
     }
 
@@ -748,7 +713,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
         private final Diff<ImmutableOpenMap<String, Custom>> customs;
 
-        public ClusterStateDiff(ClusterState before, ClusterState after) {
+        ClusterStateDiff(ClusterState before, ClusterState after) {
             fromUuid = before.stateUUID;
             toUuid = after.stateUUID;
             toVersion = after.version;
@@ -757,30 +722,19 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
             nodes = after.nodes.diff(before.nodes);
             metaData = after.metaData.diff(before.metaData);
             blocks = after.blocks.diff(before.blocks);
-            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
+            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
-        public ClusterStateDiff(StreamInput in, ClusterState proto) throws IOException {
+        ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException {
             clusterName = new ClusterName(in);
             fromUuid = in.readString();
             toUuid = in.readString();
             toVersion = in.readLong();
-            routingTable = proto.routingTable.readDiffFrom(in);
-            nodes = proto.nodes.readDiffFrom(in);
-            metaData = proto.metaData.readDiffFrom(in);
-            blocks = proto.blocks.readDiffFrom(in);
-            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                    @Override
-                    public Custom read(StreamInput in, String key) throws IOException {
-                        return lookupPrototype(key).readFrom(in);
-                    }
-
-                    @Override
-                    public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                        return lookupPrototype(key).readDiffFrom(in);
-                    }
-                });
+            routingTable = RoutingTable.readDiffFrom(in);
+            nodes = DiscoveryNodes.readDiffFrom(in, localNode);
+            metaData = MetaData.readDiffFrom(in);
+            blocks = ClusterBlocks.readDiffFrom(in);
+            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
index 6ca815b2fab..96177f7b7d8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
@@ -26,8 +26,10 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 /**
  * A utility class which simplifies interacting with the cluster state in cases where
@@ -45,7 +47,7 @@ public class ClusterStateObserver {
     volatile TimeValue timeOutValue;
 
 
-    final AtomicReference<ClusterState> lastObservedState;
+    final AtomicReference<StoredState> lastObservedState;
     final TimeoutClusterStateListener clusterStateListener = new ObserverClusterStateListener();
     // observingContext is not null when waiting on cluster state changes
     final AtomicReference<ObservingContext> observingContext = new AtomicReference<>(null);
@@ -73,7 +75,7 @@ public class ClusterStateObserver {
     public ClusterStateObserver(ClusterState initialState, ClusterService clusterService, @Nullable TimeValue timeout, Logger logger,
                                 ThreadContext contextHolder) {
         this.clusterService = clusterService;
-        this.lastObservedState = new AtomicReference<>(initialState);
+        this.lastObservedState = new AtomicReference<>(new StoredState(initialState));
         this.timeOutValue = timeout;
         if (timeOutValue != null) {
             this.startTimeNS = System.nanoTime();
@@ -82,11 +84,14 @@ public class ClusterStateObserver {
         this.contextHolder = contextHolder;
     }
 
-    /** last cluster state and status observed by this observer. Note that this may not be the current one */
-    public ClusterState observedState() {
-        ClusterState state = lastObservedState.get();
-        assert state != null;
-        return state;
+    /** sets the last observed state to the currently applied cluster state and returns it */
+    public ClusterState setAndGetObservedState() {
+        if (observingContext.get() != null) {
+            throw new ElasticsearchException("cannot set current cluster state while waiting for a cluster state change");
+        }
+        ClusterState clusterState = clusterService.state();
+        lastObservedState.set(new StoredState(clusterState));
+        return clusterState;
     }
 
     /** indicates whether this observer has timedout */
@@ -114,7 +119,7 @@ public class ClusterStateObserver {
      * @param timeOutValue    a timeout for waiting. If null the global observer timeout will be used.
      */
     public void waitForNextChange(Listener listener, Predicate<ClusterState> statePredicate, @Nullable TimeValue timeOutValue) {
-
+        listener = new ContextPreservingListener(listener, contextHolder.newRestorableContext(false));
         if (observingContext.get() != null) {
             throw new ElasticsearchException("already waiting for a cluster state change");
         }
@@ -130,7 +135,7 @@ public class ClusterStateObserver {
                     logger.trace("observer timed out. notifying listener. timeout setting [{}], time since start [{}]", timeOutValue, new TimeValue(timeSinceStartMS));
                     // update to latest, in case people want to retry
                     timedOut = true;
-                    lastObservedState.set(clusterService.state());
+                    lastObservedState.set(new StoredState(clusterService.state()));
                     listener.onTimeout(timeOutValue);
                     return;
                 }
@@ -146,16 +151,14 @@ public class ClusterStateObserver {
 
         // sample a new state
         ClusterState newState = clusterService.state();
-        ClusterState lastState = lastObservedState.get();
-        if (newState != lastState && statePredicate.test(newState)) {
+        if (lastObservedState.get().sameState(newState) == false && statePredicate.test(newState)) {
             // good enough, let's go.
             logger.trace("observer: sampled state accepted by predicate ({})", newState);
-            lastObservedState.set(newState);
+            lastObservedState.set(new StoredState(newState));
             listener.onNewClusterState(newState);
         } else {
             logger.trace("observer: sampled state rejected by predicate ({}). adding listener to ClusterService", newState);
-            ObservingContext context =
-                new ObservingContext(new ContextPreservingListener(listener, contextHolder.newStoredContext()), statePredicate);
+            final ObservingContext context = new ObservingContext(listener, statePredicate);
             if (!observingContext.compareAndSet(null, context)) {
                 throw new ElasticsearchException("already waiting for a cluster state change");
             }
@@ -177,7 +180,7 @@ public class ClusterStateObserver {
                 if (observingContext.compareAndSet(context, null)) {
                     clusterService.removeTimeoutListener(this);
                     logger.trace("observer: accepting cluster state change ({})", state);
-                    lastObservedState.set(state);
+                    lastObservedState.set(new StoredState(state));
                     context.listener.onNewClusterState(state);
                 } else {
                     logger.trace("observer: predicate approved change but observing context has changed - ignoring (new cluster state version [{}])", state.version());
@@ -195,13 +198,12 @@ public class ClusterStateObserver {
                 return;
             }
             ClusterState newState = clusterService.state();
-            ClusterState lastState = lastObservedState.get();
-            if (newState != lastState && context.statePredicate.test(newState)) {
+            if (lastObservedState.get().sameState(newState) == false && context.statePredicate.test(newState)) {
                 // double check we're still listening
                 if (observingContext.compareAndSet(context, null)) {
                     logger.trace("observer: post adding listener: accepting current cluster state ({})", newState);
                     clusterService.removeTimeoutListener(this);
-                    lastObservedState.set(newState);
+                    lastObservedState.set(new StoredState(newState));
                     context.listener.onNewClusterState(newState);
                 } else {
                     logger.trace("observer: postAdded - predicate approved state but observing context has changed - ignoring ({})", newState);
@@ -230,13 +232,30 @@ public class ClusterStateObserver {
                 long timeSinceStartMS = TimeValue.nsecToMSec(System.nanoTime() - startTimeNS);
                 logger.trace("observer: timeout notification from cluster service. timeout setting [{}], time since start [{}]", timeOutValue, new TimeValue(timeSinceStartMS));
                 // update to latest, in case people want to retry
-                lastObservedState.set(clusterService.state());
+                lastObservedState.set(new StoredState(clusterService.state()));
                 timedOut = true;
                 context.listener.onTimeout(timeOutValue);
             }
         }
     }
 
+    /**
+     * The observer considers two cluster states to be the same if they have the same version and master node id (i.e. null or set)
+     */
+    private static class StoredState {
+        private final String masterNodeId;
+        private final long version;
+
+        StoredState(ClusterState clusterState) {
+            this.masterNodeId = clusterState.nodes().getMasterNodeId();
+            this.version = clusterState.version();
+        }
+
+        public boolean sameState(ClusterState clusterState) {
+            return version == clusterState.version() && Objects.equals(masterNodeId, clusterState.nodes().getMasterNodeId());
+        }
+    }
+
     public interface Listener {
 
         /** called when a new state is observed */
@@ -252,7 +271,7 @@ public class ClusterStateObserver {
         public final Listener listener;
         public final Predicate<ClusterState> statePredicate;
 
-        public ObservingContext(Listener listener, Predicate<ClusterState> statePredicate) {
+        ObservingContext(Listener listener, Predicate<ClusterState> statePredicate) {
             this.listener = listener;
             this.statePredicate = statePredicate;
         }
@@ -260,30 +279,33 @@ public class ClusterStateObserver {
 
     private static final class ContextPreservingListener implements Listener {
         private final Listener delegate;
-        private final ThreadContext.StoredContext tempContext;
+        private final Supplier<ThreadContext.StoredContext> contextSupplier;
 
 
-        private ContextPreservingListener(Listener delegate, ThreadContext.StoredContext storedContext) {
-            this.tempContext = storedContext;
+        private ContextPreservingListener(Listener delegate, Supplier<ThreadContext.StoredContext> contextSupplier) {
+            this.contextSupplier = contextSupplier;
             this.delegate = delegate;
         }
 
         @Override
         public void onNewClusterState(ClusterState state) {
-            tempContext.restore();
-            delegate.onNewClusterState(state);
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onNewClusterState(state);
+            }
         }
 
         @Override
         public void onClusterServiceClose() {
-            tempContext.restore();
-            delegate.onClusterServiceClose();
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onClusterServiceClose();
+            }
         }
 
         @Override
         public void onTimeout(TimeValue timeout) {
-            tempContext.restore();
-            delegate.onTimeout(timeout);
+            try (ThreadContext.StoredContext context  = contextSupplier.get()) {
+                delegate.onTimeout(timeout);
+            }
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index e5493eaa955..3693447cfb6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.common.Nullable;
+
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
@@ -27,10 +29,10 @@ public interface ClusterStateTaskExecutor<T> {
      * Update the cluster state based on the current state and the given tasks. Return the *same instance* if no state
      * should be changed.
      */
-    BatchResult<T> execute(ClusterState currentState, List<T> tasks) throws Exception;
+    ClusterTasksResult<T> execute(ClusterState currentState, List<T> tasks) throws Exception;
 
     /**
-     * indicates whether this task should only run if current node is master
+     * indicates whether this executor should only run if the current node is master
      */
     default boolean runOnlyOnMaster() {
         return true;
@@ -68,18 +70,22 @@ public interface ClusterStateTaskExecutor<T> {
      * Represents the result of a batched execution of cluster state update tasks
      * @param <T> the type of the cluster state update task
      */
-    class BatchResult<T> {
+    class ClusterTasksResult<T> {
+        public final boolean noMaster;
+        @Nullable
         public final ClusterState resultingState;
         public final Map<T, TaskResult> executionResults;
 
         /**
          * Construct an execution result instance with a correspondence between the tasks and their execution result
+         * @param noMaster whether this node steps down as master or has lost connection to the master
          * @param resultingState the resulting cluster state
          * @param executionResults the correspondence between tasks and their outcome
          */
-        BatchResult(ClusterState resultingState, Map<T, TaskResult> executionResults) {
+        ClusterTasksResult(boolean noMaster, ClusterState resultingState, Map<T, TaskResult> executionResults) {
             this.resultingState = resultingState;
             this.executionResults = executionResults;
+            this.noMaster = noMaster;
         }
 
         public static <T> Builder<T> builder() {
@@ -117,8 +123,13 @@ public interface ClusterStateTaskExecutor<T> {
                 return this;
             }
 
-            public BatchResult<T> build(ClusterState resultingState) {
-                return new BatchResult<>(resultingState, executionResults);
+            public ClusterTasksResult<T> build(ClusterState resultingState) {
+                return new ClusterTasksResult<>(false, resultingState, executionResults);
+            }
+
+            ClusterTasksResult<T> build(ClusterTasksResult<T> result, ClusterState previousState) {
+                return new ClusterTasksResult<>(result.noMaster, result.resultingState == null ? previousState : result.resultingState,
+                    executionResults);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
index a679d098616..b298e7e915d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
@@ -28,7 +28,7 @@ import java.util.List;
 /**
  * A task that can update the cluster state.
  */
-public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
+public abstract class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
 
     private final Priority priority;
 
@@ -41,9 +41,9 @@ public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig,
     }
 
     @Override
-    public final BatchResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
+    public final ClusterTasksResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
         ClusterState result = execute(currentState);
-        return BatchResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
+        return ClusterTasksResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
     }
 
     @Override
@@ -75,4 +75,13 @@ public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig,
     public Priority priority() {
         return priority;
     }
+
+    /**
+     * Marked as final as cluster state update tasks should only run on master.
+     * For local requests, use {@link LocalClusterUpdateTask} instead.
+     */
+    @Override
+    public final boolean runOnlyOnMaster() {
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/Diff.java b/core/src/main/java/org/elasticsearch/cluster/Diff.java
index 76535a4b763..4e980e36868 100644
--- a/core/src/main/java/org/elasticsearch/cluster/Diff.java
+++ b/core/src/main/java/org/elasticsearch/cluster/Diff.java
@@ -19,22 +19,15 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
+import org.elasticsearch.common.io.stream.Writeable;
 
 /**
  * Represents difference between states of cluster state parts
  */
-public interface Diff<T> {
+public interface Diff<T> extends Writeable {
 
     /**
      * Applies difference to the specified part and returns the resulted part
      */
     T apply(T part);
-
-    /**
-     * Writes the differences into the output stream
-     */
-    void writeTo(StreamOutput out) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/Diffable.java b/core/src/main/java/org/elasticsearch/cluster/Diffable.java
index b039f5e9b8b..57d5ea9ed1f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/Diffable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/Diffable.java
@@ -34,13 +34,4 @@ public interface Diffable<T> extends Writeable {
      */
     Diff<T> diff(T previousState);
 
-    /**
-     * Reads the {@link org.elasticsearch.cluster.Diff} from StreamInput
-     */
-    Diff<T> readDiffFrom(StreamInput in) throws IOException;
-
-    /**
-     * Reads an object of this type from the provided {@linkplain StreamInput}. The receiving instance remains unchanged.
-     */
-    T readFrom(StreamInput in) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
index 1a3557890dd..ba769c34d0e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
+++ b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
@@ -23,10 +23,12 @@ import com.carrotsearch.hppc.cursors.IntCursor;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -74,7 +76,7 @@ public final class DiffableUtils {
     /**
      * Calculates diff between two ImmutableOpenMaps of non-diffable objects
      */
-    public static <K, T> MapDiff<K, T, ImmutableOpenMap<K, T>> diff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after, KeySerializer<K> keySerializer, NonDiffableValueSerializer<K, T> valueSerializer) {
+    public static <K, T> MapDiff<K, T, ImmutableOpenMap<K, T>> diff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
         assert after != null && before != null;
         return new ImmutableOpenMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -90,7 +92,7 @@ public final class DiffableUtils {
     /**
      * Calculates diff between two ImmutableOpenIntMaps of non-diffable objects
      */
-    public static <T> MapDiff<Integer, T, ImmutableOpenIntMap<T>> diff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after, KeySerializer<Integer> keySerializer, NonDiffableValueSerializer<Integer, T> valueSerializer) {
+    public static <T> MapDiff<Integer, T, ImmutableOpenIntMap<T>> diff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after, KeySerializer<Integer> keySerializer, ValueSerializer<Integer, T> valueSerializer) {
         assert after != null && before != null;
         return new ImmutableOpenIntMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -106,7 +108,7 @@ public final class DiffableUtils {
     /**
      * Calculates diff between two Maps of non-diffable objects
      */
-    public static <K, T> MapDiff<K, T, Map<K, T>> diff(Map<K, T> before, Map<K, T> after, KeySerializer<K> keySerializer, NonDiffableValueSerializer<K, T> valueSerializer) {
+    public static <K, T> MapDiff<K, T, Map<K, T>> diff(Map<K, T> before, Map<K, T> after, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
         assert after != null && before != null;
         return new JdkMapDiff<>(before, after, keySerializer, valueSerializer);
     }
@@ -135,22 +137,22 @@ public final class DiffableUtils {
     /**
      * Loads an object that represents difference between two ImmutableOpenMaps of Diffable objects using Diffable proto object
      */
-    public static <K, T extends Diffable<T>> MapDiff<K, T, ImmutableOpenMap<K, T>> readImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, T proto) throws IOException {
-        return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <K, T extends Diffable<T>> MapDiff<K, T, ImmutableOpenMap<K, T>> readImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new ImmutableOpenMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
      * Loads an object that represents difference between two ImmutableOpenIntMaps of Diffable objects using Diffable proto object
      */
-    public static <T extends Diffable<T>> MapDiff<Integer, T, ImmutableOpenIntMap<T>> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer<Integer> keySerializer, T proto) throws IOException {
-        return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <T extends Diffable<T>> MapDiff<Integer, T, ImmutableOpenIntMap<T>> readImmutableOpenIntMapDiff(StreamInput in, KeySerializer<Integer> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new ImmutableOpenIntMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
      * Loads an object that represents difference between two Maps of Diffable objects using Diffable proto object
      */
-    public static <K, T extends Diffable<T>> MapDiff<K, T, Map<K, T>> readJdkMapDiff(StreamInput in, KeySerializer<K> keySerializer, T proto) throws IOException {
-        return new JdkMapDiff<>(in, keySerializer, new DiffablePrototypeValueReader<>(proto));
+    public static <K, T extends Diffable<T>> MapDiff<K, T, Map<K, T>> readJdkMapDiff(StreamInput in, KeySerializer<K> keySerializer, Reader<T> reader, Reader<Diff<T>> diffReader) throws IOException {
+        return new JdkMapDiff<>(in, keySerializer, new DiffableValueReader<>(reader, diffReader));
     }
 
     /**
@@ -164,7 +166,7 @@ public final class DiffableUtils {
             super(in, keySerializer, valueSerializer);
         }
 
-        public JdkMapDiff(Map<K, T> before, Map<K, T> after,
+        JdkMapDiff(Map<K, T> before, Map<K, T> after,
                           KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
             super(keySerializer, valueSerializer);
             assert after != null && before != null;
@@ -214,12 +216,17 @@ public final class DiffableUtils {
      *
      * @param <T> the object type
      */
-    private static class ImmutableOpenMapDiff<K, T> extends MapDiff<K, T, ImmutableOpenMap<K, T>> {
+    public static class ImmutableOpenMapDiff<K, T> extends MapDiff<K, T, ImmutableOpenMap<K, T>> {
 
         protected ImmutableOpenMapDiff(StreamInput in, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) throws IOException {
             super(in, keySerializer, valueSerializer);
         }
 
+        private ImmutableOpenMapDiff(KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer,
+                                     List<K> deletes, Map<K, Diff<T>> diffs, Map<K, T> upserts) {
+            super(keySerializer, valueSerializer, deletes, diffs, upserts);
+        }
+
         public ImmutableOpenMapDiff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after,
                                     KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
             super(keySerializer, valueSerializer);
@@ -245,6 +252,21 @@ public final class DiffableUtils {
             }
         }
 
+        /**
+         * Returns a new diff map with the given key removed, does not modify the invoking instance.
+         * If the key does not exist in the diff map, the same instance is returned.
+         */
+        public ImmutableOpenMapDiff<K, T> withKeyRemoved(K key) {
+            if (this.diffs.containsKey(key) == false && this.upserts.containsKey(key) == false) {
+                return this;
+            }
+            Map<K, Diff<T>> newDiffs = new HashMap<>(this.diffs);
+            newDiffs.remove(key);
+            Map<K, T> newUpserts = new HashMap<>(this.upserts);
+            newUpserts.remove(key);
+            return new ImmutableOpenMapDiff<>(this.keySerializer, this.valueSerializer, this.deletes, newDiffs, newUpserts);
+        }
+
         @Override
         public ImmutableOpenMap<K, T> apply(ImmutableOpenMap<K, T> map) {
             ImmutableOpenMap.Builder<K, T> builder = ImmutableOpenMap.builder();
@@ -276,7 +298,7 @@ public final class DiffableUtils {
             super(in, keySerializer, valueSerializer);
         }
 
-        public ImmutableOpenIntMapDiff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after,
+        ImmutableOpenIntMapDiff(ImmutableOpenIntMap<T> before, ImmutableOpenIntMap<T> after,
                                        KeySerializer<Integer> keySerializer, ValueSerializer<Integer, T> valueSerializer) {
             super(keySerializer, valueSerializer);
             assert after != null && before != null;
@@ -346,6 +368,15 @@ public final class DiffableUtils {
             upserts = new HashMap<>();
         }
 
+        protected MapDiff(KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer,
+                          List<K> deletes, Map<K, Diff<T>> diffs, Map<K, T> upserts) {
+            this.keySerializer = keySerializer;
+            this.valueSerializer = valueSerializer;
+            this.deletes = deletes;
+            this.diffs = diffs;
+            this.upserts = upserts;
+        }
+
         protected MapDiff(StreamInput in, KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) throws IOException {
             this.keySerializer = keySerializer;
             this.valueSerializer = valueSerializer;
@@ -406,12 +437,29 @@ public final class DiffableUtils {
             for (K delete : deletes) {
                 keySerializer.writeKey(delete, out);
             }
-            out.writeVInt(diffs.size());
-            for (Map.Entry<K, Diff<T>> entry : diffs.entrySet()) {
-                keySerializer.writeKey(entry.getKey(), out);
-                valueSerializer.writeDiff(entry.getValue(), out);
+            Version version = out.getVersion();
+            // filter out custom states not supported by the other node
+            int diffCount = 0;
+            for (Diff<T> diff : diffs.values()) {
+                if(valueSerializer.supportsVersion(diff, version)) {
+                    diffCount++;
+                }
             }
-            out.writeVInt(upserts.size());
+            out.writeVInt(diffCount);
+            for (Map.Entry<K, Diff<T>> entry : diffs.entrySet()) {
+                if(valueSerializer.supportsVersion(entry.getValue(), version)) {
+                    keySerializer.writeKey(entry.getKey(), out);
+                    valueSerializer.writeDiff(entry.getValue(), out);
+                }
+            }
+            // filter out custom states not supported by the other node
+            int upsertsCount = 0;
+            for (T upsert : upserts.values()) {
+                if(valueSerializer.supportsVersion(upsert, version)) {
+                    upsertsCount++;
+                }
+            }
+            out.writeVInt(upsertsCount);
             for (Map.Entry<K, T> entry : upserts.entrySet()) {
                 keySerializer.writeKey(entry.getKey(), out);
                 valueSerializer.write(entry.getValue(), out);
@@ -511,6 +559,20 @@ public final class DiffableUtils {
          */
         boolean supportsDiffableValues();
 
+        /**
+         * Whether this serializer supports the version of the output stream
+         */
+        default boolean supportsVersion(Diff<V> value, Version version) {
+            return true;
+        }
+
+        /**
+         * Whether this serializer supports the version of the output stream
+         */
+        default boolean supportsVersion(V value, Version version) {
+            return true;
+        }
+
         /**
          * Computes diff if this serializer supports diffable values
          */
@@ -600,25 +662,27 @@ public final class DiffableUtils {
     }
 
     /**
-     * Implementation of the ValueSerializer that uses a prototype object for reading operations
+     * Implementation of the ValueSerializer that wraps value and diff readers.
      *
      * Note: this implementation is ignoring the key.
      */
-    public static class DiffablePrototypeValueReader<K, V extends Diffable<V>> extends DiffableValueSerializer<K, V> {
-        private final V proto;
+    public static class DiffableValueReader<K, V extends Diffable<V>> extends DiffableValueSerializer<K, V> {
+        private final Reader<V> reader;
+        private final Reader<Diff<V>> diffReader;
 
-        public DiffablePrototypeValueReader(V proto) {
-            this.proto = proto;
+        public DiffableValueReader(Reader<V> reader, Reader<Diff<V>> diffReader) {
+            this.reader = reader;
+            this.diffReader = diffReader;
         }
 
         @Override
         public V read(StreamInput in, K key) throws IOException {
-            return proto.readFrom(in);
+            return reader.read(in);
         }
 
         @Override
         public Diff<V> readDiff(StreamInput in, K key) throws IOException {
-            return proto.readDiffFrom(in);
+            return diffReader.read(in);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index b8ac2a5eb50..a9392d3c017 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -377,14 +377,13 @@ public class InternalClusterInfoService extends AbstractComponent
         MetaData meta = state.getMetaData();
         for (ShardStats s : stats) {
             IndexMetaData indexMeta = meta.index(s.getShardRouting().index());
-            Settings indexSettings = indexMeta == null ? null : indexMeta.getSettings();
             newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
             long size = s.getStats().getStore().sizeInBytes();
             String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
             if (logger.isTraceEnabled()) {
                 logger.trace("shard: {} size: {}", sid, size);
             }
-            if (indexSettings != null && IndexMetaData.isIndexUsingShadowReplicas(indexSettings)) {
+            if (indexMeta != null && indexMeta.isIndexUsingShadowReplicas()) {
                 // Shards on a shared filesystem should be considered of size 0
                 if (logger.isTraceEnabled()) {
                     logger.trace("shard: {} is using shadow replicas and will be treated as size 0", sid);
diff --git a/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java
new file mode 100644
index 00000000000..9692ff8d4e1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/LocalClusterUpdateTask.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.List;
+
+/**
+ * Used to apply state updates on nodes that are not necessarily master
+ */
+public abstract class LocalClusterUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<LocalClusterUpdateTask>,
+    ClusterStateTaskListener {
+
+    private final Priority priority;
+
+    public LocalClusterUpdateTask() {
+        this(Priority.NORMAL);
+    }
+
+    public LocalClusterUpdateTask(Priority priority) {
+        this.priority = priority;
+    }
+
+    public abstract ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception;
+
+    @Override
+    public final ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState,
+                                                                    List<LocalClusterUpdateTask> tasks) throws Exception {
+        assert tasks.size() == 1 && tasks.get(0) == this : "expected one-element task list containing current object but was " + tasks;
+        ClusterTasksResult<LocalClusterUpdateTask> result = execute(currentState);
+        return ClusterTasksResult.<LocalClusterUpdateTask>builder().successes(tasks).build(result, currentState);
+    }
+
+    /**
+     * node stepped down as master or has lost connection to the master
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> noMaster() {
+        return new ClusterTasksResult(true, null, null);
+    }
+
+    /**
+     * no changes were made to the cluster state. Useful to execute a runnable on the cluster state applier thread
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> unchanged() {
+        return new ClusterTasksResult(false, null, null);
+    }
+
+    /**
+     * locally apply cluster state received from a master
+     */
+    public static ClusterTasksResult<LocalClusterUpdateTask> newState(ClusterState clusterState) {
+        return new ClusterTasksResult(false, clusterState, null);
+    }
+
+    @Override
+    public String describeTasks(List<LocalClusterUpdateTask> tasks) {
+        return ""; // one of task, source is enough
+    }
+
+    @Nullable
+    public TimeValue timeout() {
+        return null;
+    }
+
+    @Override
+    public Priority priority() {
+        return priority;
+    }
+
+    @Override
+    public final boolean runOnlyOnMaster() {
+        return false;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiff.java
similarity index 63%
rename from core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java
rename to core/src/main/java/org/elasticsearch/cluster/NamedDiff.java
index 672890c2b97..9da3167ae88 100644
--- a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcherSupplier.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiff.java
@@ -17,20 +17,20 @@
  * under the License.
  */
 
-package org.elasticsearch.common;
+package org.elasticsearch.cluster;
 
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 
 /**
- * This interface should be implemented by classes like {@link QueryParseContext} or {@link QueryShardContext} that
- * are able to carry a {@link ParseFieldMatcher}.
+ * Diff that also support NamedWriteable interface
  */
-@FunctionalInterface
-public interface ParseFieldMatcherSupplier {
-
+public interface NamedDiff<T extends Diffable<T>> extends Diff<T>, NamedWriteable {
     /**
-     * @return the parseFieldMatcher
+     * The minimal version of the recipient this custom object can be sent to
      */
-    ParseFieldMatcher getParseFieldMatcher();
+    default Version getMinimalSupportedVersion() {
+        return Version.CURRENT.minimumCompatibilityVersion();
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java
similarity index 64%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
rename to core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java
index 390ac1ce366..07974422096 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiffable.java
@@ -16,18 +16,20 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
 
-import java.io.IOException;
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 
 /**
- * Base parser for a REST test suite fragment
- * @param <T> the test fragment's type that gets parsed and returned
+ * Diff that also support NamedWriteable interface
  */
-public interface ClientYamlTestFragmentParser<T> {
-
+public interface NamedDiffable<T> extends Diffable<T>, NamedWriteable {
     /**
-     * Parses a test fragment given the current {@link ClientYamlTestSuiteParseContext}
+     * The minimal version of the recipient this custom object can be sent to
      */
-    T parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException;
+    default Version getMinimalSupportedVersion() {
+        return Version.CURRENT.minimumCompatibilityVersion();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java b/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
new file mode 100644
index 00000000000..c6434db9e87
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/NamedDiffableValueSerializer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+/**
+ * Value Serializer for named diffables
+ */
+public class NamedDiffableValueSerializer<T extends NamedDiffable<T>> extends DiffableUtils.DiffableValueSerializer<String, T> {
+
+    private final Class<T> tClass;
+
+    public NamedDiffableValueSerializer(Class<T> tClass) {
+        this.tClass = tClass;
+    }
+
+    @Override
+    public T read(StreamInput in, String key) throws IOException {
+        return in.readNamedWriteable(tClass, key);
+    }
+
+    @Override
+    public boolean supportsVersion(Diff<T> value, Version version) {
+        return version.onOrAfter(((NamedDiff<?>)value).getMinimalSupportedVersion());
+    }
+
+    @Override
+    public boolean supportsVersion(T value, Version version) {
+        return version.onOrAfter(value.getMinimalSupportedVersion());
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Diff<T> readDiff(StreamInput in, String key) throws IOException {
+        return in.readNamedWriteable(NamedDiff.class, key);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 94333c10dde..bda1481130c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -36,7 +36,9 @@ import org.elasticsearch.discovery.zen.NodesFaultDetection;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledFuture;
 
@@ -76,20 +78,26 @@ public class NodeConnectionsService extends AbstractLifecycleComponent {
         this.reconnectInterval = NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(settings);
     }
 
-    public void connectToNodes(List<DiscoveryNode> addedNodes) {
+    public void connectToNodes(Iterable<DiscoveryNode> discoveryNodes) {
 
         // TODO: do this in parallel (and wait)
-        for (final DiscoveryNode node : addedNodes) {
+        for (final DiscoveryNode node : discoveryNodes) {
             try (Releasable ignored = nodeLocks.acquire(node)) {
-                Integer current = nodes.put(node, 0);
-                assert current == null : "node " + node + " was added in event but already in internal nodes";
+                nodes.putIfAbsent(node, 0);
                 validateNodeConnected(node);
             }
         }
     }
 
-    public void disconnectFromNodes(List<DiscoveryNode> removedNodes) {
-        for (final DiscoveryNode node : removedNodes) {
+    /**
+     * Disconnects from all nodes except the ones provided as parameter
+     */
+    public void disconnectFromNodesExcept(Iterable<DiscoveryNode> nodesToKeep) {
+        Set<DiscoveryNode> currentNodes = new HashSet<>(nodes.keySet());
+        for (DiscoveryNode node : nodesToKeep) {
+            currentNodes.remove(node);
+        }
+        for (final DiscoveryNode node : currentNodes) {
             try (Releasable ignored = nodeLocks.acquire(node)) {
                 Integer current = nodes.remove(node);
                 assert current != null : "node " + node + " was removed in event but not in internal nodes";
diff --git a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
index 55a09f87f75..55e70dbe644 100644
--- a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
@@ -39,12 +39,10 @@ import java.util.Objects;
 /**
  * Meta data about restore processes that are currently executing
  */
-public class RestoreInProgress extends AbstractDiffable<Custom> implements Custom {
+public class RestoreInProgress extends AbstractNamedDiffable<Custom> implements Custom {
 
     public static final String TYPE = "restore";
 
-    public static final RestoreInProgress PROTO = new RestoreInProgress();
-
     private final List<Entry> entries;
 
     /**
@@ -377,15 +375,15 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
      * {@inheritDoc}
      */
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public RestoreInProgress readFrom(StreamInput in) throws IOException {
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    public RestoreInProgress(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
             Snapshot snapshot = new Snapshot(in);
@@ -404,7 +402,7 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
             }
             entries[i] = new Entry(snapshot, state, Collections.unmodifiableList(indexBuilder), builder.build());
         }
-        return new RestoreInProgress(entries);
+        this.entries = Arrays.asList(entries);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
new file mode 100644
index 00000000000..b3ab12fe21a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState.Custom;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.snapshots.Snapshot;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A class that represents the snapshot deletions that are in progress in the cluster.
+ */
+public class SnapshotDeletionsInProgress extends AbstractNamedDiffable<Custom> implements Custom {
+
+    public static final String TYPE = "snapshot_deletions";
+    // the version where SnapshotDeletionsInProgress was introduced
+    public static final Version VERSION_INTRODUCED = Version.V_5_2_0_UNRELEASED;
+
+    // the list of snapshot deletion request entries
+    private final List<Entry> entries;
+
+    private SnapshotDeletionsInProgress(List<Entry> entries) {
+        this.entries = Collections.unmodifiableList(entries);
+    }
+
+    public SnapshotDeletionsInProgress(StreamInput in) throws IOException {
+        this.entries = Collections.unmodifiableList(in.readList(Entry::new));
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} with the given
+     * {@link Entry} added.
+     */
+    public static SnapshotDeletionsInProgress newInstance(Entry entry) {
+        return new SnapshotDeletionsInProgress(Collections.singletonList(entry));
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} which adds
+     * the given {@link Entry} to the invoking instance.
+     */
+    public SnapshotDeletionsInProgress withAddedEntry(Entry entry) {
+        List<Entry> entries = new ArrayList<>(getEntries());
+        entries.add(entry);
+        return new SnapshotDeletionsInProgress(entries);
+    }
+
+    /**
+     * Returns a new instance of {@link SnapshotDeletionsInProgress} which removes
+     * the given entry from the invoking instance.
+     */
+    public SnapshotDeletionsInProgress withRemovedEntry(Entry entry) {
+        List<Entry> entries = new ArrayList<>(getEntries());
+        entries.remove(entry);
+        return new SnapshotDeletionsInProgress(entries);
+    }
+
+    /**
+     * Returns an unmodifiable list of snapshot deletion entries.
+     */
+    public List<Entry> getEntries() {
+        return entries;
+    }
+
+    /**
+     * Returns {@code true} if there are snapshot deletions in progress in the cluster,
+     * returns {@code false} otherwise.
+     */
+    public boolean hasDeletionsInProgress() {
+        return entries.isEmpty() == false;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        SnapshotDeletionsInProgress that = (SnapshotDeletionsInProgress) o;
+        return entries.equals(that.entries);
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 + entries.hashCode();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(entries);
+    }
+
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    @Override
+    public Version getMinimalSupportedVersion() {
+        return VERSION_INTRODUCED;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(TYPE);
+        for (Entry entry : entries) {
+            builder.startObject();
+            {
+                builder.field("repository", entry.snapshot.getRepository());
+                builder.field("snapshot", entry.snapshot.getSnapshotId().getName());
+                builder.timeValueField("start_time_millis", "start_time", entry.startTime);
+                builder.field("repository_state_id", entry.repositoryStateId);
+            }
+            builder.endObject();
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    /**
+     * A class representing a snapshot deletion request entry in the cluster state.
+     */
+    public static final class Entry implements Writeable {
+        private final Snapshot snapshot;
+        private final long startTime;
+        private final long repositoryStateId;
+
+        public Entry(Snapshot snapshot, long startTime, long repositoryStateId) {
+            this.snapshot = snapshot;
+            this.startTime = startTime;
+            this.repositoryStateId = repositoryStateId;
+        }
+
+        public Entry(StreamInput in) throws IOException {
+            this.snapshot = new Snapshot(in);
+            this.startTime = in.readVLong();
+            this.repositoryStateId = in.readLong();
+        }
+
+        /**
+         * The snapshot to delete.
+         */
+        public Snapshot getSnapshot() {
+            return snapshot;
+        }
+
+        /**
+         * The start time in milliseconds for deleting the snapshots.
+         */
+        public long getStartTime() {
+            return startTime;
+        }
+
+        /**
+         * The repository state id at the time the snapshot deletion began.
+         */
+        public long getRepositoryStateId() {
+            return repositoryStateId;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Entry that = (Entry) o;
+            return snapshot.equals(that.snapshot)
+                       && startTime == that.startTime
+                       && repositoryStateId == that.repositoryStateId;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(snapshot, startTime, repositoryStateId);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            snapshot.writeTo(out);
+            out.writeVLong(startTime);
+            out.writeLong(repositoryStateId);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index 6df5f85987d..0ac1e8e4090 100644
--- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster;
 import com.carrotsearch.hppc.ObjectContainer;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState.Custom;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -43,10 +44,14 @@ import java.util.Map;
 /**
  * Meta data about snapshots that are currently executing
  */
-public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Custom {
+public class SnapshotsInProgress extends AbstractNamedDiffable<Custom> implements Custom {
     public static final String TYPE = "snapshots";
 
-    public static final SnapshotsInProgress PROTO = new SnapshotsInProgress();
+    // denotes an undefined repository state id, which will happen when receiving a cluster state with
+    // a snapshot in progress from a pre 5.2.x node
+    public static final long UNDEFINED_REPOSITORY_STATE_ID = -2L;
+    // the version where repository state ids were introduced
+    private static final Version REPOSITORY_ID_INTRODUCED_VERSION = Version.V_5_2_0_UNRELEASED;
 
     @Override
     public boolean equals(Object o) {
@@ -74,9 +79,10 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         private final List<IndexId> indices;
         private final ImmutableOpenMap<String, List<ShardId>> waitingIndices;
         private final long startTime;
+        private final long repositoryStateId;
 
         public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<IndexId> indices,
-                     long startTime, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
+                     long startTime, long repositoryStateId, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this.state = state;
             this.snapshot = snapshot;
             this.includeGlobalState = includeGlobalState;
@@ -90,10 +96,12 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
                 this.shards = shards;
                 this.waitingIndices = findWaitingIndices(shards);
             }
+            this.repositoryStateId = repositoryStateId;
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
-            this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards);
+            this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime,
+                 entry.repositoryStateId, shards);
         }
 
         public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
@@ -132,6 +140,10 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             return startTime;
         }
 
+        public long getRepositoryStateId() {
+            return repositoryStateId;
+        }
+
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
@@ -147,6 +159,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             if (!snapshot.equals(entry.snapshot)) return false;
             if (state != entry.state) return false;
             if (!waitingIndices.equals(entry.waitingIndices)) return false;
+            if (repositoryStateId != entry.repositoryStateId) return false;
 
             return true;
         }
@@ -161,6 +174,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             result = 31 * result + indices.hashCode();
             result = 31 * result + waitingIndices.hashCode();
             result = 31 * result + Long.hashCode(startTime);
+            result = 31 * result + Long.hashCode(repositoryStateId);
             return result;
         }
 
@@ -361,12 +375,15 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
     }
 
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    @Override
-    public SnapshotsInProgress readFrom(StreamInput in) throws IOException {
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
+    }
+
+    public SnapshotsInProgress(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
             Snapshot snapshot = new Snapshot(in);
@@ -387,15 +404,20 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
                 State shardState = State.fromValue(in.readByte());
                 builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState));
             }
+            long repositoryStateId = UNDEFINED_REPOSITORY_STATE_ID;
+            if (in.getVersion().onOrAfter(REPOSITORY_ID_INTRODUCED_VERSION)) {
+                repositoryStateId = in.readLong();
+            }
             entries[i] = new Entry(snapshot,
                                    includeGlobalState,
                                    partial,
                                    state,
                                    Collections.unmodifiableList(indexBuilder),
                                    startTime,
+                                   repositoryStateId,
                                    builder.build());
         }
-        return new SnapshotsInProgress(entries);
+        this.entries = Arrays.asList(entries);
     }
 
     @Override
@@ -417,6 +439,9 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
                 out.writeOptionalString(shardEntry.value.nodeId());
                 out.writeByte(shardEntry.value.state().value());
             }
+            if (out.getVersion().onOrAfter(REPOSITORY_ID_INTRODUCED_VERSION)) {
+                out.writeLong(entry.repositoryStateId);
+            }
         }
     }
 
@@ -430,6 +455,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
     private static final String INDICES = "indices";
     private static final String START_TIME_MILLIS = "start_time_millis";
     private static final String START_TIME = "start_time";
+    private static final String REPOSITORY_STATE_ID = "repository_state_id";
     private static final String SHARDS = "shards";
     private static final String INDEX = "index";
     private static final String SHARD = "shard";
@@ -461,6 +487,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         }
         builder.endArray();
         builder.timeValueField(START_TIME_MILLIS, START_TIME, entry.startTime());
+        builder.field(REPOSITORY_STATE_ID, entry.getRepositoryStateId());
         builder.startArray(SHARDS);
         {
             for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : entry.shards) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 0cf124612d0..8973890021f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -25,10 +25,10 @@ import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.MasterNodeChangePredicate;
 import org.elasticsearch.cluster.NotMasterException;
@@ -69,6 +69,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.function.Predicate;
 
 public class ShardStateAction extends AbstractComponent {
 
@@ -91,11 +92,13 @@ public class ShardStateAction extends AbstractComponent {
         transportService.registerRequestHandler(SHARD_FAILED_ACTION_NAME, ShardEntry::new, ThreadPool.Names.SAME, new ShardFailedTransportHandler(clusterService, new ShardFailedClusterStateTaskExecutor(allocationService, routingService, logger), logger));
     }
 
-    private void sendShardAction(final String actionName, final ClusterStateObserver observer, final ShardEntry shardEntry, final Listener listener) {
-        DiscoveryNode masterNode = observer.observedState().nodes().getMasterNode();
+    private void sendShardAction(final String actionName, final ClusterState currentState, final ShardEntry shardEntry, final Listener listener) {
+        ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext());
+        DiscoveryNode masterNode = currentState.nodes().getMasterNode();
+        Predicate<ClusterState> changePredicate = MasterNodeChangePredicate.build(currentState);
         if (masterNode == null) {
             logger.warn("{} no master known for action [{}] for shard entry [{}]", shardEntry.shardId, actionName, shardEntry);
-            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
         } else {
             logger.debug("{} sending [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode.getId(), shardEntry);
             transportService.sendRequest(masterNode,
@@ -108,7 +111,7 @@ public class ShardStateAction extends AbstractComponent {
                     @Override
                     public void handleException(TransportException exp) {
                         if (isMasterChannelException(exp)) {
-                            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+                            waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
                         } else {
                             logger.warn((Supplier<?>) () -> new ParameterizedMessage("{} unexpected failure while sending request [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode, shardEntry), exp);
                             listener.onFailure(exp instanceof RemoteTransportException ? (Exception) (exp.getCause() instanceof Exception ? exp.getCause() : new ElasticsearchException(exp.getCause())) : exp);
@@ -162,20 +165,19 @@ public class ShardStateAction extends AbstractComponent {
 
     private void shardFailed(final ShardId shardId, String allocationId, long primaryTerm, final String message,
                              @Nullable final Exception failure, Listener listener, ClusterState currentState) {
-        ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext());
         ShardEntry shardEntry = new ShardEntry(shardId, allocationId, primaryTerm, message, failure);
-        sendShardAction(SHARD_FAILED_ACTION_NAME, observer, shardEntry, listener);
+        sendShardAction(SHARD_FAILED_ACTION_NAME, currentState, shardEntry, listener);
     }
 
     // visible for testing
-    protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener) {
+    protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener, Predicate<ClusterState> changePredicate) {
         observer.waitForNextChange(new ClusterStateObserver.Listener() {
             @Override
             public void onNewClusterState(ClusterState state) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("new cluster state [{}] after waiting for master election to fail shard entry [{}]", state, shardEntry);
                 }
-                sendShardAction(actionName, observer, shardEntry, listener);
+                sendShardAction(actionName, state, shardEntry, listener);
             }
 
             @Override
@@ -189,7 +191,7 @@ public class ShardStateAction extends AbstractComponent {
                 // we wait indefinitely for a new master
                 assert false;
             }
-        }, MasterNodeChangePredicate.build(observer.observedState()));
+        }, changePredicate);
     }
 
     private static class ShardFailedTransportHandler implements TransportRequestHandler<ShardEntry> {
@@ -197,7 +199,7 @@ public class ShardStateAction extends AbstractComponent {
         private final ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor;
         private final Logger logger;
 
-        public ShardFailedTransportHandler(ClusterService clusterService, ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor, Logger logger) {
+        ShardFailedTransportHandler(ClusterService clusterService, ShardFailedClusterStateTaskExecutor shardFailedClusterStateTaskExecutor, Logger logger) {
             this.clusterService = clusterService;
             this.shardFailedClusterStateTaskExecutor = shardFailedClusterStateTaskExecutor;
             this.logger = logger;
@@ -258,8 +260,8 @@ public class ShardStateAction extends AbstractComponent {
         }
 
         @Override
-        public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
-            BatchResult.Builder<ShardEntry> batchResultBuilder = BatchResult.builder();
+        public ClusterTasksResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
+            ClusterTasksResult.Builder<ShardEntry> batchResultBuilder = ClusterTasksResult.builder();
             List<ShardEntry> tasksToBeApplied = new ArrayList<>();
             List<FailedShard> failedShardsToBeApplied = new ArrayList<>();
             List<StaleShard> staleShardsToBeApplied = new ArrayList<>();
@@ -354,9 +356,8 @@ public class ShardStateAction extends AbstractComponent {
         shardStarted(shardRouting, message, listener, clusterService.state());
     }
     public void shardStarted(final ShardRouting shardRouting, final String message, Listener listener, ClusterState currentState) {
-        ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext());
         ShardEntry shardEntry = new ShardEntry(shardRouting.shardId(), shardRouting.allocationId().getId(), 0L, message, null);
-        sendShardAction(SHARD_STARTED_ACTION_NAME, observer, shardEntry, listener);
+        sendShardAction(SHARD_STARTED_ACTION_NAME, currentState, shardEntry, listener);
     }
 
     private static class ShardStartedTransportHandler implements TransportRequestHandler<ShardEntry> {
@@ -364,7 +365,7 @@ public class ShardStateAction extends AbstractComponent {
         private final ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor;
         private final Logger logger;
 
-        public ShardStartedTransportHandler(ClusterService clusterService, ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor, Logger logger) {
+        ShardStartedTransportHandler(ClusterService clusterService, ShardStartedClusterStateTaskExecutor shardStartedClusterStateTaskExecutor, Logger logger) {
             this.clusterService = clusterService;
             this.shardStartedClusterStateTaskExecutor = shardStartedClusterStateTaskExecutor;
             this.logger = logger;
@@ -393,8 +394,8 @@ public class ShardStateAction extends AbstractComponent {
         }
 
         @Override
-        public BatchResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
-            BatchResult.Builder<ShardEntry> builder = BatchResult.builder();
+        public ClusterTasksResult<ShardEntry> execute(ClusterState currentState, List<ShardEntry> tasks) throws Exception {
+            ClusterTasksResult.Builder<ShardEntry> builder = ClusterTasksResult.builder();
             List<ShardEntry> tasksToBeApplied = new ArrayList<>();
             List<ShardRouting> shardRoutingsToBeApplied = new ArrayList<>(tasks.size());
             Set<ShardRouting> seenShardRoutings = new HashSet<>(); // to prevent duplicates
diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
index 12e6ee0f7ec..2bdf560580b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
+++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.block;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -48,8 +49,6 @@ import static java.util.stream.Stream.concat;
 public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
     public static final ClusterBlocks EMPTY_CLUSTER_BLOCK = new ClusterBlocks(emptySet(), ImmutableOpenMap.of());
 
-    public static final ClusterBlocks PROTO = EMPTY_CLUSTER_BLOCK;
-
     private final Set<ClusterBlock> global;
 
     private final ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks;
@@ -59,23 +58,7 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
     ClusterBlocks(Set<ClusterBlock> global, ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
         this.global = global;
         this.indicesBlocks = indicesBlocks;
-
-        levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
-        for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
-            Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
-            Set<ClusterBlock> newGlobal = unmodifiableSet(global.stream()
-                    .filter(containsLevel)
-                    .collect(toSet()));
-
-            ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder();
-            for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
-                indicesBuilder.put(entry.key, unmodifiableSet(entry.value.stream()
-                        .filter(containsLevel)
-                        .collect(toSet())));
-            }
-
-            levelHolders[level.id()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
-        }
+        levelHolders = generateLevelHolders(global, indicesBlocks);
     }
 
     public Set<ClusterBlock> global() {
@@ -98,6 +81,27 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
         return indices(level).getOrDefault(index, emptySet());
     }
 
+    private static ImmutableLevelHolder[] generateLevelHolders(Set<ClusterBlock> global,
+                                                               ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
+        ImmutableLevelHolder[] levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
+        for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
+            Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
+            Set<ClusterBlock> newGlobal = unmodifiableSet(global.stream()
+                .filter(containsLevel)
+                .collect(toSet()));
+
+            ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder();
+            for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
+                indicesBuilder.put(entry.key, unmodifiableSet(entry.value.stream()
+                    .filter(containsLevel)
+                    .collect(toSet())));
+            }
+
+            levelHolders[level.id()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
+        }
+        return levelHolders;
+    }
+
     /**
      * Returns <tt>true</tt> if one of the global blocks as its disable state persistence flag set.
      */
@@ -239,15 +243,16 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
         }
     }
 
-    @Override
-    public ClusterBlocks readFrom(StreamInput in) throws IOException {
+    public ClusterBlocks(StreamInput in) throws IOException {
         Set<ClusterBlock> global = readBlockSet(in);
         int size = in.readVInt();
         ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder(size);
         for (int j = 0; j < size; j++) {
             indicesBuilder.put(in.readString().intern(), readBlockSet(in));
         }
-        return new ClusterBlocks(global, indicesBuilder.build());
+        this.global = global;
+        this.indicesBlocks = indicesBuilder.build();
+        levelHolders = generateLevelHolders(global, indicesBlocks);
     }
 
     private static Set<ClusterBlock> readBlockSet(StreamInput in) throws IOException {
@@ -259,6 +264,10 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
         return unmodifiableSet(blocks);
     }
 
+    public static Diff<ClusterBlocks> readDiffFrom(StreamInput in) throws IOException {
+        return AbstractDiffable.readDiffFrom(ClusterBlocks::new, in);
+    }
+
     static class ImmutableLevelHolder {
 
         static final ImmutableLevelHolder EMPTY = new ImmutableLevelHolder(emptySet(), ImmutableOpenMap.of());
@@ -383,9 +392,5 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
             }
             return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
         }
-
-        public static ClusterBlocks readClusterBlocks(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
index e62a3935ad5..8071871fbfe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
@@ -21,13 +21,16 @@ package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -39,8 +42,6 @@ import static java.util.Collections.emptySet;
 
 public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
 
-    public static final AliasMetaData PROTO = new AliasMetaData("", null, null, null);
-
     private final String alias;
 
     private final CompressedXContent filter;
@@ -171,22 +172,29 @@ public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
 
     }
 
-    @Override
-    public AliasMetaData readFrom(StreamInput in) throws IOException {
-        String alias = in.readString();
-        CompressedXContent filter = null;
+    public AliasMetaData(StreamInput in) throws IOException {
+        alias = in.readString();
         if (in.readBoolean()) {
             filter = CompressedXContent.readCompressedString(in);
+        } else {
+            filter = null;
         }
-        String indexRouting = null;
         if (in.readBoolean()) {
             indexRouting = in.readString();
+        } else {
+            indexRouting = null;
         }
-        String searchRouting = null;
         if (in.readBoolean()) {
             searchRouting = in.readString();
+            searchRoutingValues = Collections.unmodifiableSet(Strings.splitStringByCommaToSet(searchRouting));
+        } else {
+            searchRouting = null;
+            searchRoutingValues = emptySet();
         }
-        return new AliasMetaData(alias, filter, indexRouting, searchRouting);
+    }
+
+    public static Diff<AliasMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(AliasMetaData::new, in);
     }
 
     public static class Builder {
@@ -225,14 +233,7 @@ public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
                 this.filter = null;
                 return this;
             }
-            try {
-                try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
-                    filter(parser.mapOrdered());
-                }
-                return this;
-            } catch (IOException e) {
-                throw new ElasticsearchGenerationException("Failed to generate [" + filter + "]", e);
-            }
+            return filter(XContentHelper.convertToMap(XContentFactory.xContent(filter), filter, true));
         }
 
         public Builder filter(Map<String, Object> filter) {
@@ -286,11 +287,7 @@ public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
                 if (binary) {
                     builder.field("filter", aliasMetaData.filter.compressed());
                 } else {
-                    byte[] data = aliasMetaData.filter().uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> filter = parser.mapOrdered();
-                        builder.field("filter", filter);
-                    }
+                    builder.field("filter", XContentHelper.convertToMap(new BytesArray(aliasMetaData.filter().uncompressed()), true).v2());
                 }
             }
             if (aliasMetaData.indexRouting() != null) {
@@ -336,14 +333,6 @@ public class AliasMetaData extends AbstractDiffable<AliasMetaData> {
             }
             return builder.build();
         }
-
-        public void writeTo(AliasMetaData aliasMetaData, StreamOutput out) throws IOException {
-            aliasMetaData.writeTo(out);
-        }
-
-        public static AliasMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
index 4ad9b7e5317..786bd9af78a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasOrIndex.java
@@ -121,7 +121,7 @@ public interface AliasOrIndex {
                         }
 
                         @Override
-                        public final void remove() {
+                        public void remove() {
                             throw new UnsupportedOperationException();
                         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index 29faa0f7956..bce6e45c793 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -25,7 +25,9 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -74,8 +76,8 @@ public class AliasValidator extends AbstractComponent {
     public void validateAliasStandalone(Alias alias) {
         validateAliasStandalone(alias.name(), alias.indexRouting());
         if (Strings.hasLength(alias.filter())) {
-            try (XContentParser parser = XContentFactory.xContent(alias.filter()).createParser(alias.filter())) {
-                parser.map();
+            try {
+                XContentHelper.convertToMap(XContentFactory.xContent(alias.filter()), alias.filter(), false);
             } catch (Exception e) {
                 throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
             }
@@ -113,9 +115,10 @@ public class AliasValidator extends AbstractComponent {
      * provided {@link org.elasticsearch.index.query.QueryShardContext}
      * @throws IllegalArgumentException if the filter is not valid
      */
-    public void validateAliasFilter(String alias, String filter, QueryShardContext queryShardContext) {
+    public void validateAliasFilter(String alias, String filter, QueryShardContext queryShardContext,
+            NamedXContentRegistry xContentRegistry) {
         assert queryShardContext != null;
-        try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
+        try (XContentParser parser = XContentFactory.xContent(filter).createParser(xContentRegistry, filter)) {
             validateAliasFilter(parser, queryShardContext);
         } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
@@ -127,9 +130,10 @@ public class AliasValidator extends AbstractComponent {
      * provided {@link org.elasticsearch.index.query.QueryShardContext}
      * @throws IllegalArgumentException if the filter is not valid
      */
-    public void validateAliasFilter(String alias, byte[] filter, QueryShardContext queryShardContext) {
+    public void validateAliasFilter(String alias, byte[] filter, QueryShardContext queryShardContext,
+            NamedXContentRegistry xContentRegistry) {
         assert queryShardContext != null;
-        try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
+        try (XContentParser parser = XContentFactory.xContent(filter).createParser(xContentRegistry, filter)) {
             validateAliasFilter(parser, queryShardContext);
         } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
index 4b4a8e54d7c..fa30abe5a73 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AutoExpandReplicas.java
@@ -33,7 +33,7 @@ final class AutoExpandReplicas {
     public static final Setting<AutoExpandReplicas> SETTING = new Setting<>(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "false", (value) -> {
         final int min;
         final int max;
-        if (Booleans.parseBoolean(value, true) == false) {
+        if (Booleans.isFalse(value)) {
             return new AutoExpandReplicas(0, 0, false);
         }
         final int dash = value.indexOf('-');
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 1a2cfe90ad6..d60617ea642 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -20,9 +20,8 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -67,10 +66,9 @@ public final class IndexGraveyard implements MetaData.Custom {
                                                                                      500, // the default maximum number of tombstones
                                                                                      Setting.Property.NodeScope);
 
-    public static final IndexGraveyard PROTO = new IndexGraveyard(new ArrayList<>());
     public static final String TYPE = "index-graveyard";
     private static final ParseField TOMBSTONES_FIELD = new ParseField("tombstones");
-    private static final ObjectParser<List<Tombstone>, ParseFieldMatcherSupplier> GRAVEYARD_PARSER;
+    private static final ObjectParser<List<Tombstone>, Void> GRAVEYARD_PARSER;
     static {
         GRAVEYARD_PARSER = new ObjectParser<>("index_graveyard", ArrayList::new);
         GRAVEYARD_PARSER.declareObjectArray(List::addAll, Tombstone.getParser(), TOMBSTONES_FIELD);
@@ -83,7 +81,7 @@ public final class IndexGraveyard implements MetaData.Custom {
         tombstones = Collections.unmodifiableList(list);
     }
 
-    private IndexGraveyard(final StreamInput in) throws IOException {
+    public IndexGraveyard(final StreamInput in) throws IOException {
         final int queueSize = in.readVInt();
         List<Tombstone> tombstones = new ArrayList<>(queueSize);
         for (int i = 0; i < queueSize; i++) {
@@ -92,12 +90,8 @@ public final class IndexGraveyard implements MetaData.Custom {
         this.tombstones = Collections.unmodifiableList(tombstones);
     }
 
-    public static IndexGraveyard fromStream(final StreamInput in) throws IOException {
-        return new IndexGraveyard(in);
-    }
-
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
@@ -144,8 +138,8 @@ public final class IndexGraveyard implements MetaData.Custom {
         return builder.endArray();
     }
 
-    public IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
-        return new IndexGraveyard(GRAVEYARD_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT));
+    public static IndexGraveyard fromXContent(final XContentParser parser) throws IOException {
+        return new IndexGraveyard(GRAVEYARD_PARSER.parse(parser, null));
     }
 
     @Override
@@ -161,19 +155,13 @@ public final class IndexGraveyard implements MetaData.Custom {
         }
     }
 
-    @Override
-    public IndexGraveyard readFrom(final StreamInput in) throws IOException {
-        return new IndexGraveyard(in);
-    }
-
     @Override
     @SuppressWarnings("unchecked")
     public Diff<MetaData.Custom> diff(final MetaData.Custom previous) {
         return new IndexGraveyardDiff((IndexGraveyard) previous, this);
     }
 
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(final StreamInput in) throws IOException {
+    public static NamedDiff<MetaData.Custom> readDiffFrom(final StreamInput in) throws IOException {
         return new IndexGraveyardDiff(in);
     }
 
@@ -273,7 +261,7 @@ public final class IndexGraveyard implements MetaData.Custom {
     /**
      * A class representing a diff of two IndexGraveyard objects.
      */
-    public static final class IndexGraveyardDiff implements Diff<MetaData.Custom> {
+    public static final class IndexGraveyardDiff implements NamedDiff<MetaData.Custom> {
 
         private final List<Tombstone> added;
         private final int removedCount;
@@ -349,6 +337,11 @@ public final class IndexGraveyard implements MetaData.Custom {
         public int getRemovedCount() {
             return removedCount;
         }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
     }
 
     /**
@@ -359,16 +352,17 @@ public final class IndexGraveyard implements MetaData.Custom {
         private static final String INDEX_KEY = "index";
         private static final String DELETE_DATE_IN_MILLIS_KEY = "delete_date_in_millis";
         private static final String DELETE_DATE_KEY = "delete_date";
-        private static final ObjectParser<Tombstone.Builder, ParseFieldMatcherSupplier> TOMBSTONE_PARSER;
+        private static final ObjectParser<Tombstone.Builder, Void> TOMBSTONE_PARSER;
         static {
             TOMBSTONE_PARSER = new ObjectParser<>("tombstoneEntry", Tombstone.Builder::new);
-            TOMBSTONE_PARSER.declareObject(Tombstone.Builder::index, Index::parseIndex, new ParseField(INDEX_KEY));
+            TOMBSTONE_PARSER.declareObject(Tombstone.Builder::index, (parser, context) -> Index.fromXContent(parser),
+                    new ParseField(INDEX_KEY));
             TOMBSTONE_PARSER.declareLong(Tombstone.Builder::deleteDateInMillis, new ParseField(DELETE_DATE_IN_MILLIS_KEY));
             TOMBSTONE_PARSER.declareString((b, s) -> {}, new ParseField(DELETE_DATE_KEY));
         }
 
-        static ContextParser<ParseFieldMatcherSupplier, Tombstone> getParser() {
-            return (p, c) -> TOMBSTONE_PARSER.apply(p, c).build();
+        static ContextParser<Void, Tombstone> getParser() {
+            return (parser, context) -> TOMBSTONE_PARSER.apply(parser, null).build();
         }
 
         private final Index index;
@@ -443,7 +437,7 @@ public final class IndexGraveyard implements MetaData.Custom {
         }
 
         public static Tombstone fromXContent(final XContentParser parser) throws IOException {
-            return TOMBSTONE_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build();
+            return TOMBSTONE_PARSER.parse(parser, null).build();
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index a1f217e1377..68575634a1e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.LongArrayList;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.cluster.Diff;
@@ -33,21 +34,23 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.cluster.routing.allocation.IndexMetaDataUpdater;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -70,13 +73,18 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
 
-public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuilder<IndexMetaData>, ToXContent {
+public class IndexMetaData implements Diffable<IndexMetaData>, ToXContent {
 
+    /**
+     * This class will be removed in v7.0
+     */
+    @Deprecated
     public interface Custom extends Diffable<Custom>, ToXContent {
 
         String type();
@@ -85,6 +93,16 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
         Custom fromXContent(XContentParser parser) throws IOException;
 
+        /**
+         * Reads the {@link org.elasticsearch.cluster.Diff} from StreamInput
+         */
+        Diff<Custom> readDiffFrom(StreamInput in) throws IOException;
+
+        /**
+         * Reads an object of this type from the provided {@linkplain StreamInput}. The receiving instance remains unchanged.
+         */
+        Custom readFrom(StreamInput in) throws IOException;
+
         /**
          * Merges from this to another, with this being more important, i.e., if something exists in this and another,
          * this will prevail.
@@ -121,7 +139,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public static final ClusterBlock INDEX_WRITE_BLOCK = new ClusterBlock(8, "index write (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.WRITE));
     public static final ClusterBlock INDEX_METADATA_BLOCK = new ClusterBlock(9, "index metadata (api)", false, false, RestStatus.FORBIDDEN, EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.METADATA_READ));
 
-    public static enum State {
+    public enum State {
         OPEN((byte) 0),
         CLOSE((byte) 1);
 
@@ -176,11 +194,15 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, Property.Dynamic, Property.IndexScope);
     public static final String SETTING_SHADOW_REPLICAS = "index.shadow_replicas";
     public static final Setting<Boolean> INDEX_SHADOW_REPLICAS_SETTING =
-        Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, Property.IndexScope, Property.Deprecated);
+
+    public static final String SETTING_ROUTING_PARTITION_SIZE = "index.routing_partition_size";
+    public static final Setting<Integer> INDEX_ROUTING_PARTITION_SIZE_SETTING =
+            Setting.intSetting(SETTING_ROUTING_PARTITION_SIZE, 1, 1, Property.IndexScope);
 
     public static final String SETTING_SHARED_FILESYSTEM = "index.shared_filesystem";
     public static final Setting<Boolean> INDEX_SHARED_FILESYSTEM_SETTING =
-        Setting.boolSetting(SETTING_SHARED_FILESYSTEM, false, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHARED_FILESYSTEM, INDEX_SHADOW_REPLICAS_SETTING, Property.IndexScope, Property.Deprecated);
 
     public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas";
     public static final Setting<AutoExpandReplicas> INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING;
@@ -221,18 +243,19 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         new Setting<>(SETTING_DATA_PATH, "", Function.identity(), Property.IndexScope);
     public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node";
     public static final Setting<Boolean> INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING =
-        Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, Property.Dynamic, Property.IndexScope);
+        Setting.boolSetting(SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false,
+            Property.Dynamic, Property.IndexScope, Property.Deprecated);
     public static final String INDEX_UUID_NA_VALUE = "_na_";
 
     public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require";
     public static final String INDEX_ROUTING_INCLUDE_GROUP_PREFIX = "index.routing.allocation.include";
     public static final String INDEX_ROUTING_EXCLUDE_GROUP_PREFIX = "index.routing.allocation.exclude";
     public static final Setting<Settings> INDEX_ROUTING_REQUIRE_GROUP_SETTING =
-        Setting.groupSetting(INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_INCLUDE_GROUP_SETTING =
-        Setting.groupSetting(INDEX_ROUTING_INCLUDE_GROUP_PREFIX + ".", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_INCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING =
-        Setting.groupSetting(INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".", Property.Dynamic, Property.IndexScope);
+        Setting.groupSetting(INDEX_ROUTING_EXCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING =
         Setting.groupSetting("index.routing.allocation.initial_recovery."); // this is only setable internally not a registered setting!!
 
@@ -246,10 +269,6 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                       Setting.Property.Dynamic,
                       Setting.Property.IndexScope);
 
-    public static final IndexMetaData PROTO = IndexMetaData.builder("")
-            .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
-            .numberOfShards(1).numberOfReplicas(0).build();
-
     public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations";
     static final String KEY_VERSION = "version";
     static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards";
@@ -262,6 +281,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
     private final int routingNumShards;
     private final int routingFactor;
+    private final int routingPartitionSize;
 
     private final int numberOfShards;
     private final int numberOfReplicas;
@@ -300,7 +320,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds,
                           DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
                           Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion,
-                          int routingNumShards, ActiveShardCount waitForActiveShards) {
+                          int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) {
 
         this.index = index;
         this.version = version;
@@ -324,6 +344,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion;
         this.routingNumShards = routingNumShards;
         this.routingFactor = routingNumShards / numberOfShards;
+        this.routingPartitionSize = routingPartitionSize;
         this.waitForActiveShards = waitForActiveShards;
         assert numberOfShards * routingFactor == routingNumShards :  routingNumShards + " must be a multiple of " + numberOfShards;
     }
@@ -403,6 +424,14 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return numberOfReplicas;
     }
 
+    public int getRoutingPartitionSize() {
+        return routingPartitionSize;
+    }
+
+    public boolean isRoutingPartitionedIndex() {
+        return routingPartitionSize != 1;
+    }
+
     public int getTotalNumberOfShards() {
         return totalNumberOfShards;
     }
@@ -564,13 +593,11 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return new IndexMetaDataDiff(previousState, this);
     }
 
-    @Override
-    public Diff<IndexMetaData> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<IndexMetaData> readDiffFrom(StreamInput in) throws IOException {
         return new IndexMetaDataDiff(in);
     }
 
-    @Override
-    public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
         return Builder.fromXContent(parser);
     }
 
@@ -593,7 +620,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         private final Diff<ImmutableOpenMap<String, Custom>> customs;
         private final Diff<ImmutableOpenIntMap<Set<String>>> inSyncAllocationIds;
 
-        public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
+        IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
             index = after.index.getName();
             version = after.version;
             routingNumShards = after.routingNumShards;
@@ -607,15 +634,17 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance());
         }
 
-        public IndexMetaDataDiff(StreamInput in) throws IOException {
+        IndexMetaDataDiff(StreamInput in) throws IOException {
             index = in.readString();
             routingNumShards = in.readInt();
             version = in.readLong();
             state = State.fromId(in.readByte());
             settings = Settings.readSettingsFromStream(in);
             primaryTerms = in.readVLongArray();
-            mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData.PROTO);
-            aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData.PROTO);
+            mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MappingMetaData::new,
+                MappingMetaData::readDiffFrom);
+            aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData::new,
+                AliasMetaData::readDiffFrom);
             customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
                 new DiffableUtils.DiffableValueSerializer<String, Custom>() {
                     @Override
@@ -623,6 +652,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                         return lookupPrototypeSafe(key).readFrom(in);
                     }
 
+                    @SuppressWarnings("unchecked")
                     @Override
                     public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
                         return lookupPrototypeSafe(key).readDiffFrom(in);
@@ -662,8 +692,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         }
     }
 
-    @Override
-    public IndexMetaData readFrom(StreamInput in) throws IOException {
+    public static IndexMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.version(in.readLong());
         builder.setRoutingNumShards(in.readInt());
@@ -672,12 +701,12 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         builder.primaryTerms(in.readVLongArray());
         int mappingsSize = in.readVInt();
         for (int i = 0; i < mappingsSize; i++) {
-            MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in);
+            MappingMetaData mappingMd = new MappingMetaData(in);
             builder.putMapping(mappingMd);
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
-            AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
+            AliasMetaData aliasMd = new AliasMetaData(in);
             builder.putAlias(aliasMd);
         }
         int customSize = in.readVInt();
@@ -799,6 +828,11 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             return routingNumShards == null ? numberOfShards() : routingNumShards;
         }
 
+        /**
+         * Returns the number of shards.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
         public int numberOfShards() {
             return settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1);
         }
@@ -808,10 +842,29 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             return this;
         }
 
+        /**
+         * Returns the number of replicas.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
         public int numberOfReplicas() {
             return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
         }
 
+        public Builder routingPartitionSize(int routingPartitionSize) {
+            settings = Settings.builder().put(settings).put(SETTING_ROUTING_PARTITION_SIZE, routingPartitionSize).build();
+            return this;
+        }
+
+        /**
+         * Returns the routing partition size.
+         *
+         * @return the provided value or -1 if it has not been set.
+         */
+        public int routingPartitionSize() {
+            return settings.getAsInt(SETTING_ROUTING_PARTITION_SIZE, -1);
+        }
+
         public Builder creationDate(long creationDate) {
             settings = Settings.builder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
             return this;
@@ -831,9 +884,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         }
 
         public Builder putMapping(String type, String source) throws IOException {
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-                putMapping(new MappingMetaData(type, parser.mapOrdered()));
-            }
+            putMapping(new MappingMetaData(type, XContentHelper.convertToMap(XContentFactory.xContent(source), source, true)));
             return this;
         }
 
@@ -877,7 +928,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         }
 
         public Builder putInSyncAllocationIds(int shardId, Set<String> allocationIds) {
-            inSyncAllocationIds.put(shardId, new HashSet(allocationIds));
+            inSyncAllocationIds.put(shardId, new HashSet<>(allocationIds));
             return this;
         }
 
@@ -956,6 +1007,12 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 throw new IllegalArgumentException("must specify non-negative number of shards for index [" + index + "]");
             }
 
+            int routingPartitionSize = INDEX_ROUTING_PARTITION_SIZE_SETTING.get(settings);
+            if (routingPartitionSize != 1 && routingPartitionSize >= getRoutingNumShards()) {
+                throw new IllegalArgumentException("routing partition size [" + routingPartitionSize + "] should be a positive number"
+                        + " less than the number of shards [" + getRoutingNumShards() + "] for [" + index + "]");
+            }
+
             // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable
             ImmutableOpenIntMap.Builder<Set<String>> filledInSyncAllocationIds = ImmutableOpenIntMap.builder();
             for (int i = 0; i < numberOfShards; i++) {
@@ -1024,7 +1081,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
             return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
                 tmpAliases.build(), customs.build(), filledInSyncAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
-                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion, getRoutingNumShards(), waitForActiveShards);
+                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion, getRoutingNumShards(), routingPartitionSize, waitForActiveShards);
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
@@ -1047,11 +1104,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                 if (binary) {
                     builder.value(cursor.value.source().compressed());
                 } else {
-                    byte[] data = cursor.value.source().uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> mapping = parser.mapOrdered();
-                        builder.map(mapping);
-                    }
+                    builder.map(XContentHelper.convertToMap(new BytesArray(cursor.value.source().uncompressed()), true).v2());
                 }
             }
             builder.endArray();
@@ -1203,12 +1256,10 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             }
             return builder.build();
         }
-
-        public static IndexMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(IndexMetaData.class));
+
     /**
      * Returns <code>true</code> iff the given settings indicate that the index
      * associated with these settings allocates it's shards on a shared
@@ -1216,8 +1267,10 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
      * is the returned value from
      * {@link #isIndexUsingShadowReplicas(org.elasticsearch.common.settings.Settings)}.
      */
-    public static boolean isOnSharedFilesystem(Settings settings) {
-        return settings.getAsBoolean(SETTING_SHARED_FILESYSTEM, isIndexUsingShadowReplicas(settings));
+    public boolean isOnSharedFilesystem(Settings settings) {
+        // don't use the setting directly, not to trigger verbose deprecation logging
+        return settings.getAsBooleanLenientForPreEs6Indices(
+            this.indexCreatedVersion, SETTING_SHARED_FILESYSTEM, isIndexUsingShadowReplicas(settings), deprecationLogger);
     }
 
     /**
@@ -1225,8 +1278,13 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
      * with these settings uses shadow replicas. Otherwise <code>false</code>. The default
      * setting for this is <code>false</code>.
      */
-    public static boolean isIndexUsingShadowReplicas(Settings settings) {
-        return settings.getAsBoolean(SETTING_SHADOW_REPLICAS, false);
+    public boolean isIndexUsingShadowReplicas() {
+        return isIndexUsingShadowReplicas(this.settings);
+    }
+
+    public boolean isIndexUsingShadowReplicas(Settings settings) {
+        // don't use the setting directly, not to trigger verbose deprecation logging
+        return settings.getAsBooleanLenientForPreEs6Indices(this.indexCreatedVersion, SETTING_SHADOW_REPLICAS, false, deprecationLogger);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 0c7a2db68dc..168fe2ad7f2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -502,11 +502,11 @@ public class IndexNameExpressionResolver extends AbstractComponent {
             this(state, options, System.currentTimeMillis(), preserveAliases);
         }
 
-        public Context(ClusterState state, IndicesOptions options, long startTime) {
+        Context(ClusterState state, IndicesOptions options, long startTime) {
            this(state, options, startTime, false);
         }
 
-        public Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) {
+        Context(ClusterState state, IndicesOptions options, long startTime, boolean preserveAliases) {
             this.state = state;
             this.options = options;
             this.startTime = startTime;
@@ -754,7 +754,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         private final String defaultDateFormatterPattern;
         private final DateTimeFormatter defaultDateFormatter;
 
-        public DateMathExpressionResolver(Settings settings) {
+        DateMathExpressionResolver(Settings settings) {
             String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC");
             this.defaultTimeZone = DateTimeZone.forID(defaultTimeZoneId);
             defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "YYYY.MM.dd");
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
index feabc380c4e..5bba34904d0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
@@ -23,7 +23,10 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -37,6 +40,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -49,7 +53,6 @@ import java.util.Set;
 
 public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaData> {
 
-    public static final IndexTemplateMetaData PROTO = IndexTemplateMetaData.builder("").build();
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(IndexTemplateMetaData.class));
 
     private final String name;
@@ -204,8 +207,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
         return result;
     }
 
-    @Override
-    public IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
+    public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.order(in.readInt());
         if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
@@ -220,7 +222,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
         }
         int aliasesSize = in.readVInt();
         for (int i = 0; i < aliasesSize; i++) {
-            AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
+            AliasMetaData aliasMd = new AliasMetaData(in);
             builder.putAlias(aliasMd);
         }
         int customSize = in.readVInt();
@@ -235,6 +237,10 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
         return builder.build();
     }
 
+    public static Diff<IndexTemplateMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(IndexTemplateMetaData::readFrom, in);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
@@ -395,10 +401,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
                 builder.startObject("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] mappingSource = cursor.value.uncompressed();
-                    Map<String, Object> mapping;
-                    try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {;
-                        mapping = parser.map();
-                    }
+                    Map<String, Object> mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), false).v2();
                     if (mapping.size() == 1 && mapping.containsKey(cursor.key)) {
                         // the type name is the root value, reduce it
                         mapping = (Map<String, Object>) mapping.get(cursor.key);
@@ -411,10 +414,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
                 builder.startArray("mappings");
                 for (ObjectObjectCursor<String, CompressedXContent> cursor : indexTemplateMetaData.mappings()) {
                     byte[] data = cursor.value.uncompressed();
-                    try (XContentParser parser = XContentFactory.xContent(data).createParser(data)) {
-                        Map<String, Object> mapping = parser.mapOrdered();
-                        builder.map(mapping);
-                    }
+                    builder.map(XContentHelper.convertToMap(new BytesArray(data), true).v2());
                 }
                 builder.endArray();
             }
@@ -529,10 +529,6 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
 
             return null;
         }
-
-        public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
index 3ea61385f1c..08d5211e12d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
@@ -21,28 +21,26 @@ package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DocumentMapper;
 
 import java.io.IOException;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 
 /**
  * Mapping configuration for a type.
  */
 public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
 
-    public static final MappingMetaData PROTO = new MappingMetaData();
-
     public static class Routing {
 
         public static final Routing EMPTY = new Routing(false);
@@ -89,10 +87,7 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
 
     public MappingMetaData(CompressedXContent mapping) throws IOException {
         this.source = mapping;
-        Map<String, Object> mappingMap;
-        try (XContentParser parser = XContentHelper.createParser(mapping.compressedReference())) {
-            mappingMap = parser.mapOrdered();
-        }
+        Map<String, Object> mappingMap = XContentHelper.convertToMap(mapping.compressedReference(), true).v2();
         if (mappingMap.size() != 1) {
             throw new IllegalStateException("Can't derive type from mapping, no root type: " + mapping.string());
         }
@@ -100,10 +95,6 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
         initMappers((Map<String, Object>) mappingMap.get(this.type));
     }
 
-    public MappingMetaData(Map<String, Object> mapping) throws IOException {
-        this(mapping.keySet().iterator().next(), mapping);
-    }
-
     public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
         this.type = type;
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().map(mapping);
@@ -132,7 +123,12 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("required")) {
-                    required = lenientNodeBooleanValue(fieldNode);
+                    try {
+                        required = nodeBooleanValue(fieldNode);
+                    } catch (IllegalArgumentException ex) {
+                        throw new IllegalArgumentException("Failed to create mapping for type [" + this.type() + "]. " +
+                            "Illegal value in field [_routing.required].", ex);
+                    }
                 }
             }
             this.routing = new Routing(required);
@@ -204,7 +200,7 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
             // timestamp
             out.writeBoolean(false); // enabled
             out.writeString(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format());
-            out.writeOptionalString(null);
+            out.writeOptionalString("now"); // 5.x default
             out.writeOptionalBoolean(null);
         }
         out.writeBoolean(hasParentField());
@@ -232,11 +228,11 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
         return result;
     }
 
-    public MappingMetaData readFrom(StreamInput in) throws IOException {
-        String type = in.readString();
-        CompressedXContent source = CompressedXContent.readCompressedString(in);
+    public MappingMetaData(StreamInput in) throws IOException {
+        type = in.readString();
+        source = CompressedXContent.readCompressedString(in);
         // routing
-        Routing routing = new Routing(in.readBoolean());
+        routing = new Routing(in.readBoolean());
         if (in.getVersion().before(Version.V_6_0_0_alpha1_UNRELEASED)) {
             // timestamp
             boolean enabled = in.readBoolean();
@@ -247,9 +243,11 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
             in.readOptionalString(); // defaultTimestamp
             in.readOptionalBoolean(); // ignoreMissing
         }
+        hasParentField = in.readBoolean();
+    }
 
-        final boolean hasParentField = in.readBoolean();
-        return new MappingMetaData(type, source, routing, hasParentField);
+    public static Diff<MappingMetaData> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(MappingMetaData::new, in);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index f091159aaec..a13e5e21e5e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -24,40 +24,36 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.Diffable;
 import org.elasticsearch.cluster.DiffableUtils;
-import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.NamedDiffable;
+import org.elasticsearch.cluster.NamedDiffableValueSerializer;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.HppcMaps;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.script.ScriptMetaData;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -69,18 +65,15 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.settings.Settings.readSettingsFromStream;
 import static org.elasticsearch.common.settings.Settings.writeSettingsToStream;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
-public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, FromXContentBuilder<MetaData>, ToXContent {
+public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, ToXContent {
 
-    public static final MetaData PROTO = builder().build();
+    private static final Logger logger = Loggers.getLogger(MetaData.class);
 
     public static final String ALL = "_all";
 
@@ -118,48 +111,11 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
      */
     public static EnumSet<XContentContext> ALL_CONTEXTS = EnumSet.allOf(XContentContext.class);
 
-    public interface Custom extends Diffable<Custom>, ToXContent {
-
-        String type();
-
-        Custom fromXContent(XContentParser parser) throws IOException;
+    public interface Custom extends NamedDiffable<Custom>, ToXContent {
 
         EnumSet<XContentContext> context();
     }
 
-    public static Map<String, Custom> customPrototypes = new HashMap<>();
-
-    static {
-        // register non plugin custom metadata
-        registerPrototype(RepositoriesMetaData.TYPE, RepositoriesMetaData.PROTO);
-        registerPrototype(IngestMetadata.TYPE, IngestMetadata.PROTO);
-        registerPrototype(ScriptMetaData.TYPE, ScriptMetaData.PROTO);
-        registerPrototype(IndexGraveyard.TYPE, IndexGraveyard.PROTO);
-    }
-
-    /**
-     * Register a custom index meta data factory. Make sure to call it from a static block.
-     */
-    public static void registerPrototype(String type, Custom proto) {
-        customPrototypes.put(type, proto);
-    }
-
-    @Nullable
-    public static <T extends Custom> T lookupPrototype(String type) {
-        //noinspection unchecked
-        return (T) customPrototypes.get(type);
-    }
-
-    public static <T extends Custom> T lookupPrototypeSafe(String type) {
-        //noinspection unchecked
-        T proto = (T) customPrototypes.get(type);
-        if (proto == null) {
-            throw new IllegalArgumentException("No custom metadata prototype registered for type [" + type + "], node likely missing plugins");
-        }
-        return proto;
-    }
-
-
     public static final Setting<Boolean> SETTING_READ_ONLY_SETTING =
         Setting.boolSetting("cluster.blocks.read_only", false, Property.Dynamic, Property.NodeScope);
 
@@ -175,6 +131,8 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
 
     public static final String GLOBAL_STATE_FILE_PREFIX = "global-";
 
+    private static final NamedDiffableValueSerializer<Custom> CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class);
+
     private final String clusterUUID;
     private final long version;
 
@@ -596,14 +554,14 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         // Check if any persistent metadata needs to be saved
         int customCount1 = 0;
         for (ObjectObjectCursor<String, Custom> cursor : metaData1.customs) {
-            if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
+            if (cursor.value.context().contains(XContentContext.GATEWAY)) {
                 if (!cursor.value.equals(metaData2.custom(cursor.key))) return false;
                 customCount1++;
             }
         }
         int customCount2 = 0;
-        for (ObjectObjectCursor<String, Custom> cursor : metaData2.customs) {
-            if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
+        for (ObjectCursor<Custom> cursor : metaData2.customs.values()) {
+            if (cursor.value.context().contains(XContentContext.GATEWAY)) {
                 customCount2++;
             }
         }
@@ -616,13 +574,11 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         return new MetaDataDiff(previousState, this);
     }
 
-    @Override
-    public Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
         return new MetaDataDiff(in);
     }
 
-    @Override
-    public MetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static MetaData fromXContent(XContentParser parser) throws IOException {
         return Builder.fromXContent(parser);
     }
 
@@ -644,35 +600,26 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         private Diff<ImmutableOpenMap<String, IndexTemplateMetaData>> templates;
         private Diff<ImmutableOpenMap<String, Custom>> customs;
 
-        public MetaDataDiff(MetaData before, MetaData after) {
+        MetaDataDiff(MetaData before, MetaData after) {
             clusterUUID = after.clusterUUID;
             version = after.version;
             transientSettings = after.transientSettings;
             persistentSettings = after.persistentSettings;
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
             templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
-            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
+            customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
-        public MetaDataDiff(StreamInput in) throws IOException {
+        MetaDataDiff(StreamInput in) throws IOException {
             clusterUUID = in.readString();
             version = in.readLong();
             transientSettings = Settings.readSettingsFromStream(in);
             persistentSettings = Settings.readSettingsFromStream(in);
-            indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData.PROTO);
-            templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData.PROTO);
-            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(),
-                    new DiffableUtils.DiffableValueSerializer<String, Custom>() {
-                @Override
-                public Custom read(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readFrom(in);
-                }
-
-                @Override
-                public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
-                    return lookupPrototypeSafe(key).readDiffFrom(in);
-                }
-            });
+            indices = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexMetaData::readFrom,
+                IndexMetaData::readDiffFrom);
+            templates = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData::readFrom,
+                IndexTemplateMetaData::readDiffFrom);
+            customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
         @Override
@@ -700,8 +647,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         }
     }
 
-    @Override
-    public MetaData readFrom(StreamInput in) throws IOException {
+    public static MetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder();
         builder.version = in.readLong();
         builder.clusterUUID = in.readString();
@@ -709,17 +655,16 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         builder.persistentSettings(readSettingsFromStream(in));
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            builder.put(IndexMetaData.Builder.readFrom(in), false);
+            builder.put(IndexMetaData.readFrom(in), false);
         }
         size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            builder.put(IndexTemplateMetaData.Builder.readFrom(in));
+            builder.put(IndexTemplateMetaData.readFrom(in));
         }
         int customSize = in.readVInt();
         for (int i = 0; i < customSize; i++) {
-            String type = in.readString();
-            Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in);
-            builder.putCustom(type, customIndexMetaData);
+            Custom customIndexMetaData = in.readNamedWriteable(Custom.class);
+            builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData);
         }
         return builder.build();
     }
@@ -738,10 +683,18 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         for (ObjectCursor<IndexTemplateMetaData> cursor : templates.values()) {
             cursor.value.writeTo(out);
         }
-        out.writeVInt(customs.size());
-        for (ObjectObjectCursor<String, Custom> cursor : customs) {
-            out.writeString(cursor.key);
-            cursor.value.writeTo(out);
+        // filter out custom states not supported by the other node
+        int numberOfCustoms = 0;
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                numberOfCustoms++;
+            }
+        }
+        out.writeVInt(numberOfCustoms);
+        for (ObjectCursor<Custom> cursor : customs.values()) {
+            if (out.getVersion().onOrAfter(cursor.value.getMinimalSupportedVersion())) {
+                out.writeNamedWriteable(cursor.value);
+            }
         }
     }
 
@@ -1020,7 +973,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
             builder.field("version", metaData.version());
             builder.field("cluster_uuid", metaData.clusterUUID);
 
-            if (!metaData.persistentSettings().getAsMap().isEmpty()) {
+            if (!metaData.persistentSettings().isEmpty()) {
                 builder.startObject("settings");
                 for (Map.Entry<String, String> entry : metaData.persistentSettings().getAsMap().entrySet()) {
                     builder.field(entry.getKey(), entry.getValue());
@@ -1028,7 +981,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
                 builder.endObject();
             }
 
-            if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) {
+            if (context == XContentContext.API && !metaData.transientSettings().isEmpty()) {
                 builder.startObject("transient_settings");
                 for (Map.Entry<String, String> entry : metaData.transientSettings().getAsMap().entrySet()) {
                     builder.field(entry.getKey(), entry.getValue());
@@ -1051,8 +1004,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
             }
 
             for (ObjectObjectCursor<String, Custom> cursor : metaData.customs()) {
-                Custom proto = lookupPrototypeSafe(cursor.key);
-                if (proto.context().contains(context)) {
+                if (cursor.value.context().contains(context)) {
                     builder.startObject(cursor.key);
                     cursor.value.toXContent(builder, params);
                     builder.endObject();
@@ -1103,14 +1055,12 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
                             builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
                         }
                     } else {
-                        // check if its a custom index metadata
-                        Custom proto = lookupPrototype(currentFieldName);
-                        if (proto == null) {
-                            //TODO warn
+                        try {
+                            Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
+                            builder.putCustom(custom.getWriteableName(), custom);
+                        } catch (UnknownNamedObjectException ex) {
+                            logger.warn("Skipping unknown custom object with type {}", currentFieldName);
                             parser.skipChildren();
-                        } else {
-                            Custom custom = proto.fromXContent(parser);
-                            builder.putCustom(custom.type(), custom);
                         }
                     }
                 } else if (token.isValue()) {
@@ -1127,10 +1077,6 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
             }
             return builder.build();
         }
-
-        public static MetaData readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
     private static final ToXContent.Params FORMAT_PARAMS;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 422492e396b..0bde4a23b03 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.ActiveShardsObserver;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
@@ -59,14 +60,15 @@ import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.IndexCreationException;
 import org.elasticsearch.indices.IndicesService;
@@ -112,12 +114,15 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     private final Environment env;
     private final IndexScopedSettings indexScopedSettings;
     private final ActiveShardsObserver activeShardsObserver;
+    private final NamedXContentRegistry xContentRegistry;
+    private final ThreadPool threadPool;
 
     @Inject
     public MetaDataCreateIndexService(Settings settings, ClusterService clusterService,
                                       IndicesService indicesService, AllocationService allocationService,
                                       AliasValidator aliasValidator, Environment env,
-                                      IndexScopedSettings indexScopedSettings, ThreadPool threadPool) {
+                                      IndexScopedSettings indexScopedSettings, ThreadPool threadPool,
+                                      NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
@@ -126,6 +131,8 @@ public class MetaDataCreateIndexService extends AbstractComponent {
         this.env = env;
         this.indexScopedSettings = indexScopedSettings;
         this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool);
+        this.threadPool = threadPool;
+        this.xContentRegistry = xContentRegistry;
     }
 
     /**
@@ -215,7 +222,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
         request.settings(updatedSettingsBuilder.build());
 
         clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
-                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, wrapPreservingContext(listener)) {
                     @Override
                     protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
                         return new ClusterStateUpdateResponse(acknowledged);
@@ -247,7 +254,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             List<String> templateNames = new ArrayList<>();
 
                             for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
-                                mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
+                                mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
                             }
 
                             for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
@@ -258,10 +265,13 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             for (IndexTemplateMetaData template : templates) {
                                 templateNames.add(template.getName());
                                 for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
+                                    String mappingString = cursor.value.string();
                                     if (mappings.containsKey(cursor.key)) {
-                                        XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
+                                        XContentHelper.mergeDefaults(mappings.get(cursor.key),
+                                                MapperService.parseMapping(xContentRegistry, mappingString));
                                     } else {
-                                        mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
+                                        mappings.put(cursor.key,
+                                            MapperService.parseMapping(xContentRegistry, mappingString));
                                     }
                                 }
                                 // handle custom
@@ -356,10 +366,10 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             // now add the mappings
                             MapperService mapperService = indexService.mapperService();
                             try {
-                                mapperService.merge(mappings, request.updateAllTypes());
-                            } catch (MapperParsingException mpe) {
+                                mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, request.updateAllTypes());
+                            } catch (Exception e) {
                                 removalExtraInfo = "failed on parsing default mapping/mappings on index creation";
-                                throw mpe;
+                                throw e;
                             }
 
                             // the context is only used for validation so it's fine to pass fake values for the shard id and the current
@@ -367,12 +377,13 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             final QueryShardContext queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L);
                             for (Alias alias : request.aliases()) {
                                 if (Strings.hasLength(alias.filter())) {
-                                    aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext);
+                                    aliasValidator.validateAliasFilter(alias.name(), alias.filter(), queryShardContext, xContentRegistry);
                                 }
                             }
                             for (AliasMetaData aliasMetaData : templatesAliases.values()) {
                                 if (aliasMetaData.filter() != null) {
-                                    aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(), queryShardContext);
+                                    aliasValidator.validateAliasFilter(aliasMetaData.alias(), aliasMetaData.filter().uncompressed(),
+                                            queryShardContext, xContentRegistry);
                                 }
                             }
 
@@ -420,7 +431,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                                     .put(indexMetaData, false)
                                     .build();
 
-                            String maybeShadowIndicator = IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) ? "s" : "";
+                            String maybeShadowIndicator = indexMetaData.isIndexUsingShadowReplicas() ? "s" : "";
                             logger.info("[{}] creating index, cause [{}], templates {}, shards [{}]/[{}{}], mappings {}",
                                     request.index(), request.cause(), templateNames, indexMetaData.getNumberOfShards(),
                                     indexMetaData.getNumberOfReplicas(), maybeShadowIndicator, mappings.keySet());
@@ -465,6 +476,10 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                 });
     }
 
+    private ContextPreservingActionListener<ClusterStateUpdateResponse> wrapPreservingContext(ActionListener<ClusterStateUpdateResponse> listener) {
+        return new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(false), listener);
+    }
+
     private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state) throws IOException {
         List<IndexTemplateMetaData> templateMetadata = new ArrayList<>();
         for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
@@ -538,6 +553,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
             throw new IllegalArgumentException("mappings are not allowed when shrinking indices" +
                 ", all mappings are copied from the source index");
         }
+
         if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
             // this method applies all necessary checks ie. if the target shards are less than the source shards
             // of if the source shards are divisible by the number of target shards
@@ -581,9 +597,15 @@ public class MetaDataCreateIndexService extends AbstractComponent {
             .put("index.allocation.max_retries", 1)
             // now copy all similarity / analysis settings - this overrides all settings from the user unless they
             // wanna add extra settings
+            .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion())
+            .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion())
             .put(sourceMetaData.getSettings().filter(analysisSimilarityPredicate))
+            .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize())
             .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), shrinkFromIndex.getName())
             .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), shrinkFromIndex.getUUID());
+        if (sourceMetaData.getMinimumCompatibleVersion() != null) {
+            indexSettingsBuilder.put(IndexMetaData.SETTING_VERSION_MINIMUM_COMPATIBLE, sourceMetaData.getMinimumCompatibleVersion());
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index 7dbc06cba0f..a2212b5c3f0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -97,7 +97,7 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
         final int previousGraveyardSize = graveyardBuilder.tombstones().size();
         for (final Index index : indices) {
             String indexName = index.getName();
-            logger.debug("[{}] deleting index", index);
+            logger.info("{} deleting index", index);
             routingTableBuilder.remove(indexName);
             clusterBlocksBuilder.removeIndexBlocks(indexName);
             metaDataBuilder.remove(indexName);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
index c1de936d9c7..81fbddce46a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesService.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
@@ -64,18 +65,17 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
 
     private final MetaDataDeleteIndexService deleteIndexService;
 
+    private final NamedXContentRegistry xContentRegistry;
+
     @Inject
-    public MetaDataIndexAliasesService(
-        Settings settings,
-        ClusterService clusterService,
-        IndicesService indicesService,
-        AliasValidator aliasValidator,
-        MetaDataDeleteIndexService deleteIndexService) {
+    public MetaDataIndexAliasesService(Settings settings, ClusterService clusterService, IndicesService indicesService,
+            AliasValidator aliasValidator, MetaDataDeleteIndexService deleteIndexService, NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.aliasValidator = aliasValidator;
         this.deleteIndexService = deleteIndexService;
+        this.xContentRegistry = xContentRegistry;
     }
 
     public void indicesAliases(final IndicesAliasesClusterStateUpdateRequest request,
@@ -141,21 +141,18 @@ public class MetaDataIndexAliasesService extends AbstractComponent {
                                 // temporarily create the index and add mappings so we can parse the filter
                                 try {
                                     indexService = indicesService.createIndex(index, emptyList(), shardId -> {});
+                                    indicesToClose.add(index.getIndex());
                                 } catch (IOException e) {
                                     throw new ElasticsearchException("Failed to create temporary index for parsing the alias", e);
                                 }
-                                for (ObjectCursor<MappingMetaData> cursor : index.getMappings().values()) {
-                                    MappingMetaData mappingMetaData = cursor.value;
-                                    indexService.mapperService().merge(mappingMetaData.type(), mappingMetaData.source(),
-                                        MapperService.MergeReason.MAPPING_RECOVERY, false);
-                                }
-                                indicesToClose.add(index.getIndex());
+                                indexService.mapperService().merge(index, MapperService.MergeReason.MAPPING_RECOVERY, false);
                             }
                             indices.put(action.getIndex(), indexService);
                         }
                         // the context is only used for validation so it's fine to pass fake values for the shard id and the current
                         // timestamp
-                        aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, null, () -> 0L));
+                        aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, null, () -> 0L),
+                                xContentRegistry);
                     }
                 };
                 changed |= action.apply(newAliasValidator, metadata, index);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 020a1d75231..e35fc7c837c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
@@ -35,10 +36,12 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.indices.IndexTemplateMissingException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
@@ -64,18 +67,20 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
     private final IndicesService indicesService;
     private final MetaDataCreateIndexService metaDataCreateIndexService;
     private final IndexScopedSettings indexScopedSettings;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService,
                                         MetaDataCreateIndexService metaDataCreateIndexService,
                                         AliasValidator aliasValidator, IndicesService indicesService,
-                                        IndexScopedSettings indexScopedSettings) {
+                                        IndexScopedSettings indexScopedSettings, NamedXContentRegistry xContentRegistry) {
         super(settings);
         this.clusterService = clusterService;
         this.aliasValidator = aliasValidator;
         this.indicesService = indicesService;
         this.metaDataCreateIndexService = metaDataCreateIndexService;
         this.indexScopedSettings = indexScopedSettings;
+        this.xContentRegistry = xContentRegistry;
     }
 
     public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
@@ -164,7 +169,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
                     throw new IllegalArgumentException("index_template [" + request.name + "] already exists");
                 }
 
-                validateAndAddTemplate(request, templateBuilder, indicesService);
+                validateAndAddTemplate(request, templateBuilder, indicesService, xContentRegistry);
 
                 for (Alias alias : request.aliases) {
                     AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@@ -189,16 +194,20 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
     }
 
     private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder,
-            IndicesService indicesService) throws Exception {
+            IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
         Index createdIndex = null;
         final String temporaryIndexName = UUIDs.randomBase64UUID();
         try {
+            // use the provided values, otherwise just pick valid dummy values
+            int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
+            int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS,
+                    dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
 
             //create index service for parsing and validating "mappings"
             Settings dummySettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(request.settings)
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
                 .build();
@@ -219,10 +228,10 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
                 } catch (Exception e) {
                     throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
                 }
-                mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
+                mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
             }
 
-            dummyIndexService.mapperService().merge(mappingsForValidation, false);
+            dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false);
 
         } finally {
             if (createdIndex != null) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 2a8b80b9e68..614d12547fc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -50,12 +51,15 @@ import java.util.Set;
  */
 public class MetaDataIndexUpgradeService extends AbstractComponent {
 
+    private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
 
     @Inject
-    public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry, IndexScopedSettings indexScopedSettings) {
+    public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
+            IndexScopedSettings indexScopedSettings) {
         super(settings);
+        this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
     }
@@ -141,16 +145,13 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
 
                 @Override
                 public Set<Entry<String, NamedAnalyzer>> entrySet() {
-                    // just to ensure we can iterate over this single analzyer
-                    return Collections.singletonMap(fakeDefault.name(), fakeDefault).entrySet();
+                    return Collections.emptySet();
                 }
             };
-            try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap)) {
-                MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, similarityService, mapperRegistry, () -> null);
-                for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
-                    MappingMetaData mappingMetaData = cursor.value;
-                    mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
-                }
+            try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(indexSettings, fakeDefault, fakeDefault, fakeDefault, analyzerMap, analyzerMap)) {
+                MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService,
+                        mapperRegistry, () -> null);
+                mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, false);
             }
         } catch (Exception ex) {
             // Wrap the inner exception so we have the index name in the exception message
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index 4e9b114ff13..c0032a4b6a4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -26,9 +26,9 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest;
 import org.elasticsearch.cluster.AckedClusterStateTaskListener;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -43,6 +43,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidTypeNameException;
 
@@ -63,8 +64,8 @@ public class MetaDataMappingService extends AbstractComponent {
     private final ClusterService clusterService;
     private final IndicesService indicesService;
 
-    final ClusterStateTaskExecutor<RefreshTask> refreshExecutor = new RefreshTaskExecutor();
-    final ClusterStateTaskExecutor<PutMappingClusterStateUpdateRequest> putMappingExecutor = new PutMappingExecutor();
+    final RefreshTaskExecutor refreshExecutor = new RefreshTaskExecutor();
+    final PutMappingExecutor putMappingExecutor = new PutMappingExecutor();
 
 
     @Inject
@@ -91,9 +92,9 @@ public class MetaDataMappingService extends AbstractComponent {
 
     class RefreshTaskExecutor implements ClusterStateTaskExecutor<RefreshTask> {
         @Override
-        public BatchResult<RefreshTask> execute(ClusterState currentState, List<RefreshTask> tasks) throws Exception {
+        public ClusterTasksResult<RefreshTask> execute(ClusterState currentState, List<RefreshTask> tasks) throws Exception {
             ClusterState newClusterState = executeRefresh(currentState, tasks);
-            return BatchResult.<RefreshTask>builder().successes(tasks).build(newClusterState);
+            return ClusterTasksResult.<RefreshTask>builder().successes(tasks).build(newClusterState);
         }
     }
 
@@ -146,10 +147,7 @@ public class MetaDataMappingService extends AbstractComponent {
                 // we need to create the index here, and add the current mapping to it, so we can merge
                 indexService = indicesService.createIndex(indexMetaData, Collections.emptyList(), shardId -> {});
                 removeIndex = true;
-                for (ObjectCursor<MappingMetaData> metaData : indexMetaData.getMappings().values()) {
-                    // don't apply the default mapping, it has been applied when the mapping was created
-                    indexService.mapperService().merge(metaData.value.type(), metaData.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
-                }
+                indexService.mapperService().merge(indexMetaData, MergeReason.MAPPING_RECOVERY, true);
             }
 
             IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData);
@@ -213,10 +211,10 @@ public class MetaDataMappingService extends AbstractComponent {
 
     class PutMappingExecutor implements ClusterStateTaskExecutor<PutMappingClusterStateUpdateRequest> {
         @Override
-        public BatchResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState,
-                                                                        List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
+        public ClusterTasksResult<PutMappingClusterStateUpdateRequest> execute(ClusterState currentState,
+                                                                               List<PutMappingClusterStateUpdateRequest> tasks) throws Exception {
             Map<Index, MapperService> indexMapperServices = new HashMap<>();
-            BatchResult.Builder<PutMappingClusterStateUpdateRequest> builder = BatchResult.builder();
+            ClusterTasksResult.Builder<PutMappingClusterStateUpdateRequest> builder = ClusterTasksResult.builder();
             try {
                 for (PutMappingClusterStateUpdateRequest request : tasks) {
                     try {
@@ -226,10 +224,7 @@ public class MetaDataMappingService extends AbstractComponent {
                                 MapperService mapperService = indicesService.createIndexMapperService(indexMetaData);
                                 indexMapperServices.put(index, mapperService);
                                 // add mappings for all types, we need them for cross-type validation
-                                for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
-                                    mapperService.merge(mapping.value.type(), mapping.value.source(),
-                                        MapperService.MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
-                                }
+                                mapperService.merge(indexMetaData, MergeReason.MAPPING_RECOVERY, request.updateAllTypes());
                             }
                         }
                         currentState = applyRequest(currentState, request, indexMapperServices);
@@ -313,7 +308,7 @@ public class MetaDataMappingService extends AbstractComponent {
                 if (existingMapper != null) {
                     existingSource = existingMapper.mappingSource();
                 }
-                DocumentMapper mergedMapper = mapperService.merge(mappingType, mappingUpdateSource, MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
+                DocumentMapper mergedMapper = mapperService.merge(mappingType, mappingUpdateSource, MergeReason.MAPPING_UPDATE, request.updateAllTypes());
                 CompressedXContent updatedSource = mergedMapper.mappingSource();
 
                 if (existingSource != null) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 6b19e2d4bf3..6bee999a9e5 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -24,6 +24,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeSettingsClusterStateUpdateRequest;
+import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
@@ -44,6 +45,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -65,12 +67,14 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
     private final IndexScopedSettings indexScopedSettings;
     private final IndicesService indicesService;
+    private final ThreadPool threadPool;
 
     @Inject
     public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService,
-                                         IndexScopedSettings indexScopedSettings, IndicesService indicesService) {
+                                         IndexScopedSettings indexScopedSettings, IndicesService indicesService, ThreadPool threadPool) {
         super(settings);
         this.clusterService = clusterService;
+        this.threadPool = threadPool;
         this.clusterService.addListener(this);
         this.allocationService = allocationService;
         this.indexScopedSettings = indexScopedSettings;
@@ -180,7 +184,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         final boolean preserveExisting = request.isPreserveExisting();
 
         clusterService.submitStateUpdateTask("update-settings",
-                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+                new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, wrapPreservingContext(listener)) {
 
             @Override
             protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
@@ -214,7 +218,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                             closeIndices
                     ));
                 }
-                if (!skippedSettigns.getAsMap().isEmpty() && !openIndices.isEmpty()) {
+                if (!skippedSettigns.isEmpty() && !openIndices.isEmpty()) {
                     throw new IllegalArgumentException(String.format(Locale.ROOT,
                             "Can't update non dynamic settings [%s] for open indices %s",
                             skippedSettigns.getAsMap().keySet(),
@@ -290,6 +294,10 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
         });
     }
 
+    private ContextPreservingActionListener<ClusterStateUpdateResponse> wrapPreservingContext(ActionListener<ClusterStateUpdateResponse> listener) {
+        return new ContextPreservingActionListener<>(threadPool.getThreadContext().newRestorableContext(false), listener);
+    }
+
     /**
      * Updates the cluster block only iff the setting exists in the given settings
      */
@@ -308,9 +316,8 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
 
     public void upgradeIndexSettings(final UpgradeSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
-
-
-        clusterService.submitStateUpdateTask("update-index-compatibility-versions", new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
+        clusterService.submitStateUpdateTask("update-index-compatibility-versions",
+            new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, wrapPreservingContext(listener)) {
 
             @Override
             protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
index 2dc842ceaae..67909bff614 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoriesMetaData.java
@@ -21,6 +21,9 @@ package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData.Custom;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -39,12 +42,10 @@ import java.util.List;
 /**
  * Contains metadata about registered snapshot repositories
  */
-public class RepositoriesMetaData extends AbstractDiffable<Custom> implements MetaData.Custom {
+public class RepositoriesMetaData extends AbstractNamedDiffable<Custom> implements Custom {
 
     public static final String TYPE = "repositories";
 
-    public static final RepositoriesMetaData PROTO = new RepositoriesMetaData();
-
     private final List<RepositoryMetaData> repositories;
 
     /**
@@ -100,20 +101,20 @@ public class RepositoriesMetaData extends AbstractDiffable<Custom> implements Me
      * {@inheritDoc}
      */
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public Custom readFrom(StreamInput in) throws IOException {
+    public RepositoriesMetaData(StreamInput in) throws IOException {
         RepositoryMetaData[] repository = new RepositoryMetaData[in.readVInt()];
         for (int i = 0; i < repository.length; i++) {
-            repository[i] = RepositoryMetaData.readFrom(in);
+            repository[i] = new RepositoryMetaData(in);
         }
-        return new RepositoriesMetaData(repository);
+        this.repositories = Arrays.asList(repository);
+    }
+
+    public static NamedDiff<Custom> readDiffFrom(StreamInput in) throws  IOException {
+        return readDiffFrom(Custom.class, TYPE, in);
     }
 
     /**
@@ -127,11 +128,7 @@ public class RepositoriesMetaData extends AbstractDiffable<Custom> implements Me
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public RepositoriesMetaData fromXContent(XContentParser parser) throws IOException {
+    public static RepositoriesMetaData fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         List<RepositoryMetaData> repository = new ArrayList<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
index 3c13a10c1cf..847db915b8b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/RepositoryMetaData.java
@@ -73,17 +73,10 @@ public class RepositoryMetaData {
     }
 
 
-    /**
-     * Reads repository metadata from stream input
-     *
-     * @param in stream input
-     * @return repository metadata
-     */
-    public static RepositoryMetaData readFrom(StreamInput in) throws IOException {
-        String name = in.readString();
-        String type = in.readString();
-        Settings settings = Settings.readSettingsFromStream(in);
-        return new RepositoryMetaData(name, type, settings);
+    public RepositoryMetaData(StreamInput in) throws IOException {
+        name = in.readString();
+        type = in.readString();
+        settings = Settings.readSettingsFromStream(in);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index a8290d317e8..c81161f1deb 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -202,7 +202,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
             roles.add(DiscoveryNode.Role.DATA);
         }
 
-        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress,attributes, roles, Version.CURRENT);
+        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes, roles, Version.CURRENT);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
index fad86caa7cc..3fcfdc08722 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.node;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -28,6 +29,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Consumer;
 
 public class DiscoveryNodeFilters {
 
@@ -36,6 +38,25 @@ public class DiscoveryNodeFilters {
         OR
     }
 
+    /**
+     * Validates the IP addresses in a group of {@link Settings} by looking for the keys
+     * "_ip", "_host_ip", and "_publish_ip" and ensuring each of their comma separated values
+     * is a valid IP address.
+     */
+    public static final Consumer<Settings> IP_VALIDATOR = (settings) -> {
+        Map<String, String> settingsMap = settings.getAsMap();
+        for (Map.Entry<String, String> entry : settingsMap.entrySet()) {
+            String propertyKey = entry.getKey();
+            if ("_ip".equals(propertyKey) || "_host_ip".equals(propertyKey) || "_publish_ip".equals(propertyKey)) {
+                for (String value : Strings.tokenizeToStringArray(entry.getValue(), ",")) {
+                    if (InetAddresses.isInetAddress(value) == false) {
+                        throw new IllegalArgumentException("invalid IP address [" + value + "] for [" + propertyKey + "]");
+                    }
+                }
+            }
+        }
+    };
+
     public static DiscoveryNodeFilters buildFromSettings(OpType opType, String prefix, Settings settings) {
         return buildFromKeyValue(opType, settings.getByPrefix(prefix).getAsMap());
     }
@@ -43,7 +64,7 @@ public class DiscoveryNodeFilters {
     public static DiscoveryNodeFilters buildFromKeyValue(OpType opType, Map<String, String> filters) {
         Map<String, String[]> bFilters = new HashMap<>();
         for (Map.Entry<String, String> entry : filters.entrySet()) {
-            String[] values = Strings.splitStringByCommaToArray(entry.getValue());
+            String[] values = Strings.tokenizeToStringArray(entry.getValue(), ",");
             if (values.length > 0) {
                 bFilters.put(entry.getKey(), values);
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 6d80a9573ba..4829c9c9753 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -24,6 +24,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -46,7 +47,6 @@ import java.util.Map;
 public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements Iterable<DiscoveryNode> {
 
     public static final DiscoveryNodes EMPTY_NODES = builder().build();
-    public static final DiscoveryNodes PROTO = EMPTY_NODES;
 
     private final ImmutableOpenMap<String, DiscoveryNode> nodes;
     private final ImmutableOpenMap<String, DiscoveryNode> dataNodes;
@@ -523,7 +523,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         }
     }
 
-    private DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+    public static DiscoveryNodes readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
         Builder builder = new Builder();
         if (in.readBoolean()) {
             builder.masterNodeId(in.readString());
@@ -546,9 +546,8 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         return builder.build();
     }
 
-    @Override
-    public DiscoveryNodes readFrom(StreamInput in) throws IOException {
-        return readFrom(in, getLocalNode());
+    public static Diff<DiscoveryNodes> readDiffFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
+        return AbstractDiffable.readDiffFrom(in1 -> readFrom(in1, localNode), in);
     }
 
     public static Builder builder() {
@@ -678,10 +677,6 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
             );
         }
 
-        public static DiscoveryNodes readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
-            return PROTO.readFrom(in, localNode);
-        }
-
         public boolean isLocalNodeElectedMaster() {
             return masterNodeId != null && masterNodeId.equals(localNodeId);
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
index cb0fb487693..9ab91f9f8de 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
@@ -21,8 +21,6 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,7 +46,7 @@ public class AllocationId implements ToXContent, Writeable {
     private static final String ID_KEY = "id";
     private static final String RELOCATION_ID_KEY = "relocation_id";
 
-    private static final ObjectParser<AllocationId.Builder, ParseFieldMatcherSupplier> ALLOCATION_ID_PARSER = new ObjectParser<>(
+    private static final ObjectParser<AllocationId.Builder, Void> ALLOCATION_ID_PARSER = new ObjectParser<>(
             "allocationId");
 
     static {
@@ -203,6 +201,6 @@ public class AllocationId implements ToXContent, Writeable {
     }
 
     public static AllocationId fromXContent(XContentParser parser) throws IOException {
-        return ALLOCATION_ID_PARSER.parse(parser, new AllocationId.Builder(), () -> ParseFieldMatcher.STRICT).build();
+        return ALLOCATION_ID_PARSER.parse(parser, new AllocationId.Builder(), null).build();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 58ee6d70f2c..53590550d84 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -25,6 +25,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource;
@@ -63,8 +64,6 @@ import java.util.Set;
  */
 public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> implements Iterable<IndexShardRoutingTable> {
 
-    public static final IndexRoutingTable PROTO = builder(new Index("", "_na_")).build();
-
     private final Index index;
     private final ShardShuffler shuffler;
 
@@ -141,7 +140,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
                 }
 
                 if (indexMetaData.getCreationVersion().onOrAfter(Version.V_5_0_0_alpha1) &&
-                    IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) == false && // see #20650
+                    indexMetaData.isIndexUsingShadowReplicas() == false && // see #20650
                     shardRouting.primary() && shardRouting.initializing() && shardRouting.relocating() == false &&
                     RecoverySource.isInitialRecovery(shardRouting.recoverySource().getType()) == false &&
                     inSyncAllocationIds.contains(shardRouting.allocationId().getId()) == false)
@@ -319,8 +318,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
         return result;
     }
 
-    @Override
-    public IndexRoutingTable readFrom(StreamInput in) throws IOException {
+    public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
         Index index = new Index(in);
         Builder builder = new Builder(index);
 
@@ -332,6 +330,10 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
         return builder.build();
     }
 
+    public static Diff<IndexRoutingTable> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(IndexRoutingTable::readFrom, in);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         index.writeTo(out);
@@ -354,17 +356,6 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
             this.index = index;
         }
 
-        /**
-         * Reads an {@link IndexRoutingTable} from an {@link StreamInput}
-         *
-         * @param in {@link StreamInput} to read the {@link IndexRoutingTable} from
-         * @return {@link IndexRoutingTable} read
-         * @throws IOException if something happens during read
-         */
-        public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
-
         /**
          * Initializes a new empty index, as if it was created from an API.
          */
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 94cb4b8c8e8..6881cc75657 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -92,13 +92,10 @@ public class OperationRouting extends AbstractComponent {
             final Set<String> effectiveRouting = routing.get(index);
             if (effectiveRouting != null) {
                 for (String r : effectiveRouting) {
-                    int shardId = generateShardId(indexMetaData, null, r);
-                    IndexShardRoutingTable indexShard = indexRouting.shard(shardId);
-                    if (indexShard == null) {
-                        throw new ShardNotFoundException(new ShardId(indexRouting.getIndex(), shardId));
+                    final int routingPartitionSize = indexMetaData.getRoutingPartitionSize();
+                    for (int partitionOffset = 0; partitionOffset < routingPartitionSize; partitionOffset++) {
+                        set.add(shardRoutingTable(indexRouting, calculateScaledShardId(indexMetaData, r, partitionOffset)));
                     }
-                    // we might get duplicates, but that's ok, they will override one another
-                    set.add(indexShard);
                 }
             } else {
                 for (IndexShardRoutingTable indexShard : indexRouting) {
@@ -187,6 +184,14 @@ public class OperationRouting extends AbstractComponent {
         }
     }
 
+    private IndexShardRoutingTable shardRoutingTable(IndexRoutingTable indexRouting, int shardId) {
+        IndexShardRoutingTable indexShard = indexRouting.shard(shardId);
+        if (indexShard == null) {
+            throw new ShardNotFoundException(new ShardId(indexRouting.getIndex(), shardId));
+        }
+        return indexShard;
+    }
+
     protected IndexRoutingTable indexRoutingTable(ClusterState clusterState, String index) {
         IndexRoutingTable indexRouting = clusterState.routingTable().index(index);
         if (indexRouting == null) {
@@ -213,15 +218,33 @@ public class OperationRouting extends AbstractComponent {
         return new ShardId(indexMetaData.getIndex(), generateShardId(indexMetaData, id, routing));
     }
 
-    static int generateShardId(IndexMetaData indexMetaData, String id, @Nullable String routing) {
-        final int hash;
+    static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) {
+        final String effectiveRouting;
+        final int partitionOffset;
+
         if (routing == null) {
-            hash = Murmur3HashFunction.hash(id);
+            assert(indexMetaData.isRoutingPartitionedIndex() == false) : "A routing value is required for gets from a partitioned index";
+            effectiveRouting = id;
         } else {
-            hash = Murmur3HashFunction.hash(routing);
+            effectiveRouting = routing;
         }
+
+        if (indexMetaData.isRoutingPartitionedIndex()) {
+            partitionOffset = Math.floorMod(Murmur3HashFunction.hash(id), indexMetaData.getRoutingPartitionSize());
+        } else {
+            // we would have still got 0 above but this check just saves us an unnecessary hash calculation
+            partitionOffset = 0;
+        }
+
+        return calculateScaledShardId(indexMetaData, effectiveRouting, partitionOffset);
+    }
+
+    private static int calculateScaledShardId(IndexMetaData indexMetaData, String effectiveRouting, int partitionOffset) {
+        final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset;
+
         // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size
         // of original index to hash documents
         return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 6b0b1324717..45d567b657e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -540,7 +540,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                 if (failedShard.primary()) {
                     // promote active replica to primary if active replica exists (only the case for shadow replicas)
                     ShardRouting activeReplica = activeReplica(failedShard.shardId());
-                    assert activeReplica == null || IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings()) :
+                    assert activeReplica == null || indexMetaData.isIndexUsingShadowReplicas() :
                         "initializing primary [" + failedShard + "] with active replicas [" + activeReplica + "] only expected when " +
                             "using shadow replicas";
                     if (activeReplica == null) {
@@ -599,7 +599,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         assert activeReplica.started() : "replica relocation should have been cancelled: " + activeReplica;
         ShardRouting primarySwappedCandidate = promoteActiveReplicaShardToPrimary(activeReplica);
         routingChangesObserver.replicaPromoted(activeReplica);
-        if (IndexMetaData.isIndexUsingShadowReplicas(indexMetaData.getSettings())) {
+        if (indexMetaData.isIndexUsingShadowReplicas()) {
             ShardRouting initializedShard = reinitShadowPrimary(primarySwappedCandidate);
             routingChangesObserver.startedPrimaryReinitialized(primarySwappedCandidate, initializedShard);
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
index 051fd12a12b..0b1a0044567 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
@@ -56,8 +56,6 @@ import java.util.function.Predicate;
  */
 public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<RoutingTable> {
 
-    public static RoutingTable PROTO = builder().build();
-
     public static final RoutingTable EMPTY_ROUTING_TABLE = builder().build();
 
     private final long version;
@@ -349,18 +347,16 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
         return new RoutingTableDiff(previousState, this);
     }
 
-    @Override
-    public Diff<RoutingTable> readDiffFrom(StreamInput in) throws IOException {
+    public static Diff<RoutingTable> readDiffFrom(StreamInput in) throws IOException {
         return new RoutingTableDiff(in);
     }
 
-    @Override
-    public RoutingTable readFrom(StreamInput in) throws IOException {
+    public static RoutingTable readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder();
         builder.version = in.readLong();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            IndexRoutingTable index = IndexRoutingTable.Builder.readFrom(in);
+            IndexRoutingTable index = IndexRoutingTable.readFrom(in);
             builder.add(index);
         }
 
@@ -382,14 +378,15 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
 
         private final Diff<ImmutableOpenMap<String, IndexRoutingTable>> indicesRouting;
 
-        public RoutingTableDiff(RoutingTable before, RoutingTable after) {
+        RoutingTableDiff(RoutingTable before, RoutingTable after) {
             version = after.version;
             indicesRouting = DiffableUtils.diff(before.indicesRouting, after.indicesRouting, DiffableUtils.getStringKeySerializer());
         }
 
-        public RoutingTableDiff(StreamInput in) throws IOException {
+        RoutingTableDiff(StreamInput in) throws IOException {
             version = in.readLong();
-            indicesRouting = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexRoutingTable.PROTO);
+            indicesRouting = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), IndexRoutingTable::readFrom,
+                IndexRoutingTable::readDiffFrom);
         }
 
         @Override
@@ -607,10 +604,6 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
             indicesRouting = null;
             return table;
         }
-
-        public static RoutingTable readFrom(StreamInput in) throws IOException {
-            return PROTO.readFrom(in);
-        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
index 586b37846cf..7ee17c558f7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AbstractAllocationDecision.java
@@ -32,6 +32,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 /**
@@ -139,7 +140,7 @@ public abstract class AbstractAllocationDecision implements ToXContent, Writeabl
     public XContentBuilder nodeDecisionsToXContent(List<NodeAllocationResult> nodeDecisions, XContentBuilder builder, Params params)
         throws IOException {
 
-        if (nodeDecisions != null) {
+        if (nodeDecisions != null && nodeDecisions.isEmpty() == false) {
             builder.startArray("node_allocation_decisions");
             {
                 for (NodeAllocationResult explanation : nodeDecisions) {
@@ -166,4 +167,21 @@ public abstract class AbstractAllocationDecision implements ToXContent, Writeabl
         return false;
     }
 
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null || other instanceof AbstractAllocationDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") AbstractAllocationDecision that = (AbstractAllocationDecision) other;
+        return Objects.equals(targetNode, that.targetNode) && Objects.equals(nodeDecisions, that.nodeDecisions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(targetNode, nodeDecisions);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
index 1100d4d70d3..49b9604e345 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java
@@ -174,7 +174,7 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision {
 
     @Override
     public boolean isDecisionTaken() {
-        return this != NOT_TAKEN;
+        return allocationStatus != AllocationStatus.NO_ATTEMPT;
     }
 
     /**
@@ -238,35 +238,41 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision {
     @Override
     public String getExplanation() {
         checkDecisionState();
-        String explanation;
-        if (allocationStatus == null) {
-            explanation = "can allocate the shard";
-        } else if (allocationStatus == AllocationStatus.DECIDERS_THROTTLED) {
-            explanation = "allocation temporarily throttled";
-        } else if (allocationStatus == AllocationStatus.FETCHING_SHARD_DATA) {
-            explanation = "cannot allocate because information about existing shard data is still being retrieved from " +
-                              "some of the nodes";
-        } else if (allocationStatus == AllocationStatus.NO_VALID_SHARD_COPY) {
-            if (getNodeDecisions() != null && getNodeDecisions().size() > 0) {
-                explanation = "cannot allocate because all existing copies of the shard are unreadable";
+        AllocationDecision allocationDecision = getAllocationDecision();
+        if (allocationDecision == AllocationDecision.YES) {
+            return "can allocate the shard";
+        } else if (allocationDecision == AllocationDecision.THROTTLED) {
+            return "allocation temporarily throttled";
+        } else if (allocationDecision == AllocationDecision.AWAITING_INFO) {
+            return "cannot allocate because information about existing shard data is still being retrieved from some of the nodes";
+        } else if (allocationDecision == AllocationDecision.NO_VALID_SHARD_COPY) {
+            if (hasNodeWithStaleOrCorruptShard()) {
+                return "cannot allocate because all found copies of the shard are either stale or corrupt";
             } else {
-                explanation = "cannot allocate because a previous copy of the shard existed but could not be found";
+                return "cannot allocate because a previous copy of the primary shard existed but can no longer be found on " +
+                       "the nodes in the cluster";
             }
-        } else if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
-            explanation = "cannot allocate because the cluster is still waiting " +
+        } else if (allocationDecision == AllocationDecision.ALLOCATION_DELAYED) {
+            return "cannot allocate because the cluster is still waiting " +
                               TimeValue.timeValueMillis(remainingDelayInMillis) +
                               " for the departed node holding a replica to rejoin" +
                               (atLeastOneNodeWithYesDecision() ?
                                    ", despite being allowed to allocate the shard to at least one other node" : "");
         } else {
-            assert allocationStatus == AllocationStatus.DECIDERS_NO;
+            assert allocationDecision == AllocationDecision.NO;
             if (reuseStore) {
-                explanation = "cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy";
+                return "cannot allocate because allocation is not permitted to any of the nodes that hold an in-sync shard copy";
             } else {
-                explanation = "cannot allocate because allocation is not permitted to any of the nodes";
+                return "cannot allocate because allocation is not permitted to any of the nodes";
             }
         }
-        return explanation;
+    }
+
+    private boolean hasNodeWithStaleOrCorruptShard() {
+        return getNodeDecisions() != null && getNodeDecisions().stream().anyMatch(result ->
+                result.getShardStoreInfo() != null
+                    && (result.getShardStoreInfo().getAllocationId() != null
+                            || result.getShardStoreInfo().getStoreException() != null));
     }
 
     @Override
@@ -300,4 +306,26 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision {
         out.writeVLong(configuredDelayInMillis);
     }
 
+    @Override
+    public boolean equals(Object other) {
+        if (super.equals(other) == false) {
+            return false;
+        }
+        if (other instanceof AllocateUnassignedDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") AllocateUnassignedDecision that = (AllocateUnassignedDecision) other;
+        return Objects.equals(allocationStatus, that.allocationStatus)
+                   && Objects.equals(allocationId, that.allocationId)
+                   && reuseStore == that.reuseStore
+                   && configuredDelayInMillis == that.configuredDelayInMillis
+                   && remainingDelayInMillis == that.remainingDelayInMillis;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Objects.hash(allocationStatus, allocationId, reuseStore,
+            configuredDelayInMillis, remainingDelayInMillis);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java
index 5a49cfb142e..0fe9549635e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationDecision.java
@@ -40,7 +40,7 @@ public enum AllocationDecision implements Writeable {
     /**
      * The allocation attempt was throttled for the shard.
      */
-    THROTTLE((byte) 1),
+    THROTTLED((byte) 1),
     /**
      * The shard cannot be allocated, which can happen for any number of reasons,
      * including the allocation deciders gave a NO decision for allocating.
@@ -56,12 +56,12 @@ public enum AllocationDecision implements Writeable {
      * Waiting on getting shard data from all nodes before making a decision
      * about where to allocate the shard.
      */
-    FETCH_PENDING((byte) 4),
+    AWAITING_INFO((byte) 4),
     /**
      * The allocation decision has been delayed waiting for a replica with a shard copy
      * that left the cluster to rejoin.
      */
-    DELAYED_ALLOCATION((byte) 5),
+    ALLOCATION_DELAYED((byte) 5),
     /**
      * The shard was denied allocation because there were no valid shard copies
      * found for it amongst the nodes in the cluster.
@@ -90,15 +90,15 @@ public enum AllocationDecision implements Writeable {
             case 0:
                 return YES;
             case 1:
-                return THROTTLE;
+                return THROTTLED;
             case 2:
                 return NO;
             case 3:
                 return WORSE_BALANCE;
             case 4:
-                return FETCH_PENDING;
+                return AWAITING_INFO;
             case 5:
-                return DELAYED_ALLOCATION;
+                return ALLOCATION_DELAYED;
             case 6:
                 return NO_VALID_SHARD_COPY;
             case 7:
@@ -117,11 +117,11 @@ public enum AllocationDecision implements Writeable {
         } else {
             switch (allocationStatus) {
                 case DECIDERS_THROTTLED:
-                    return THROTTLE;
+                    return THROTTLED;
                 case FETCHING_SHARD_DATA:
-                    return FETCH_PENDING;
+                    return AWAITING_INFO;
                 case DELAYED_ALLOCATION:
-                    return DELAYED_ALLOCATION;
+                    return ALLOCATION_DELAYED;
                 case NO_VALID_SHARD_COPY:
                     return NO_VALID_SHARD_COPY;
                 case NO_ATTEMPT:
@@ -141,7 +141,7 @@ public enum AllocationDecision implements Writeable {
             case YES:
                 return YES;
             case THROTTLE:
-                return THROTTLE;
+                return THROTTLED;
             default:
                 assert type == Decision.Type.NO;
                 return NO;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
index 676e6107cb2..de9795ff4c2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Represents a decision to move a started shard, either because it is no longer allowed to remain on its current node
@@ -48,15 +49,15 @@ public final class MoveDecision extends AbstractAllocationDecision {
     @Nullable
     private final Decision canRemainDecision;
     @Nullable
-    private final Decision canRebalanceDecision;
+    private final Decision clusterRebalanceDecision;
     private final int currentNodeRanking;
 
-    private MoveDecision(Decision canRemainDecision, Decision canRebalanceDecision, AllocationDecision allocationDecision,
+    private MoveDecision(Decision canRemainDecision, Decision clusterRebalanceDecision, AllocationDecision allocationDecision,
                          DiscoveryNode assignedNode, List<NodeAllocationResult> nodeDecisions, int currentNodeRanking) {
         super(assignedNode, nodeDecisions);
         this.allocationDecision = allocationDecision;
         this.canRemainDecision = canRemainDecision;
-        this.canRebalanceDecision = canRebalanceDecision;
+        this.clusterRebalanceDecision = clusterRebalanceDecision;
         this.currentNodeRanking = currentNodeRanking;
     }
 
@@ -64,7 +65,7 @@ public final class MoveDecision extends AbstractAllocationDecision {
         super(in);
         allocationDecision = in.readOptionalWriteable(AllocationDecision::readFrom);
         canRemainDecision = in.readOptionalWriteable(Decision::readFrom);
-        canRebalanceDecision = in.readOptionalWriteable(Decision::readFrom);
+        clusterRebalanceDecision = in.readOptionalWriteable(Decision::readFrom);
         currentNodeRanking = in.readVInt();
     }
 
@@ -73,7 +74,7 @@ public final class MoveDecision extends AbstractAllocationDecision {
         super.writeTo(out);
         out.writeOptionalWriteable(allocationDecision);
         out.writeOptionalWriteable(canRemainDecision);
-        out.writeOptionalWriteable(canRebalanceDecision);
+        out.writeOptionalWriteable(clusterRebalanceDecision);
         out.writeVInt(currentNodeRanking);
     }
 
@@ -131,7 +132,15 @@ public final class MoveDecision extends AbstractAllocationDecision {
 
     @Override
     public boolean isDecisionTaken() {
-        return this != NOT_TAKEN;
+        return canRemainDecision != null || clusterRebalanceDecision != null;
+    }
+
+    /**
+     * Creates a new move decision from this decision, plus adding a remain decision.
+     */
+    public MoveDecision withRemainDecision(Decision canRemainDecision) {
+        return new MoveDecision(canRemainDecision, clusterRebalanceDecision, allocationDecision,
+                                   targetNode, nodeDecisions, currentNodeRanking);
     }
 
     /**
@@ -164,13 +173,13 @@ public final class MoveDecision extends AbstractAllocationDecision {
 
     /**
      * Returns {@code true} if the shard is allowed to be rebalanced to another node in the cluster,
-     * returns {@code false} otherwise.  If {@link #getCanRebalanceDecision()} returns {@code null}, then
+     * returns {@code false} otherwise.  If {@link #getClusterRebalanceDecision()} returns {@code null}, then
      * the result of this method is meaningless, as no rebalance decision was taken.  If {@link #isDecisionTaken()}
      * returns {@code false}, then invoking this method will throw an {@code IllegalStateException}.
      */
-    public boolean canRebalance() {
+    public boolean canRebalanceCluster() {
         checkDecisionState();
-        return canRebalanceDecision.type() == Type.YES;
+        return clusterRebalanceDecision != null && clusterRebalanceDecision.type() == Type.YES;
     }
 
     /**
@@ -182,9 +191,9 @@ public final class MoveDecision extends AbstractAllocationDecision {
      * {@code IllegalStateException}.
      */
     @Nullable
-    public Decision getCanRebalanceDecision() {
+    public Decision getClusterRebalanceDecision() {
         checkDecisionState();
-        return canRebalanceDecision;
+        return clusterRebalanceDecision;
     }
 
     /**
@@ -199,7 +208,7 @@ public final class MoveDecision extends AbstractAllocationDecision {
     /**
      * Gets the current ranking of the node to which the shard is currently assigned, relative to the
      * other nodes in the cluster as reported in {@link NodeAllocationResult#getWeightRanking()}.  The
-     * ranking will only return a meaningful positive integer if {@link #getCanRebalanceDecision()} returns
+     * ranking will only return a meaningful positive integer if {@link #getClusterRebalanceDecision()} returns
      * a non-null value; otherwise, 0 will be returned.  If {@link #isDecisionTaken()} returns
      * {@code false}, then invoking this method will throw an {@code IllegalStateException}.
      */
@@ -212,18 +221,19 @@ public final class MoveDecision extends AbstractAllocationDecision {
     public String getExplanation() {
         checkDecisionState();
         String explanation;
-        if (canRebalanceDecision != null) {
+        if (clusterRebalanceDecision != null) {
             // it was a decision to rebalance the shard, because the shard was allowed to remain on its current node
-            if (allocationDecision == AllocationDecision.FETCH_PENDING) {
+            if (allocationDecision == AllocationDecision.AWAITING_INFO) {
                 explanation = "cannot rebalance as information about existing copies of this shard in the cluster is still being gathered";
-            } else if (canRebalanceDecision.type() == Type.NO) {
-                explanation = "rebalancing is not allowed on the cluster" + (atLeastOneNodeWithYesDecision() ? ", even though there " +
+            } else if (clusterRebalanceDecision.type() == Type.NO) {
+                explanation = "rebalancing is not allowed" + (atLeastOneNodeWithYesDecision() ? ", even though there " +
                               "is at least one node on which the shard can be allocated" : "");
-            } else if (canRebalanceDecision.type() == Type.THROTTLE) {
+            } else if (clusterRebalanceDecision.type() == Type.THROTTLE) {
                 explanation = "rebalancing is throttled";
             } else {
+                assert clusterRebalanceDecision.type() == Type.YES;
                 if (getTargetNode() != null) {
-                    if (allocationDecision == AllocationDecision.THROTTLE) {
+                    if (allocationDecision == AllocationDecision.THROTTLED) {
                         explanation = "shard rebalancing throttled";
                     } else {
                         explanation = "can rebalance shard";
@@ -235,11 +245,10 @@ public final class MoveDecision extends AbstractAllocationDecision {
             }
         } else {
             // it was a decision to force move the shard
-            if (canRemain()) {
-                explanation = "shard can remain on its current node";
-            } else if (allocationDecision == AllocationDecision.YES) {
+            assert canRemain() == false;
+            if (allocationDecision == AllocationDecision.YES) {
                 explanation = "shard cannot remain on this node and is force-moved to another node";
-            } else if (allocationDecision == AllocationDecision.THROTTLE) {
+            } else if (allocationDecision == AllocationDecision.THROTTLED) {
                 explanation = "shard cannot remain on this node but is throttled on moving to another node";
             } else {
                 assert allocationDecision == AllocationDecision.NO;
@@ -263,23 +272,44 @@ public final class MoveDecision extends AbstractAllocationDecision {
             canRemainDecision.toXContent(builder, params);
             builder.endArray();
         }
-        if (canRebalanceDecision != null) {
-            AllocationDecision rebalanceDecision = AllocationDecision.fromDecisionType(canRebalanceDecision.type());
+        if (clusterRebalanceDecision != null) {
+            AllocationDecision rebalanceDecision = AllocationDecision.fromDecisionType(clusterRebalanceDecision.type());
             builder.field("can_rebalance_cluster", rebalanceDecision);
-            if (rebalanceDecision != AllocationDecision.YES && canRebalanceDecision.getDecisions().isEmpty() == false) {
+            if (rebalanceDecision != AllocationDecision.YES && clusterRebalanceDecision.getDecisions().isEmpty() == false) {
                 builder.startArray("can_rebalance_cluster_decisions");
-                canRebalanceDecision.toXContent(builder, params);
+                clusterRebalanceDecision.toXContent(builder, params);
                 builder.endArray();
             }
         }
-        if (canRebalanceDecision != null) {
+        if (clusterRebalanceDecision != null) {
             builder.field("can_rebalance_to_other_node", allocationDecision);
+            builder.field("rebalance_explanation", getExplanation());
         } else {
             builder.field("can_move_to_other_node", forceMove() ? "yes" : "no");
+            builder.field("move_explanation", getExplanation());
         }
-        builder.field(canRebalanceDecision != null ? "rebalance_explanation" : "move_explanation", getExplanation());
         nodeDecisionsToXContent(nodeDecisions, builder, params);
         return builder;
     }
 
+    @Override
+    public boolean equals(Object other) {
+        if (super.equals(other) == false) {
+            return false;
+        }
+        if (other instanceof MoveDecision == false) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") MoveDecision that = (MoveDecision) other;
+        return Objects.equals(allocationDecision, that.allocationDecision)
+                   && Objects.equals(canRemainDecision, that.canRemainDecision)
+                   && Objects.equals(clusterRebalanceDecision, that.clusterRebalanceDecision)
+                   && currentNodeRanking == that.currentNodeRanking;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Objects.hash(allocationDecision, canRemainDecision, clusterRebalanceDecision, currentNodeRanking);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java
index 701bb66f8d9..3740ded3060 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationResult.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
@@ -48,14 +49,15 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
     @Nullable
     private final ShardStoreInfo shardStoreInfo;
     private final AllocationDecision nodeDecision;
+    @Nullable
     private final Decision canAllocateDecision;
     private final int weightRanking;
 
-    public NodeAllocationResult(DiscoveryNode node, ShardStoreInfo shardStoreInfo, Decision decision) {
+    public NodeAllocationResult(DiscoveryNode node, ShardStoreInfo shardStoreInfo, @Nullable Decision decision) {
         this.node = node;
         this.shardStoreInfo = shardStoreInfo;
         this.canAllocateDecision = decision;
-        this.nodeDecision = AllocationDecision.fromDecisionType(canAllocateDecision.type());
+        this.nodeDecision = decision != null ? AllocationDecision.fromDecisionType(canAllocateDecision.type()) : AllocationDecision.NO;
         this.weightRanking = 0;
     }
 
@@ -78,7 +80,11 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
     public NodeAllocationResult(StreamInput in) throws IOException {
         node = new DiscoveryNode(in);
         shardStoreInfo = in.readOptionalWriteable(ShardStoreInfo::new);
-        canAllocateDecision = Decision.readFrom(in);
+        if (in.getVersion().before(Version.V_5_2_1_UNRELEASED)) {
+            canAllocateDecision = Decision.readFrom(in);
+        } else {
+            canAllocateDecision = in.readOptionalWriteable(Decision::readFrom);
+        }
         nodeDecision = AllocationDecision.readFrom(in);
         weightRanking = in.readVInt();
     }
@@ -87,7 +93,15 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
     public void writeTo(StreamOutput out) throws IOException {
         node.writeTo(out);
         out.writeOptionalWriteable(shardStoreInfo);
-        canAllocateDecision.writeTo(out);
+        if (out.getVersion().before(Version.V_5_2_1_UNRELEASED)) {
+            if (canAllocateDecision == null) {
+                Decision.NO.writeTo(out);
+            } else {
+                canAllocateDecision.writeTo(out);
+            }
+        } else {
+            out.writeOptionalWriteable(canAllocateDecision);
+        }
         nodeDecision.writeTo(out);
         out.writeVInt(weightRanking);
     }
@@ -108,8 +122,11 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
     }
 
     /**
-     * The decision details for allocating to this node.
+     * The decision details for allocating to this node.  Returns {@code null} if
+     * no allocation decision was taken on the node; in this case, {@link #getNodeDecision()}
+     * will return {@link AllocationDecision#NO}.
      */
+    @Nullable
     public Decision getCanAllocateDecision() {
         return canAllocateDecision;
     }
@@ -153,7 +170,7 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
             if (isWeightRanked()) {
                 builder.field("weight_ranking", getWeightRanking());
             }
-            if (canAllocateDecision.getDecisions().isEmpty() == false) {
+            if (canAllocateDecision != null && canAllocateDecision.getDecisions().isEmpty() == false) {
                 builder.startArray("deciders");
                 canAllocateDecision.toXContent(builder, params);
                 builder.endArray();
@@ -234,6 +251,15 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
             return matchingBytes;
         }
 
+        /**
+         * Gets the store exception when trying to read the store, if there was an error.  If
+         * there was no error, returns {@code null}.
+         */
+        @Nullable
+        public Exception getStoreException() {
+            return storeException;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(inSync);
@@ -248,7 +274,12 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
             {
                 if (matchingBytes < 0) {
                     // dealing with a primary shard
-                    builder.field("in_sync", inSync);
+                    if (allocationId == null && storeException == null) {
+                        // there was no information we could obtain of any shard data on the node
+                        builder.field("found", false);
+                    } else {
+                        builder.field("in_sync", inSync);
+                    }
                 }
                 if (allocationId != null) {
                     builder.field("allocation_id", allocationId);
@@ -262,7 +293,7 @@ public class NodeAllocationResult implements ToXContent, Writeable, Comparable<N
                 }
                 if (storeException != null) {
                     builder.startObject("store_exception");
-                    ElasticsearchException.toXContent(builder, params, storeException);
+                    ElasticsearchException.generateThrowableXContent(builder, params, storeException);
                     builder.endObject();
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 1899ea1cc55..e1ae367bebf 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -68,7 +68,7 @@ public class RoutingAllocation {
 
     private final boolean retryFailed;
 
-    private boolean debugDecision = false;
+    private DebugMode debugDecision = DebugMode.OFF;
 
     private boolean hasPendingAsyncFetch = false;
 
@@ -167,11 +167,19 @@ public class RoutingAllocation {
         return this.ignoreDisable;
     }
 
-    public void debugDecision(boolean debug) {
+    public void setDebugMode(DebugMode debug) {
         this.debugDecision = debug;
     }
 
+    public void debugDecision(boolean debug) {
+        this.debugDecision = debug ? DebugMode.ON : DebugMode.OFF;
+    }
+
     public boolean debugDecision() {
+        return this.debugDecision != DebugMode.OFF;
+    }
+
+    public DebugMode getDebugMode() {
         return this.debugDecision;
     }
 
@@ -280,4 +288,20 @@ public class RoutingAllocation {
     public boolean isRetryFailed() {
         return retryFailed;
     }
+
+    public enum DebugMode {
+        /**
+         * debug mode is off
+         */
+        OFF,
+        /**
+         * debug mode is on
+         */
+        ON,
+        /**
+         * debug mode is on, but YES decisions from a {@link org.elasticsearch.cluster.routing.allocation.decider.Decision.Multi}
+         * are not included.
+         */
+        EXCLUDE_YES_DECISIONS
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java
new file mode 100644
index 00000000000..557ce9300c6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/ShardAllocationDecision.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Represents the decision taken for the allocation of a single shard.  If
+ * the shard is unassigned, {@link #getAllocateDecision()} will return an
+ * object containing the decision and its explanation, and {@link #getMoveDecision()}
+ * will return an object for which {@link MoveDecision#isDecisionTaken()} returns
+ * {@code false}.  If the shard is in the started state, then {@link #getMoveDecision()}
+ * will return an object containing the decision to move/rebalance the shard, and
+ * {@link #getAllocateDecision()} will return an object for which
+ * {@link AllocateUnassignedDecision#isDecisionTaken()} returns {@code false}.  If
+ * the shard is neither unassigned nor started (i.e. it is initializing or relocating),
+ * then both {@link #getAllocateDecision()} and {@link #getMoveDecision()} will return
+ * objects whose {@code isDecisionTaken()} method returns {@code false}.
+ */
+public final class ShardAllocationDecision implements ToXContent, Writeable {
+    public static final ShardAllocationDecision NOT_TAKEN =
+        new ShardAllocationDecision(AllocateUnassignedDecision.NOT_TAKEN, MoveDecision.NOT_TAKEN);
+
+    private final AllocateUnassignedDecision allocateDecision;
+    private final MoveDecision moveDecision;
+
+    public ShardAllocationDecision(AllocateUnassignedDecision allocateDecision,
+                                   MoveDecision moveDecision) {
+        this.allocateDecision = allocateDecision;
+        this.moveDecision = moveDecision;
+    }
+
+    public ShardAllocationDecision(StreamInput in) throws IOException {
+        allocateDecision = new AllocateUnassignedDecision(in);
+        moveDecision = new MoveDecision(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        allocateDecision.writeTo(out);
+        moveDecision.writeTo(out);
+    }
+
+    /**
+     * Returns {@code true} if either an allocation decision or a move decision was taken
+     * for the shard.  If no decision was taken, as in the case of initializing or relocating
+     * shards, then this method returns {@code false}.
+     */
+    public boolean isDecisionTaken() {
+        return allocateDecision.isDecisionTaken() || moveDecision.isDecisionTaken();
+    }
+
+    /**
+     * Gets the unassigned allocation decision for the shard.  If the shard was not in the unassigned state,
+     * the instance of {@link AllocateUnassignedDecision} that is returned will have {@link AllocateUnassignedDecision#isDecisionTaken()}
+     * return {@code false}.
+     */
+    public AllocateUnassignedDecision getAllocateDecision() {
+        return allocateDecision;
+    }
+
+    /**
+     * Gets the move decision for the shard.  If the shard was not in the started state,
+     * the instance of {@link MoveDecision} that is returned will have {@link MoveDecision#isDecisionTaken()}
+     * return {@code false}.
+     */
+    public MoveDecision getMoveDecision() {
+        return moveDecision;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (allocateDecision.isDecisionTaken()) {
+            allocateDecision.toXContent(builder, params);
+        }
+        if (moveDecision.isDecisionTaken()) {
+            moveDecision.toXContent(builder, params);
+        }
+        return builder;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 46a2a2524cb..a60287a8487 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
@@ -46,6 +47,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.PriorityComparator;
 
 import java.util.ArrayList;
@@ -110,12 +112,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         this.threshold = threshold;
     }
 
-    @Override
-    public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-        final Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
-        return balancer.weighShard(shard);
-    }
-
     @Override
     public void allocate(RoutingAllocation allocation) {
         if (allocation.routingNodes().size() == 0) {
@@ -128,16 +124,21 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         balancer.balance();
     }
 
-    /**
-     * Returns a decision on rebalancing a single shard to form a more optimal cluster balance.  This
-     * method is not used in itself for cluster rebalancing because all shards from all indices are
-     * taken into account when making rebalancing decisions.  This method is only intended to be used
-     * from the cluster allocation explain API to explain possible rebalancing decisions for a single
-     * shard.
-     */
-    public MoveDecision decideRebalance(final ShardRouting shard, final RoutingAllocation allocation) {
-        assert allocation.debugDecision() : "debugDecision should be set in explain mode";
-        return new Balancer(logger, allocation, weightFunction, threshold).decideRebalance(shard);
+    @Override
+    public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, final RoutingAllocation allocation) {
+        Balancer balancer = new Balancer(logger, allocation, weightFunction, threshold);
+        AllocateUnassignedDecision allocateUnassignedDecision = AllocateUnassignedDecision.NOT_TAKEN;
+        MoveDecision moveDecision = MoveDecision.NOT_TAKEN;
+        if (shard.unassigned()) {
+            allocateUnassignedDecision = balancer.decideAllocateUnassigned(shard, Sets.newHashSet());
+        } else {
+            moveDecision = balancer.decideMove(shard);
+            if (moveDecision.isDecisionTaken() && moveDecision.canRemain()) {
+                MoveDecision rebalanceDecision = balancer.decideRebalance(shard);
+                moveDecision = rebalanceDecision.withRemainDecision(moveDecision.getCanRemainDecision());
+            }
+        }
+        return new ShardAllocationDecision(allocateUnassignedDecision, moveDecision);
     }
 
     /**
@@ -337,7 +338,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          */
         private MoveDecision decideRebalance(final ShardRouting shard) {
             if (shard.started() == false) {
-                // cannot rebalance a shard that isn't started
+                // we can only rebalance started shards
                 return MoveDecision.NOT_TAKEN;
             }
 
@@ -437,7 +438,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             }
 
             if (canRebalance.type() != Type.YES || allocation.hasPendingAsyncFetch()) {
-                AllocationDecision allocationDecision = allocation.hasPendingAsyncFetch() ? AllocationDecision.FETCH_PENDING :
+                AllocationDecision allocationDecision = allocation.hasPendingAsyncFetch() ? AllocationDecision.AWAITING_INFO :
                                                             AllocationDecision.fromDecisionType(canRebalance.type());
                 return MoveDecision.cannotRebalance(canRebalance, allocationDecision, currentNodeWeightRanking, nodeDecisions);
             } else {
@@ -644,7 +645,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             // offloading the shards.
             for (Iterator<ShardRouting> it = allocation.routingNodes().nodeInterleavedShardIterator(); it.hasNext(); ) {
                 ShardRouting shardRouting = it.next();
-                final MoveDecision moveDecision = makeMoveDecision(shardRouting);
+                final MoveDecision moveDecision = decideMove(shardRouting);
                 if (moveDecision.isDecisionTaken() && moveDecision.forceMove()) {
                     final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
                     final ModelNode targetNode = nodes.get(moveDecision.getTargetNode().getId());
@@ -673,7 +674,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
          *   4. If the method is invoked in explain mode (e.g. from the cluster allocation explain APIs), then
          *      {@link MoveDecision#nodeDecisions} will have a non-null value.
          */
-        public MoveDecision makeMoveDecision(final ShardRouting shardRouting) {
+        public MoveDecision decideMove(final ShardRouting shardRouting) {
             if (shardRouting.started() == false) {
                 // we can only move started shards
                 return MoveDecision.NOT_TAKEN;
@@ -1051,7 +1052,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         private int numShards = 0;
         private final RoutingNode routingNode;
 
-        public ModelNode(RoutingNode routingNode) {
+        ModelNode(RoutingNode routingNode) {
             this.routingNode = routingNode;
         }
 
@@ -1129,7 +1130,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         private final Set<ShardRouting> shards = new HashSet<>(4); // expect few shards of same index to be allocated on same node
         private int highestPrimary = -1;
 
-        public ModelIndex(String id) {
+        ModelIndex(String id) {
             this.id = id;
         }
 
@@ -1186,7 +1187,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
         private final Balancer balancer;
         private float pivotWeight;
 
-        public NodeSorter(ModelNode[] modelNodes, WeightFunction function, Balancer balancer) {
+        NodeSorter(ModelNode[] modelNodes, WeightFunction function, Balancer balancer) {
             this.function = function;
             this.balancer = balancer;
             this.modelNodes = modelNodes;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
index 35f3b265418..7e9d15b4528 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocator.java
@@ -19,11 +19,12 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 
-import java.util.Map;
 /**
  * <p>
  * A {@link ShardsAllocator} is the main entry point for shard allocation on nodes in the cluster.
@@ -44,13 +45,17 @@ public interface ShardsAllocator {
     void allocate(RoutingAllocation allocation);
 
     /**
-     * Returns a map of node to a float "weight" of where the allocator would like to place the shard.
-     * Higher weights signify greater desire to place the shard on that node.
-     * Does not modify the allocation at all.
+     * Returns the decision for where a shard should reside in the cluster.  If the shard is unassigned,
+     * then the {@link AllocateUnassignedDecision} will be non-null.  If the shard is not in the unassigned
+     * state, then the {@link MoveDecision} will be non-null.
      *
-     * @param allocation current node allocation
-     * @param shard shard to weigh
-     * @return map of nodes to float weights
+     * This method is primarily used by the cluster allocation explain API to provide detailed explanations
+     * for the allocation of a single shard.  Implementations of the {@link #allocate(RoutingAllocation)} method
+     * may use the results of this method implementation to decide on allocating shards in the routing table
+     * to the cluster.
+     *
+     * If an implementation of this interface does not support explaining decisions for a single shard through
+     * the cluster explain API, then this method should throw a {@code UnsupportedOperationException}.
      */
-    Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard);
+    ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation);
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
index 90a591b1199..4ffd70aee1c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
@@ -30,7 +30,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -50,8 +49,8 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
     private static final String SHARD_FIELD = "shard";
     private static final String NODE_FIELD = "node";
 
-    protected static <T extends Builder<?>> ObjectParser<T, ParseFieldMatcherSupplier> createAllocateParser(String command) {
-        ObjectParser<T, ParseFieldMatcherSupplier> parser = new ObjectParser<>(command);
+    protected static <T extends Builder<?>> ObjectParser<T, Void> createAllocateParser(String command) {
+        ObjectParser<T, Void> parser = new ObjectParser<>(command);
         parser.declareString(Builder::setIndex, new ParseField(INDEX_FIELD));
         parser.declareInt(Builder::setShard, new ParseField(SHARD_FIELD));
         parser.declareString(Builder::setNode, new ParseField(NODE_FIELD));
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index 82d6f436d2a..157acc0e537 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -30,8 +30,6 @@ import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -49,8 +47,7 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
     public static final String NAME = "allocate_empty_primary";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> EMPTY_PRIMARY_PARSER = BasePrimaryAllocationCommand
-            .createAllocatePrimaryParser(NAME);
+    private static final ObjectParser<Builder, Void> EMPTY_PRIMARY_PARSER = BasePrimaryAllocationCommand.createAllocatePrimaryParser(NAME);
 
     /**
      * Creates a new {@link AllocateEmptyPrimaryAllocationCommand}
@@ -83,7 +80,7 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return EMPTY_PRIMARY_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return EMPTY_PRIMARY_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
index 8c47deee66f..6ec09a9bbbb 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
@@ -28,8 +28,6 @@ import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -46,8 +44,7 @@ public class AllocateReplicaAllocationCommand extends AbstractAllocateAllocation
     public static final String NAME = "allocate_replica";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<AllocateReplicaAllocationCommand.Builder, ParseFieldMatcherSupplier> REPLICA_PARSER =
-            createAllocateParser(NAME);
+    private static final ObjectParser<AllocateReplicaAllocationCommand.Builder, Void> REPLICA_PARSER = createAllocateParser(NAME);
 
     /**
      * Creates a new {@link AllocateReplicaAllocationCommand}
@@ -80,7 +77,7 @@ public class AllocateReplicaAllocationCommand extends AbstractAllocateAllocation
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return REPLICA_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return REPLICA_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
index acdd5cae30b..c643fb5c948 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java
@@ -28,8 +28,6 @@ import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -46,8 +44,7 @@ public class AllocateStalePrimaryAllocationCommand extends BasePrimaryAllocation
     public static final String NAME = "allocate_stale_primary";
     public static final ParseField COMMAND_NAME_FIELD = new ParseField(NAME);
 
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> STALE_PRIMARY_PARSER = BasePrimaryAllocationCommand
-            .createAllocatePrimaryParser(NAME);
+    private static final ObjectParser<Builder, Void> STALE_PRIMARY_PARSER = BasePrimaryAllocationCommand.createAllocatePrimaryParser(NAME);
 
     /**
      * Creates a new {@link AllocateStalePrimaryAllocationCommand}
@@ -81,7 +78,7 @@ public class AllocateStalePrimaryAllocationCommand extends BasePrimaryAllocation
 
         @Override
         public Builder parse(XContentParser parser) throws IOException {
-            return STALE_PRIMARY_PARSER.parse(parser, this, () -> ParseFieldMatcher.STRICT);
+            return STALE_PRIMARY_PARSER.parse(parser, this, null);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index 10ba3f55944..5098b027f61 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -23,7 +23,6 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -126,12 +125,10 @@ public class AllocationCommands extends ToXContentToBytes {
      *     }
      * </pre>
      * @param parser {@link XContentParser} to read the commands from
-     * @param registry of allocation command parsers
      * @return {@link AllocationCommands} read
      * @throws IOException if something bad happens while reading the stream
      */
-    public static AllocationCommands fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher,
-            AllocationCommandRegistry registry) throws IOException {
+    public static AllocationCommands fromXContent(XContentParser parser) throws IOException {
         AllocationCommands commands = new AllocationCommands();
 
         XContentParser.Token token = parser.currentToken();
@@ -160,7 +157,7 @@ public class AllocationCommands extends ToXContentToBytes {
                 token = parser.nextToken();
                 String commandName = parser.currentName();
                 token = parser.nextToken();
-                commands.add(registry.lookup(commandName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(parser));
+                commands.add(parser.namedObject(AllocationCommand.class, commandName, null));
                 // move to the end object one
                 if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                     throw new ElasticsearchParseException("allocation command is malformed, done parsing a command, but didn't get END_OBJECT, got [{}] instead", token);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
index f4dc4fba4b8..2cb04260125 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -35,8 +34,8 @@ public abstract class BasePrimaryAllocationCommand extends AbstractAllocateAlloc
 
     private static final String ACCEPT_DATA_LOSS_FIELD = "accept_data_loss";
 
-    protected static <T extends Builder<?>> ObjectParser<T, ParseFieldMatcherSupplier> createAllocatePrimaryParser(String command) {
-        ObjectParser<T, ParseFieldMatcherSupplier> parser = AbstractAllocateAllocationCommand.createAllocateParser(command);
+    protected static <T extends Builder<?>> ObjectParser<T, Void> createAllocatePrimaryParser(String command) {
+        ObjectParser<T, Void> parser = AbstractAllocateAllocationCommand.createAllocateParser(command);
         parser.declareBoolean(Builder::setAcceptDataLoss, new ParseField(ACCEPT_DATA_LOSS_FIELD));
         return parser;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
index 986613e5a42..53e67ba25a4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java
@@ -23,13 +23,12 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
-import java.util.Set;
+
+import static org.elasticsearch.cluster.routing.allocation.RoutingAllocation.DebugMode.EXCLUDE_YES_DECISIONS;
 
 /**
  * A composite {@link AllocationDecider} combining the "decision" of multiple
@@ -56,7 +55,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -82,7 +82,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 // the assumption is that a decider that returns the static instance Decision#ALWAYS
                 // does not really implements canAllocate
                 ret.add(decision);
@@ -112,7 +113,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -131,7 +133,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -150,7 +153,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -169,7 +173,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -188,7 +193,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
@@ -216,7 +222,8 @@ public class AllocationDeciders extends AllocationDecider {
                 } else {
                     ret.add(decision);
                 }
-            } else if (decision != Decision.ALWAYS) {
+            } else if (decision != Decision.ALWAYS
+                        && (allocation.getDebugMode() != EXCLUDE_YES_DECISIONS || decision.type() != Decision.Type.YES)) {
                 ret.add(decision);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index 93c45e7832f..4160fd224aa 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -78,12 +78,12 @@ public class AwarenessAllocationDecider extends AllocationDecider {
     public static final String NAME = "awareness";
 
     public static final Setting<String[]> CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING =
-        new Setting<>("cluster.routing.allocation.awareness.attributes", "", Strings::splitStringByCommaToArray , Property.Dynamic,
+        new Setting<>("cluster.routing.allocation.awareness.attributes", "", s -> Strings.tokenizeToStringArray(s, ","), Property.Dynamic,
             Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING =
         Setting.groupSetting("cluster.routing.allocation.awareness.force.", Property.Dynamic, Property.NodeScope);
 
-    private String[] awarenessAttributes;
+    private volatile String[] awarenessAttributes;
 
     private volatile Map<String, String[]> forcedAwarenessAttributes;
 
@@ -125,7 +125,7 @@ public class AwarenessAllocationDecider extends AllocationDecider {
     private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) {
         if (awarenessAttributes.length == 0) {
             return allocation.decision(Decision.YES, NAME,
-                "allocation awareness is not enabled, set [%s] to enable it",
+                "allocation awareness is not enabled, set cluster setting [%s] to enable it",
                 CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey());
         }
 
@@ -135,7 +135,7 @@ public class AwarenessAllocationDecider extends AllocationDecider {
             // the node the shard exists on must be associated with an awareness attribute
             if (!node.node().getAttributes().containsKey(awarenessAttribute)) {
                 return allocation.decision(Decision.NO, NAME,
-                    "node does not contain the awareness attribute [%s]; required attributes [%s=%s]",
+                    "node does not contain the awareness attribute [%s]; required attributes cluster setting [%s=%s]",
                     awarenessAttribute, CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(),
                     allocation.debugDecision() ? Strings.arrayToCommaDelimitedString(awarenessAttributes) : null);
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
index 4e4fb58799b..281f6a603c3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java
@@ -120,13 +120,13 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider {
             // check if there are unassigned primaries.
             if ( allocation.routingNodes().hasUnassignedPrimaries() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned primary shards and [%s] is set to [%s]",
+                        "the cluster has unassigned primary shards and cluster setting [%s] is set to [%s]",
                         CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
             // check if there are initializing primaries that don't have a relocatingNodeId entry.
             if ( allocation.routingNodes().hasInactivePrimaries() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive primary shards and [%s] is set to [%s]",
+                        "the cluster has inactive primary shards and cluster setting [%s] is set to [%s]",
                         CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
 
@@ -136,14 +136,14 @@ public class ClusterRebalanceAllocationDecider extends AllocationDecider {
             // check if there are unassigned shards.
             if (allocation.routingNodes().hasUnassignedShards() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has unassigned shards and [%s] is set to [%s]",
+                        "the cluster has unassigned shards and cluster setting [%s] is set to [%s]",
                         CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
             // in case all indices are assigned, are there initializing shards which
             // are not relocating?
             if ( allocation.routingNodes().hasInactiveShards() ) {
                 return allocation.decision(Decision.NO, NAME,
-                        "the cluster has inactive shards and [%s] is set to [%s]",
+                        "the cluster has inactive shards and cluster setting [%s] is set to [%s]",
                         CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE, type);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
index 6ec123ddab3..63fbad59b92 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java
@@ -67,7 +67,7 @@ public class ConcurrentRebalanceAllocationDecider extends AllocationDecider {
         int relocatingShards = allocation.routingNodes().getRelocatingShardCount();
         if (relocatingShards >= clusterConcurrentRebalance) {
             return allocation.decision(Decision.THROTTLE, NAME,
-                    "reached the limit of concurrently rebalancing shards [%d], [%s=%d]",
+                    "reached the limit of concurrently rebalancing shards [%d], cluster setting [%s=%d]",
                     relocatingShards,
                     CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING.getKey(),
                     clusterConcurrentRebalance);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
index 34f612f6ac9..a2198ad90d9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/Decision.java
@@ -140,6 +140,12 @@ public abstract class Decision implements ToXContent, Writeable {
     @Nullable
     public abstract String label();
 
+    /**
+     * Get the explanation for this decision.
+     */
+    @Nullable
+    public abstract String getExplanation();
+
     /**
      * Return the list of all decisions that make up this decision
      */
@@ -200,6 +206,7 @@ public abstract class Decision implements ToXContent, Writeable {
         /**
          * Returns the explanation string, fully formatted.  Only formats the string once.
          */
+        @Override
         @Nullable
         public String getExplanation() {
             if (explanationString == null && explanation != null) {
@@ -301,6 +308,12 @@ public abstract class Decision implements ToXContent, Writeable {
             return null;
         }
 
+        @Override
+        @Nullable
+        public String getExplanation() {
+            throw new UnsupportedOperationException("multi-level decisions do not have an explanation");
+        }
+
         @Override
         public List<Decision> getDecisions() {
             return Collections.unmodifiableList(this.decisions);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 5eb1ae1751e..56663be1ef4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -138,7 +138,8 @@ public class DiskThresholdDecider extends AllocationDecider {
                             diskThresholdSettings.getFreeBytesThresholdLow(), freeBytes, node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                    "the node is above the low watermark [%s=%s], having less than the minimum required [%s] free space, actual free: [%s]",
+                    "the node is above the low watermark cluster setting [%s=%s], having less than the minimum required [%s] free " +
+                    "space, actual free: [%s]",
                     CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
                     diskThresholdSettings.getLowWatermarkRaw(),
                     diskThresholdSettings.getFreeBytesThresholdLow(), new ByteSizeValue(freeBytes));
@@ -162,8 +163,8 @@ public class DiskThresholdDecider extends AllocationDecider {
                             diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytes, node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                    "the node is above the high watermark [%s=%s], having less than the minimum required [%s] free space, " +
-                    "actual free: [%s]",
+                    "the node is above the high watermark cluster setting [%s=%s], having less than the minimum required [%s] free " +
+                    "space, actual free: [%s]",
                     CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                     diskThresholdSettings.getHighWatermarkRaw(),
                     diskThresholdSettings.getFreeBytesThresholdHigh(), new ByteSizeValue(freeBytes));
@@ -180,8 +181,8 @@ public class DiskThresholdDecider extends AllocationDecider {
                             Strings.format1Decimals(usedDiskPercentage, "%"), node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                    "the node is above the low watermark [%s=%s], using more disk space than the maximum allowed [%s%%], " +
-                    "actual free: [%s%%]",
+                    "the node is above the low watermark cluster setting [%s=%s], using more disk space than the maximum allowed " +
+                    "[%s%%], actual free: [%s%%]",
                     CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(),
                     diskThresholdSettings.getLowWatermarkRaw(), usedDiskThresholdLow, freeDiskPercentage);
             } else if (freeDiskPercentage > diskThresholdSettings.getFreeDiskThresholdHigh()) {
@@ -206,8 +207,8 @@ public class DiskThresholdDecider extends AllocationDecider {
                             Strings.format1Decimals(freeDiskPercentage, "%"), node.nodeId());
                 }
                 return allocation.decision(Decision.NO, NAME,
-                    "the node is above the high watermark [%s=%s], using more disk space than the maximum allowed [%s%%], " +
-                    "actual free: [%s%%]",
+                    "the node is above the high watermark cluster setting [%s=%s], using more disk space than the maximum allowed " +
+                    "[%s%%], actual free: [%s%%]",
                     CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                     diskThresholdSettings.getHighWatermarkRaw(), usedDiskThresholdHigh, freeDiskPercentage);
             }
@@ -222,7 +223,7 @@ public class DiskThresholdDecider extends AllocationDecider {
                     "{} free bytes threshold ({} bytes free), preventing allocation",
                     node.nodeId(), diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytesAfterShard);
             return allocation.decision(Decision.NO, NAME,
-                "allocating the shard to this node will bring the node above the high watermark [%s=%s] " +
+                "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " +
                     "and cause it to have less than the minimum required [%s] of free space (free bytes after shard added: [%s])",
                 CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                 diskThresholdSettings.getHighWatermarkRaw(),
@@ -234,7 +235,7 @@ public class DiskThresholdDecider extends AllocationDecider {
                     node.nodeId(), Strings.format1Decimals(diskThresholdSettings.getFreeDiskThresholdHigh(), "%"),
                                                            Strings.format1Decimals(freeSpaceAfterShard, "%"));
             return allocation.decision(Decision.NO, NAME,
-                "allocating the shard to this node will bring the node above the high watermark [%s=%s] " +
+                "allocating the shard to this node will bring the node above the high watermark cluster setting [%s=%s] " +
                     "and cause it to use more disk space than the maximum allowed [%s%%] (free space after shard added: [%s%%])",
                 CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                 diskThresholdSettings.getHighWatermarkRaw(), usedDiskThresholdHigh, freeSpaceAfterShard);
@@ -279,7 +280,7 @@ public class DiskThresholdDecider extends AllocationDecider {
                         diskThresholdSettings.getFreeBytesThresholdHigh(), freeBytes, node.nodeId());
             }
             return allocation.decision(Decision.NO, NAME,
-                "the shard cannot remain on this node because it is above the high watermark [%s=%s] " +
+                "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " +
                     "and there is less than the required [%s] free space on node, actual free: [%s]",
                 CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                 diskThresholdSettings.getHighWatermarkRaw(),
@@ -291,7 +292,7 @@ public class DiskThresholdDecider extends AllocationDecider {
                         diskThresholdSettings.getFreeDiskThresholdHigh(), freeDiskPercentage, node.nodeId());
             }
             return allocation.decision(Decision.NO, NAME,
-                "the shard cannot remain on this node because it is above the high watermark [%s=%s] " +
+                "the shard cannot remain on this node because it is above the high watermark cluster setting [%s=%s] " +
                     "and there is less than the required [%s%%] free disk on node, actual free: [%s%%]",
                 CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(),
                 diskThresholdSettings.getHighWatermarkRaw(),
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index b6995c6b972..7bb073a4c45 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -178,10 +178,12 @@ public class EnableAllocationDecider extends AllocationDecider {
     }
 
     private static String setting(Allocation allocation, boolean usedIndexSetting) {
-        StringBuilder buf = new StringBuilder("[");
+        StringBuilder buf = new StringBuilder();
         if (usedIndexSetting) {
+            buf.append("index setting [");
             buf.append(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey());
         } else {
+            buf.append("cluster setting [");
             buf.append(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey());
         }
         buf.append("=").append(allocation.toString().toLowerCase(Locale.ROOT)).append("]");
@@ -189,10 +191,12 @@ public class EnableAllocationDecider extends AllocationDecider {
     }
 
     private static String setting(Rebalance rebalance, boolean usedIndexSetting) {
-        StringBuilder buf = new StringBuilder("[");
+        StringBuilder buf = new StringBuilder();
         if (usedIndexSetting) {
+            buf.append("index setting [");
             buf.append(INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey());
         } else {
+            buf.append("cluster setting [");
             buf.append(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey());
         }
         buf.append("=").append(rebalance.toString().toLowerCase(Locale.ROOT)).append("]");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index 21b6b3d1354..85069392eb6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
+import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.IP_VALIDATOR;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND;
 import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR;
 
@@ -68,11 +69,11 @@ public class FilterAllocationDecider extends AllocationDecider {
     private static final String CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX = "cluster.routing.allocation.include";
     private static final String CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX = "cluster.routing.allocation.exclude";
     public static final Setting<Settings> CLUSTER_ROUTING_REQUIRE_GROUP_SETTING =
-        Setting.groupSetting(CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX + ".", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_INCLUDE_GROUP_SETTING =
-        Setting.groupSetting(CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX + ".", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
     public static final Setting<Settings> CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING =
-        Setting.groupSetting(CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX + ".", Property.Dynamic, Property.NodeScope);
+        Setting.groupSetting(CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX + ".", IP_VALIDATOR, Property.Dynamic, Property.NodeScope);
 
     private volatile DiscoveryNodeFilters clusterRequireFilters;
     private volatile DiscoveryNodeFilters clusterIncludeFilters;
@@ -141,19 +142,19 @@ public class FilterAllocationDecider extends AllocationDecider {
     private Decision shouldIndexFilter(IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) {
         if (indexMd.requireFilters() != null) {
             if (!indexMd.requireFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node does not match index setting [%s] filters [%s]",
                     IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX, indexMd.requireFilters());
             }
         }
         if (indexMd.includeFilters() != null) {
             if (!indexMd.includeFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node does not match index setting [%s] filters [%s]",
                     IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_PREFIX, indexMd.includeFilters());
             }
         }
         if (indexMd.excludeFilters() != null) {
             if (indexMd.excludeFilters().match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node matches [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node matches index setting [%s] filters [%s]",
                     IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey(), indexMd.excludeFilters());
             }
         }
@@ -163,19 +164,19 @@ public class FilterAllocationDecider extends AllocationDecider {
     private Decision shouldClusterFilter(RoutingNode node, RoutingAllocation allocation) {
         if (clusterRequireFilters != null) {
             if (!clusterRequireFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not match [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node does not match cluster setting [%s] filters [%s]",
                     CLUSTER_ROUTING_REQUIRE_GROUP_PREFIX, clusterRequireFilters);
             }
         }
         if (clusterIncludeFilters != null) {
             if (!clusterIncludeFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node does not [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node does not cluster setting [%s] filters [%s]",
                     CLUSTER_ROUTING_INCLUDE_GROUP_PREFIX, clusterIncludeFilters);
             }
         }
         if (clusterExcludeFilters != null) {
             if (clusterExcludeFilters.match(node.node())) {
-                return allocation.decision(Decision.NO, NAME, "node matches [%s] filters [%s]",
+                return allocation.decision(Decision.NO, NAME, "node matches cluster setting [%s] filters [%s]",
                     CLUSTER_ROUTING_EXCLUDE_GROUP_PREFIX, clusterExcludeFilters);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index 30fed539b79..387360f1a04 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -88,7 +88,8 @@ public class SameShardAllocationDecider extends AllocationDecider {
                             String host = checkNodeOnSameHostAddress ? node.node().getHostAddress() : node.node().getHostName();
                             return allocation.decision(Decision.NO, NAME,
                                 "the shard cannot be allocated on host %s [%s], where it already exists on node [%s]; " +
-                                    "set [%s] to false to allow multiple nodes on the same host to hold the same shard copies",
+                                    "set cluster setting [%s] to false to allow multiple nodes on the same host to hold the same " +
+                                    "shard copies",
                                 hostType, host, node.nodeId(), CLUSTER_ROUTING_ALLOCATION_SAME_HOST_SETTING.getKey());
                         }
                     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index e6f0fbcd645..2118d37fe47 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -122,12 +122,12 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
 
         if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
             return allocation.decision(Decision.NO, NAME,
-                "too many shards [%d] allocated to this node, [%s=%d]",
+                "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
                 nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
         }
         if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
             return allocation.decision(Decision.NO, NAME,
-                "too many shards [%d] allocated to this node for index [%s], [%s=%d]",
+                "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
                 indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
         }
         return allocation.decision(Decision.YES, NAME,
@@ -157,7 +157,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
         }
         if (clusterShardLimit >= 0 && nodeShardCount >= clusterShardLimit) {
             return allocation.decision(Decision.NO, NAME,
-                "too many shards [%d] allocated to this node, [%s=%d]",
+                "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
                 nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
         }
         return allocation.decision(Decision.YES, NAME,
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index a59f543ac3f..721de71435d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -126,7 +126,8 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
             }
             if (primariesInRecovery >= primariesInitialRecoveries) {
                 // TODO: Should index creation not be throttled for primary shards?
-                return allocation.decision(THROTTLE, NAME, "reached the limit of ongoing initial primary recoveries [%d], [%s=%d]",
+                return allocation.decision(THROTTLE, NAME,
+                    "reached the limit of ongoing initial primary recoveries [%d], cluster setting [%s=%d]",
                     primariesInRecovery, CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(),
                     primariesInitialRecoveries);
             } else {
@@ -140,7 +141,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
             int currentInRecoveries = allocation.routingNodes().getIncomingRecoveries(node.nodeId());
             if (currentInRecoveries >= concurrentIncomingRecoveries) {
                 return allocation.decision(THROTTLE, NAME,
-                    "reached the limit of incoming shard recoveries [%d], [%s=%d] (can also be set via [%s])",
+                    "reached the limit of incoming shard recoveries [%d], cluster setting [%s=%d] (can also be set via [%s])",
                     currentInRecoveries, CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(),
                     concurrentIncomingRecoveries,
                     CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey());
@@ -154,7 +155,7 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
                 if (primaryNodeOutRecoveries >= concurrentOutgoingRecoveries) {
                     return allocation.decision(THROTTLE, NAME,
                         "reached the limit of outgoing shard recoveries [%d] on the node [%s] which holds the primary, " +
-                        "[%s=%d] (can also be set via [%s])",
+                        "cluster setting [%s=%d] (can also be set via [%s])",
                         primaryNodeOutRecoveries, node.nodeId(),
                         CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(),
                         concurrentOutgoingRecoveries,
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index d213cea4d33..e2040312e09 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -31,9 +31,8 @@ import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor.BatchResult;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.TimeoutClusterStateListener;
@@ -64,6 +63,7 @@ import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor;
 import org.elasticsearch.common.util.concurrent.PrioritizedRunnable;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
@@ -102,6 +102,7 @@ public class ClusterService extends AbstractLifecycleComponent {
     public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
     private final ThreadPool threadPool;
     private final ClusterName clusterName;
+    private final Supplier<DiscoveryNode> localNodeSupplier;
 
     private BiConsumer<ClusterChangedEvent, Discovery.AckListener> clusterStatePublisher;
 
@@ -137,9 +138,12 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     private NodeConnectionsService nodeConnectionsService;
 
+    private DiscoverySettings discoverySettings;
+
     public ClusterService(Settings settings,
-                          ClusterSettings clusterSettings, ThreadPool threadPool) {
+                          ClusterSettings clusterSettings, ThreadPool threadPool, Supplier<DiscoveryNode> localNodeSupplier) {
         super(settings);
+        this.localNodeSupplier = localNodeSupplier;
         this.operationRouting = new OperationRouting(settings, clusterSettings);
         this.threadPool = threadPool;
         this.clusterSettings = clusterSettings;
@@ -165,14 +169,6 @@ public class ClusterService extends AbstractLifecycleComponent {
         clusterStatePublisher = publisher;
     }
 
-    public synchronized void setLocalNode(DiscoveryNode localNode) {
-        assert state().nodes().getLocalNodeId() == null : "local node is already set";
-        updateState(clusterState -> {
-            DiscoveryNodes nodes = DiscoveryNodes.builder(clusterState.nodes()).add(localNode).localNodeId(localNode.getId()).build();
-            return ClusterState.builder(clusterState).nodes(nodes).build();
-        });
-    }
-
     private void updateState(UnaryOperator<ClusterState> updateFunction) {
         this.state.getAndUpdate(updateFunction);
     }
@@ -212,10 +208,16 @@ public class ClusterService extends AbstractLifecycleComponent {
     @Override
     protected synchronized void doStart() {
         Objects.requireNonNull(clusterStatePublisher, "please set a cluster state publisher before starting");
-        Objects.requireNonNull(state().nodes().getLocalNode(), "please set the local node before starting");
         Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
+        Objects.requireNonNull(discoverySettings, "please set discovery settings before starting");
         addListener(localNodeMasterListeners);
-        updateState(state -> ClusterState.builder(state).blocks(initialBlocks).build());
+        DiscoveryNode localNode = localNodeSupplier.get();
+        assert localNode != null;
+        updateState(state -> {
+            assert state.nodes().getLocalNodeId() == null : "local node is already set";
+            DiscoveryNodes nodes = DiscoveryNodes.builder(state.nodes()).add(localNode).localNodeId(localNode.getId()).build();
+            return ClusterState.builder(state).nodes(nodes).blocks(initialBlocks).build();
+        });
         this.threadPoolExecutor = EsExecutors.newSinglePrioritizing(UPDATE_THREAD_NAME, daemonThreadFactory(settings, UPDATE_THREAD_NAME),
                 threadPool.getThreadContext());
     }
@@ -379,11 +381,11 @@ public class ClusterService extends AbstractLifecycleComponent {
      *                   task
      *
      */
-    public void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask) {
+    public <T extends ClusterStateTaskConfig & ClusterStateTaskExecutor<T> & ClusterStateTaskListener> void submitStateUpdateTask(
+        final String source, final T updateTask) {
         submitStateUpdateTask(source, updateTask, updateTask, updateTask, updateTask);
     }
 
-
     /**
      * Submits a cluster state update task; submitted updates will be
      * batched across the same instance of executor. The exact batching
@@ -573,10 +575,14 @@ public class ClusterService extends AbstractLifecycleComponent {
         return clusterName;
     }
 
+    public void setDiscoverySettings(DiscoverySettings discoverySettings) {
+        this.discoverySettings = discoverySettings;
+    }
+
     abstract static class SourcePrioritizedRunnable extends PrioritizedRunnable {
         protected final String source;
 
-        public SourcePrioritizedRunnable(Priority priority, String source) {
+        SourcePrioritizedRunnable(Priority priority, String source) {
             super(priority);
             this.source = source;
         }
@@ -643,29 +649,28 @@ public class ClusterService extends AbstractLifecycleComponent {
     }
 
     public TaskOutputs calculateTaskOutputs(TaskInputs taskInputs, ClusterState previousClusterState, long startTimeNS) {
-        BatchResult<Object> batchResult = executeTasks(taskInputs, startTimeNS, previousClusterState);
-        ClusterState newClusterState = batchResult.resultingState;
+        ClusterTasksResult<Object> clusterTasksResult = executeTasks(taskInputs, startTimeNS, previousClusterState);
         // extract those that are waiting for results
         List<UpdateTask> nonFailedTasks = new ArrayList<>();
         for (UpdateTask updateTask : taskInputs.updateTasks) {
-            assert batchResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask;
+            assert clusterTasksResult.executionResults.containsKey(updateTask.task) : "missing " + updateTask;
             final ClusterStateTaskExecutor.TaskResult taskResult =
-                batchResult.executionResults.get(updateTask.task);
+                clusterTasksResult.executionResults.get(updateTask.task);
             if (taskResult.isSuccess()) {
                 nonFailedTasks.add(updateTask);
             }
         }
-        newClusterState = patchVersions(previousClusterState, newClusterState);
+        ClusterState newClusterState = patchVersionsAndNoMasterBlocks(previousClusterState, clusterTasksResult);
 
         return new TaskOutputs(taskInputs, previousClusterState, newClusterState, nonFailedTasks,
-                                  batchResult.executionResults);
+                                  clusterTasksResult.executionResults);
     }
 
-    private BatchResult<Object> executeTasks(TaskInputs taskInputs, long startTimeNS, ClusterState previousClusterState) {
-        BatchResult<Object> batchResult;
+    private ClusterTasksResult<Object> executeTasks(TaskInputs taskInputs, long startTimeNS, ClusterState previousClusterState) {
+        ClusterTasksResult<Object> clusterTasksResult;
         try {
             List<Object> inputs = taskInputs.updateTasks.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
-            batchResult = taskInputs.executor.execute(previousClusterState, inputs);
+            clusterTasksResult = taskInputs.executor.execute(previousClusterState, inputs);
         } catch (Exception e) {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             if (logger.isTraceEnabled()) {
@@ -681,42 +686,70 @@ public class ClusterService extends AbstractLifecycleComponent {
                     e);
             }
             warnAboutSlowTaskIfNeeded(executionTime, taskInputs.summary);
-            batchResult = BatchResult.builder()
+            clusterTasksResult = ClusterTasksResult.builder()
                 .failures(taskInputs.updateTasks.stream().map(updateTask -> updateTask.task)::iterator, e)
                 .build(previousClusterState);
         }
 
-        assert batchResult.executionResults != null;
-        assert batchResult.executionResults.size() == taskInputs.updateTasks.size()
+        assert clusterTasksResult.executionResults != null;
+        assert clusterTasksResult.executionResults.size() == taskInputs.updateTasks.size()
                 : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", taskInputs.updateTasks.size(),
-                taskInputs.updateTasks.size() == 1 ? "" : "s", batchResult.executionResults.size());
+                taskInputs.updateTasks.size() == 1 ? "" : "s", clusterTasksResult.executionResults.size());
         boolean assertsEnabled = false;
         assert (assertsEnabled = true);
         if (assertsEnabled) {
             for (UpdateTask updateTask : taskInputs.updateTasks) {
-                assert batchResult.executionResults.containsKey(updateTask.task) :
+                assert clusterTasksResult.executionResults.containsKey(updateTask.task) :
                     "missing task result for " + updateTask;
             }
         }
 
-        return batchResult;
+        return clusterTasksResult;
     }
 
-    private ClusterState patchVersions(ClusterState previousClusterState, ClusterState newClusterState) {
-        if (previousClusterState != newClusterState) {
-            if (newClusterState.nodes().isLocalNodeElectedMaster()) {
-                // only the master controls the version numbers
-                Builder builder = ClusterState.builder(newClusterState).incrementVersion();
-                if (previousClusterState.routingTable() != newClusterState.routingTable()) {
-                    builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
-                            .version(newClusterState.routingTable().version() + 1).build());
-                }
-                if (previousClusterState.metaData() != newClusterState.metaData()) {
-                    builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
-                }
-                newClusterState = builder.build();
+    private ClusterState patchVersionsAndNoMasterBlocks(ClusterState previousClusterState, ClusterTasksResult<Object> executionResult) {
+        ClusterState newClusterState = executionResult.resultingState;
+
+        if (executionResult.noMaster) {
+            assert newClusterState == previousClusterState : "state can only be changed by ClusterService when noMaster = true";
+            if (previousClusterState.nodes().getMasterNodeId() != null) {
+                // remove block if it already exists before adding new one
+                assert previousClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id()) == false :
+                    "NO_MASTER_BLOCK should only be added by ClusterService";
+                ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(previousClusterState.blocks())
+                    .addGlobalBlock(discoverySettings.getNoMasterBlock())
+                    .build();
+
+                DiscoveryNodes discoveryNodes = new DiscoveryNodes.Builder(previousClusterState.nodes()).masterNodeId(null).build();
+                newClusterState = ClusterState.builder(previousClusterState)
+                    .blocks(clusterBlocks)
+                    .nodes(discoveryNodes)
+                    .build();
             }
+        } else if (newClusterState.nodes().isLocalNodeElectedMaster() && previousClusterState != newClusterState) {
+            // only the master controls the version numbers
+            Builder builder = ClusterState.builder(newClusterState).incrementVersion();
+            if (previousClusterState.routingTable() != newClusterState.routingTable()) {
+                builder.routingTable(RoutingTable.builder(newClusterState.routingTable())
+                    .version(newClusterState.routingTable().version() + 1).build());
+            }
+            if (previousClusterState.metaData() != newClusterState.metaData()) {
+                builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1));
+            }
+
+            // remove the no master block, if it exists
+            if (newClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id())) {
+                builder.blocks(ClusterBlocks.builder().blocks(newClusterState.blocks())
+                    .removeGlobalBlock(discoverySettings.getNoMasterBlock().id()));
+            }
+
+            newClusterState = builder.build();
         }
+
+        assert newClusterState.nodes().getMasterNodeId() == null ||
+            newClusterState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock().id()) == false :
+            "cluster state with master node must not have NO_MASTER_BLOCK";
+
         return newClusterState;
     }
 
@@ -738,7 +771,7 @@ public class ClusterService extends AbstractLifecycleComponent {
             taskOutputs.createAckListener(threadPool, newClusterState) :
             null;
 
-        nodeConnectionsService.connectToNodes(clusterChangedEvent.nodesDelta().addedNodes());
+        nodeConnectionsService.connectToNodes(newClusterState.nodes());
 
         // if we are the master, publish the new state to all nodes
         // we publish here before we send a notification to all the listeners, since if it fails
@@ -754,7 +787,8 @@ public class ClusterService extends AbstractLifecycleComponent {
                         "failing [{}]: failed to commit cluster state version [{}]", taskInputs.summary, version),
                     t);
                 // ensure that list of connected nodes in NodeConnectionsService is in-sync with the nodes of the current cluster state
-                nodeConnectionsService.disconnectFromNodes(clusterChangedEvent.nodesDelta().addedNodes());
+                nodeConnectionsService.connectToNodes(previousClusterState.nodes());
+                nodeConnectionsService.disconnectFromNodesExcept(previousClusterState.nodes());
                 taskOutputs.publishingFailed(t);
                 return;
             }
@@ -774,7 +808,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         logger.debug("set local cluster state to version {}", newClusterState.version());
         callClusterStateAppliers(newClusterState, clusterChangedEvent);
 
-        nodeConnectionsService.disconnectFromNodes(clusterChangedEvent.nodesDelta().removedNodes());
+        nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes());
 
         updateState(css -> newClusterState);
 
@@ -801,14 +835,16 @@ public class ClusterService extends AbstractLifecycleComponent {
 
         taskOutputs.processedDifferentClusterState(previousClusterState, newClusterState);
 
-        try {
-            taskOutputs.clusterStatePublished(clusterChangedEvent);
-        } catch (Exception e) {
-            logger.error(
-                (Supplier<?>) () -> new ParameterizedMessage(
-                    "exception thrown while notifying executor of new cluster state publication [{}]",
-                    taskInputs.summary),
-                e);
+        if (newClusterState.nodes().isLocalNodeElectedMaster()) {
+            try {
+                taskOutputs.clusterStatePublished(clusterChangedEvent);
+            } catch (Exception e) {
+                logger.error(
+                    (Supplier<?>) () -> new ParameterizedMessage(
+                        "exception thrown while notifying executor of new cluster state publication [{}]",
+                        taskInputs.summary),
+                    e);
+            }
         }
     }
 
@@ -856,7 +892,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         public final List<UpdateTask> nonFailedTasks;
         public final Map<Object, ClusterStateTaskExecutor.TaskResult> executionResults;
 
-        public TaskOutputs(TaskInputs taskInputs, ClusterState previousClusterState,
+        TaskOutputs(TaskInputs taskInputs, ClusterState previousClusterState,
                            ClusterState newClusterState, List<UpdateTask> nonFailedTasks,
                            Map<Object, ClusterStateTaskExecutor.TaskResult> executionResults) {
             this.taskInputs = taskInputs;
@@ -946,7 +982,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         private final ClusterStateTaskListener listener;
         private final Logger logger;
 
-        public SafeClusterStateTaskListener(ClusterStateTaskListener listener, Logger logger) {
+        SafeClusterStateTaskListener(ClusterStateTaskListener listener, Logger logger) {
             this.listener = listener;
             this.logger = logger;
         }
@@ -993,7 +1029,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         private final AckedClusterStateTaskListener listener;
         private final Logger logger;
 
-        public SafeAckedClusterStateTaskListener(AckedClusterStateTaskListener listener, Logger logger) {
+        SafeAckedClusterStateTaskListener(AckedClusterStateTaskListener listener, Logger logger) {
             super(listener, logger);
             this.listener = listener;
             this.logger = logger;
diff --git a/core/src/main/java/org/elasticsearch/common/Booleans.java b/core/src/main/java/org/elasticsearch/common/Booleans.java
index 9ec1ac968ac..025174c477d 100644
--- a/core/src/main/java/org/elasticsearch/common/Booleans.java
+++ b/core/src/main/java/org/elasticsearch/common/Booleans.java
@@ -19,14 +19,128 @@
 
 package org.elasticsearch.common;
 
-public class Booleans {
+public final class Booleans {
+    private Booleans() {
+        throw new AssertionError("No instances intended");
+    }
+
+    /**
+     * Parses a char[] representation of a boolean value to <code>boolean</code>.
+     *
+     * @return <code>true</code> iff the sequence of chars is "true", <code>false</code> iff the sequence of chars is "false" or the
+     * provided default value iff either text is <code>null</code> or length == 0.
+     * @throws IllegalArgumentException if the string cannot be parsed to boolean.
+     */
+    public static boolean parseBoolean(char[] text, int offset, int length, boolean defaultValue) {
+        if (text == null || length == 0) {
+            return defaultValue;
+        } else {
+            return parseBoolean(new String(text, offset, length));
+        }
+    }
+
+    /**
+     * returns true iff the sequence of chars is one of "true","false".
+     *
+     * @param text   sequence to check
+     * @param offset offset to start
+     * @param length length to check
+     */
+    public static boolean isBoolean(char[] text, int offset, int length) {
+        if (text == null || length == 0) {
+            return false;
+        }
+        return isBoolean(new String(text, offset, length));
+    }
+
+    public static boolean isBoolean(String value) {
+        return isFalse(value) || isTrue(value);
+    }
+
+    /**
+     * Parses a string representation of a boolean value to <code>boolean</code>.
+     *
+     * @return <code>true</code> iff the provided value is "true". <code>false</code> iff the provided value is "false".
+     * @throws IllegalArgumentException if the string cannot be parsed to boolean.
+     */
+    public static boolean parseBoolean(String value) {
+        if (isFalse(value)) {
+            return false;
+        }
+        if (isTrue(value)) {
+            return true;
+        }
+        throw new IllegalArgumentException("Failed to parse value [" + value + "] as only [true] or [false] are allowed.");
+    }
+
+    /**
+     *
+     * @param value text to parse.
+     * @param defaultValue The default value to return if the provided value is <code>null</code>.
+     * @return see {@link #parseBoolean(String)}
+     */
+    public static boolean parseBoolean(String value, boolean defaultValue) {
+        if (Strings.hasText(value)) {
+            return parseBoolean(value);
+        }
+        return defaultValue;
+    }
+
+    public static Boolean parseBoolean(String value, Boolean defaultValue) {
+        if (Strings.hasText(value)) {
+            return parseBoolean(value);
+        }
+        return defaultValue;
+    }
 
     /**
      * Returns <code>false</code> if text is in <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>; else, true
+     *
+     * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(String, Boolean)} instead.
      */
-    public static boolean parseBoolean(char[] text, int offset, int length, boolean defaultValue) {
-        // TODO: the leniency here is very dangerous: a simple typo will be misinterpreted and the user won't know.
-        // We should remove it and cutover to https://github.com/rmuir/booleanparser
+    @Deprecated
+    public static Boolean parseBooleanLenient(String value, Boolean defaultValue) {
+        if (value == null) { // only for the null case we do that here!
+            return defaultValue;
+        }
+        return parseBooleanLenient(value, false);
+    }
+    /**
+     * Returns <code>true</code> iff the value is neither of the following:
+     *   <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>
+     *   otherwise <code>false</code>
+     *
+     * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(String, boolean)} instead.
+     */
+    @Deprecated
+    public static boolean parseBooleanLenient(String value, boolean defaultValue) {
+        if (value == null) {
+            return defaultValue;
+        }
+        return !(value.equals("false") || value.equals("0") || value.equals("off") || value.equals("no"));
+    }
+
+    /**
+     * @return <code>true</code> iff the value is <tt>false</tt>, otherwise <code>false</code>.
+     */
+    public static boolean isFalse(String value) {
+        return "false".equals(value);
+    }
+
+    /**
+     * @return <code>true</code> iff the value is <tt>true</tt>, otherwise <code>false</code>
+     */
+    public static boolean isTrue(String value) {
+        return "true".equals(value);
+    }
+
+    /**
+     * Returns <code>false</code> if text is in <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>; else, true
+     *
+     * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #parseBoolean(char[], int, int, boolean)} instead
+     */
+    @Deprecated
+    public static boolean parseBooleanLenient(char[] text, int offset, int length, boolean defaultValue) {
         if (text == null || length == 0) {
             return defaultValue;
         }
@@ -40,7 +154,8 @@ public class Booleans {
             return !(text[offset] == 'o' && text[offset + 1] == 'f' && text[offset + 2] == 'f');
         }
         if (length == 5) {
-            return !(text[offset] == 'f' && text[offset + 1] == 'a' && text[offset + 2] == 'l' && text[offset + 3] == 's' && text[offset + 4] == 'e');
+            return !(text[offset] == 'f' && text[offset + 1] == 'a' && text[offset + 2] == 'l' && text[offset + 3] == 's' &&
+                text[offset + 4] == 'e');
         }
         return true;
     }
@@ -51,8 +166,11 @@ public class Booleans {
      * @param text   sequence to check
      * @param offset offset to start
      * @param length length to check
+     *
+     * @deprecated Only kept to provide automatic upgrades for pre 6.0 indices. Use {@link #isBoolean(char[], int, int)} instead.
      */
-    public static boolean isBoolean(char[] text, int offset, int length) {
+    @Deprecated
+    public static boolean isBooleanLenient(char[] text, int offset, int length) {
         if (text == null || length == 0) {
             return false;
         }
@@ -64,69 +182,16 @@ public class Booleans {
         }
         if (length == 3) {
             return (text[offset] == 'o' && text[offset + 1] == 'f' && text[offset + 2] == 'f') ||
-                    (text[offset] == 'y' && text[offset + 1] == 'e' && text[offset + 2] == 's');
+                (text[offset] == 'y' && text[offset + 1] == 'e' && text[offset + 2] == 's');
         }
         if (length == 4) {
             return (text[offset] == 't' && text[offset + 1] == 'r' && text[offset + 2] == 'u' && text[offset + 3] == 'e');
         }
         if (length == 5) {
-            return (text[offset] == 'f' && text[offset + 1] == 'a' && text[offset + 2] == 'l' && text[offset + 3] == 's' && text[offset + 4] == 'e');
+            return (text[offset] == 'f' && text[offset + 1] == 'a' && text[offset + 2] == 'l' && text[offset + 3] == 's' &&
+                text[offset + 4] == 'e');
         }
         return false;
     }
 
-    /***
-     *
-     * @return true/false
-     * throws exception if string cannot be parsed to boolean
-     */
-    public static Boolean parseBooleanExact(String value) {
-        boolean isFalse = isExplicitFalse(value);
-        if (isFalse) {
-            return false;
-        }
-        boolean isTrue = isExplicitTrue(value);
-        if (isTrue) {
-            return true;
-        }
-
-        throw new IllegalArgumentException("Failed to parse value [" + value + "] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]");
-    }
-
-    public static Boolean parseBoolean(String value, Boolean defaultValue) {
-        if (value == null) { // only for the null case we do that here!
-            return defaultValue;
-        }
-        return parseBoolean(value, false);
-    }
-    /**
-     * Returns <code>true</code> iff the value is neither of the following:
-     *   <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>
-     *   otherwise <code>false</code>
-     */
-    public static boolean parseBoolean(String value, boolean defaultValue) {
-        if (value == null) {
-            return defaultValue;
-        }
-        return !(value.equals("false") || value.equals("0") || value.equals("off") || value.equals("no"));
-    }
-
-    /**
-     * Returns <code>true</code> iff the value is either of the following:
-     *   <tt>false</tt>, <tt>0</tt>, <tt>off</tt>, <tt>no</tt>
-     *   otherwise <code>false</code>
-     */
-    public static boolean isExplicitFalse(String value) {
-        return value != null && (value.equals("false") || value.equals("0") || value.equals("off") || value.equals("no"));
-    }
-
-    /**
-     * Returns <code>true</code> iff the value is either of the following:
-     *   <tt>true</tt>, <tt>1</tt>, <tt>on</tt>, <tt>yes</tt>
-     *   otherwise <code>false</code>
-     */
-    public static boolean isExplicitTrue(String value) {
-        return value != null && (value.equals("true") || value.equals("1") || value.equals("on") || value.equals("yes"));
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java b/core/src/main/java/org/elasticsearch/common/CheckedFunction.java
similarity index 73%
rename from core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java
rename to core/src/main/java/org/elasticsearch/common/CheckedFunction.java
index dd04145ba7d..4a2d222db0b 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchExtRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/CheckedFunction.java
@@ -17,16 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.search;
+package org.elasticsearch.common;
 
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
+import java.util.function.Function;
 
 /**
- * Extensions to ParseFieldRegistry to make Guice happy.
+ * A {@link Function}-like interface which allows throwing checked exceptions.
  */
-public class SearchExtRegistry extends ParseFieldRegistry<SearchExtParser> {
-
-    public SearchExtRegistry() {
-        super("ext");
-    }
+@FunctionalInterface
+public interface CheckedFunction<T, R, E extends Exception> {
+    R apply(T t) throws E;
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java b/core/src/main/java/org/elasticsearch/common/CheckedRunnable.java
similarity index 69%
rename from core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
rename to core/src/main/java/org/elasticsearch/common/CheckedRunnable.java
index f1d45b55495..196eb53a878 100644
--- a/core/src/main/java/org/elasticsearch/indices/query/IndicesQueriesRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/CheckedRunnable.java
@@ -17,16 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.indices.query;
+package org.elasticsearch.common;
 
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.index.query.QueryParser;
+import java.lang.Runnable;
 
 /**
- * Extensions to ParseFieldRegistry to make Guice happy.
+ * A {@link Runnable}-like interface which allows throwing checked exceptions.
  */
-public class IndicesQueriesRegistry extends ParseFieldRegistry<QueryParser<?>> {
-    public IndicesQueriesRegistry() {
-        super("query");
-    }
+@FunctionalInterface
+public interface CheckedRunnable<E extends Exception> {
+    void run() throws E;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java b/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java
new file mode 100644
index 00000000000..8f4d2edea0c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/CheckedSupplier.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.Supplier;
+
+/**
+ * A {@link Supplier}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedSupplier<R, E extends Exception> {
+    R get() throws E;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/ParseField.java b/core/src/main/java/org/elasticsearch/common/ParseField.java
index 7121be7d1d8..fc9377eeb2f 100644
--- a/core/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/core/src/main/java/org/elasticsearch/common/ParseField.java
@@ -101,14 +101,10 @@ public class ParseField {
     /**
      * @param fieldName
      *            the field name to match against this {@link ParseField}
-     * @param strict
-     *            if true an exception will be thrown if a deprecated field name
-     *            is given. If false the deprecated name will be matched but a
-     *            message will also be logged to the {@link DeprecationLogger}
      * @return true if <code>fieldName</code> matches any of the acceptable
      *         names for this {@link ParseField}.
      */
-    boolean match(String fieldName, boolean strict) {
+    public boolean match(String fieldName) {
         Objects.requireNonNull(fieldName, "fieldName cannot be null");
         // if this parse field has not been completely deprecated then try to
         // match the preferred name
@@ -128,11 +124,7 @@ public class ParseField {
                     // message to indicate what should be used instead
                     msg = "Deprecated field [" + fieldName + "] used, replaced by [" + allReplacedWith + "]";
                 }
-                if (strict) {
-                    throw new IllegalArgumentException(msg);
-                } else {
-                    DEPRECATION_LOGGER.deprecated(msg);
-                }
+                DEPRECATION_LOGGER.deprecated(msg);
                 return true;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java b/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java
deleted file mode 100644
index 9866694a230..00000000000
--- a/core/src/main/java/org/elasticsearch/common/ParseFieldMatcher.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common;
-
-import org.elasticsearch.common.settings.Settings;
-
-/**
- * Matcher to use in combination with {@link ParseField} while parsing requests. Matches a {@link ParseField}
- * against a field name and throw deprecation exception depending on the current value of the {@link #PARSE_STRICT} setting.
- */
-public class ParseFieldMatcher {
-    public static final String PARSE_STRICT = "index.query.parse.strict";
-    public static final ParseFieldMatcher EMPTY = new ParseFieldMatcher(false);
-    public static final ParseFieldMatcher STRICT = new ParseFieldMatcher(true);
-
-    private final boolean strict;
-
-    public ParseFieldMatcher(Settings settings) {
-        this(settings.getAsBoolean(PARSE_STRICT, false));
-    }
-
-    public ParseFieldMatcher(boolean strict) {
-        this.strict = strict;
-    }
-
-    /** Should deprecated settings be rejected? */
-    public boolean isStrict() {
-        return strict;
-    }
-
-    /**
-     * Matches a {@link ParseField} against a field name, and throws deprecation exception depending on the current
-     * value of the {@link #PARSE_STRICT} setting.
-     * @param fieldName the field name found in the request while parsing
-     * @param parseField the parse field that we are looking for
-     * @throws IllegalArgumentException whenever we are in strict mode and the request contained a deprecated field
-     * @return true whenever the parse field that we are looking for was found, false otherwise
-     */
-    public boolean match(String fieldName, ParseField parseField) {
-        return parseField.match(fieldName, strict);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/ParsingException.java b/core/src/main/java/org/elasticsearch/common/ParsingException.java
index 0519ab38339..5dc2c8a74e4 100644
--- a/core/src/main/java/org/elasticsearch/common/ParsingException.java
+++ b/core/src/main/java/org/elasticsearch/common/ParsingException.java
@@ -95,12 +95,11 @@ public class ParsingException extends ElasticsearchException {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         if (lineNumber != UNKNOWN_POSITION) {
             builder.field("line", lineNumber);
             builder.field("col", columnNumber);
         }
-        super.innerToXContent(builder, params);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java
index 1ef13e3bc70..2f55ab46e7e 100644
--- a/core/src/main/java/org/elasticsearch/common/Strings.java
+++ b/core/src/main/java/org/elasticsearch/common/Strings.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 
@@ -276,26 +277,6 @@ public class Strings {
         return true;
     }
 
-    /**
-     * Count the occurrences of the substring in string s.
-     *
-     * @param str string to search in. Return 0 if this is null.
-     * @param sub string to search for. Return 0 if this is null.
-     */
-    public static int countOccurrencesOf(String str, String sub) {
-        if (str == null || sub == null || str.length() == 0 || sub.length() == 0) {
-            return 0;
-        }
-        int count = 0;
-        int pos = 0;
-        int idx;
-        while ((idx = str.indexOf(sub, pos)) != -1) {
-            ++count;
-            pos = idx + sub.length();
-        }
-        return count;
-    }
-
     /**
      * Replace all occurrences of a substring within a string with
      * another string.
@@ -877,26 +858,17 @@ public class Strings {
     }
 
     /**
-     * Return a {@link String} that is the json representation of the provided
-     * {@link ToXContent}.
+     * Return a {@link String} that is the json representation of the provided {@link ToXContent}.
+     * Wraps the output into an anonymous object.
      */
     public static String toString(ToXContent toXContent) {
-        return toString(toXContent, false);
-    }
-
-    /**
-     * Return a {@link String} that is the json representation of the provided
-     * {@link ToXContent}.
-     * @param wrapInObject set this to true if the ToXContent instance expects to be inside an object
-     */
-    public static String toString(ToXContent toXContent, boolean wrapInObject) {
         try {
             XContentBuilder builder = JsonXContent.contentBuilder();
-            if (wrapInObject) {
+            if (toXContent.isFragment()) {
                 builder.startObject();
             }
             toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            if (wrapInObject) {
+            if (toXContent.isFragment()) {
                 builder.endObject();
             }
             return builder.string();
diff --git a/core/src/main/java/org/elasticsearch/common/TriFunction.java b/core/src/main/java/org/elasticsearch/common/TriFunction.java
new file mode 100644
index 00000000000..85655863a4f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/TriFunction.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.elasticsearch.common;
+
+/**
+ * Represents a function that accepts three arguments and produces a result.
+ *
+ * @param <S> the type of the first argument
+ * @param <T> the type of the second argument
+ * @param <U> the type of the third argument
+ * @param <R> the return type
+ */
+@FunctionalInterface
+public interface TriFunction<S, T, U, R> {
+    /**
+     * Applies this function to the given arguments.
+     *
+     * @param s the first function argument
+     * @param t the second function argument
+     * @param u the third function argument
+     * @return the result
+     */
+    R apply(S s, T t, U u);
+}
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
index 68bf52e9e0d..e4019f9f665 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java
@@ -90,12 +90,12 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker {
     @Override
     public void circuitBreak(String fieldName, long bytesNeeded) {
         this.trippedCount.incrementAndGet();
-        final String message = "[" + this.name + "] Data too large, data for [" +
-                fieldName + "] would be larger than limit of [" +
+        final String message = "[" + this.name + "] Data too large, data for [" + fieldName + "]" +
+                " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
+                ", which is larger than the limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
         logger.debug("{}", message);
-        throw new CircuitBreakingException(message,
-                bytesNeeded, this.memoryBytesLimit);
+        throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
index e700d301644..e01fe1beee2 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/CircuitBreakingException.java
@@ -73,9 +73,8 @@ public class CircuitBreakingException extends ElasticsearchException {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("bytes_wanted", bytesWanted);
         builder.field("bytes_limit", byteLimit);
-        super.innerToXContent(builder, params);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
index 23e76a9fd35..dbd1fe92ffe 100644
--- a/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
+++ b/core/src/main/java/org/elasticsearch/common/breaker/MemoryCircuitBreaker.java
@@ -79,7 +79,9 @@ public class MemoryCircuitBreaker implements CircuitBreaker {
     @Override
     public void circuitBreak(String fieldName, long bytesNeeded) throws CircuitBreakingException {
         this.trippedCount.incrementAndGet();
-        final String message = "Data too large, data for field [" + fieldName + "] would be larger than limit of [" +
+        final String message = "[" + getName() + "] Data too large, data for field [" + fieldName + "]" +
+                " would be [" + bytesNeeded + "/" + new ByteSizeValue(bytesNeeded) + "]" +
+                ", which is larger than the limit of [" +
                 memoryBytesLimit + "/" + new ByteSizeValue(memoryBytesLimit) + "]";
         logger.debug("{}", message);
         throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit);
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
index 4dcd92bf549..fff6392f238 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
@@ -37,7 +37,7 @@ final class BytesReferenceStreamInput extends StreamInput {
     private final int length; // the total size of the stream
     private int offset; // the current position of the stream
 
-    public BytesReferenceStreamInput(BytesRefIterator iterator, final int length) throws IOException {
+    BytesReferenceStreamInput(BytesRefIterator iterator, final int length) throws IOException {
         this.iterator = iterator;
         this.slice = iterator.next();
         this.length = length;
diff --git a/core/src/main/java/org/elasticsearch/common/cache/Cache.java b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
index fd83f5f1494..2297df67655 100644
--- a/core/src/main/java/org/elasticsearch/common/cache/Cache.java
+++ b/core/src/main/java/org/elasticsearch/common/cache/Cache.java
@@ -166,7 +166,7 @@ public class Cache<K, V> {
         Entry<K, V> after;
         State state = State.NEW;
 
-        public Entry(K key, V value, long writeTime) {
+        Entry(K key, V value, long writeTime) {
             this.key = key;
             this.value = value;
             this.writeTime = this.accessTime = writeTime;
diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
index 53167686736..85d7eda8363 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
@@ -433,7 +433,7 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
         private final Deque<Map.Entry<K, V>> entries;
         private final Deque<Node<K, V>> nodes;
 
-        public EntryIterator(Node<K, V> node) {
+        EntryIterator(Node<K, V> node) {
             entries = new ArrayDeque<>();
             nodes = new ArrayDeque<>();
             node.visit(entries, nodes);
diff --git a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
index d44bf7341c4..a8c811a0d06 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/Iterators.java
@@ -36,7 +36,7 @@ public class Iterators {
         private final Iterator<? extends T>[] iterators;
         private int index = 0;
 
-        public ConcatenatedIterator(Iterator<? extends T>... iterators) {
+        ConcatenatedIterator(Iterator<? extends T>... iterators) {
             if (iterators == null) {
                 throw new NullPointerException("iterators");
             }
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
index 067d4666722..a355a12d672 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class CompressorFactory {
 
@@ -68,9 +69,10 @@ public class CompressorFactory {
 
     /**
      * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(BytesReference)}.
+     * @throws NullPointerException a NullPointerException will be thrown when bytes is null
      */
     public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException {
-        Compressor compressor = compressor(bytes);
+        Compressor compressor = compressor(Objects.requireNonNull(bytes, "the BytesReference must not be null"));
         BytesReference uncompressed;
         if (compressor != null) {
             uncompressed = uncompress(bytes, compressor);
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
index f63636174f6..b1767e73103 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
@@ -19,18 +19,10 @@
 
 package org.elasticsearch.common.geo;
 
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.SloppyMath;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.GeoPointValues;
-import org.elasticsearch.index.fielddata.MultiGeoPointValues;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -39,101 +31,9 @@ import java.util.Locale;
  * Geo distance calculation.
  */
 public enum GeoDistance implements Writeable {
-    /**
-     * Calculates distance as points on a plane. Faster, but less accurate than {@link #ARC}.
-     * @deprecated use {@link GeoUtils#planeDistance}
-     */
-    @Deprecated
-    PLANE {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double px = targetLongitude - sourceLongitude;
-            double py = targetLatitude - sourceLatitude;
-            return Math.sqrt(px * px + py * py) * unit.getDistancePerDegree();
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new PlaneFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    },
-
-    /**
-     * Calculates distance factor.
-     * Note: {@code calculate} is simply returning the RHS of the spherical law of cosines from 2 lat,lon points.
-     * {@code normalize} also returns the RHS of the spherical law of cosines for a given distance
-     * @deprecated use {@link SloppyMath#haversinMeters} to get distance in meters, law of cosines is being removed
-     */
-    @Deprecated
-    FACTOR {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double longitudeDifference = targetLongitude - sourceLongitude;
-            double a = Math.toRadians(90D - sourceLatitude);
-            double c = Math.toRadians(90D - targetLatitude);
-            return (Math.cos(a) * Math.cos(c)) + (Math.sin(a) * Math.sin(c) * Math.cos(Math.toRadians(longitudeDifference)));
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return Math.cos(distance / unit.getEarthRadius());
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new FactorFixedSourceDistance(sourceLatitude, sourceLongitude);
-        }
-    },
-    /**
-     * Calculates distance as points on a globe.
-     * @deprecated use {@link GeoUtils#arcDistance}
-     */
-    @Deprecated
-    ARC {
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            double result = SloppyMath.haversinMeters(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude);
-            return unit.fromMeters(result);
-        }
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new ArcFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    },
-    /**
-     * Calculates distance as points on a globe in a sloppy way. Close to the pole areas the accuracy
-     * of this function decreases.
-     */
-    @Deprecated
-    SLOPPY_ARC {
-
-        @Override
-        public double normalize(double distance, DistanceUnit unit) {
-            return distance;
-        }
-
-        @Override
-        public double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit) {
-            return unit.fromMeters(SloppyMath.haversinMeters(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude));
-        }
-
-        @Override
-        public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new SloppyArcFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
-        }
-    };
+    PLANE, ARC;
 
+    /** Creates a GeoDistance instance from an input stream */
     public static GeoDistance readFromStream(StreamInput in) throws IOException {
         int ord = in.readVInt();
         if (ord < 0 || ord >= values().length) {
@@ -142,70 +42,17 @@ public enum GeoDistance implements Writeable {
         return GeoDistance.values()[ord];
     }
 
+    /** Writes an instance of a GeoDistance object to an output stream */
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(this.ordinal());
     }
 
-    /**
-     * Default {@link GeoDistance} function. This method should be used, If no specific function has been selected.
-     * This is an alias for <code>SLOPPY_ARC</code>
-     */
-    @Deprecated
-    public static final GeoDistance DEFAULT = SLOPPY_ARC;
-
-    public abstract double normalize(double distance, DistanceUnit unit);
-
-    public abstract double calculate(double sourceLatitude, double sourceLongitude, double targetLatitude, double targetLongitude, DistanceUnit unit);
-
-    public abstract FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit);
-
-    private static final double MIN_LAT = Math.toRadians(-90d);  // -PI/2
-    private static final double MAX_LAT = Math.toRadians(90d);   //  PI/2
-    private static final double MIN_LON = Math.toRadians(-180d); // -PI
-    private static final double MAX_LON = Math.toRadians(180d);  //  PI
-
-    public static DistanceBoundingCheck distanceBoundingCheck(double sourceLatitude, double sourceLongitude, double distance, DistanceUnit unit) {
-        // angular distance in radians on a great circle
-        // assume worst-case: use the minor axis
-        double radDist = unit.toMeters(distance) / GeoUtils.EARTH_SEMI_MINOR_AXIS;
-
-        double radLat = Math.toRadians(sourceLatitude);
-        double radLon = Math.toRadians(sourceLongitude);
-
-        double minLat = radLat - radDist;
-        double maxLat = radLat + radDist;
-
-        double minLon, maxLon;
-        if (minLat > MIN_LAT && maxLat < MAX_LAT) {
-            double deltaLon = Math.asin(Math.sin(radDist) / Math.cos(radLat));
-            minLon = radLon - deltaLon;
-            if (minLon < MIN_LON) minLon += 2d * Math.PI;
-            maxLon = radLon + deltaLon;
-            if (maxLon > MAX_LON) maxLon -= 2d * Math.PI;
-        } else {
-            // a pole is within the distance
-            minLat = Math.max(minLat, MIN_LAT);
-            maxLat = Math.min(maxLat, MAX_LAT);
-            minLon = MIN_LON;
-            maxLon = MAX_LON;
-        }
-
-        GeoPoint topLeft = new GeoPoint(Math.toDegrees(maxLat), Math.toDegrees(minLon));
-        GeoPoint bottomRight = new GeoPoint(Math.toDegrees(minLat), Math.toDegrees(maxLon));
-        if (minLon > maxLon) {
-            return new Meridian180DistanceBoundingCheck(topLeft, bottomRight);
-        }
-        return new SimpleDistanceBoundingCheck(topLeft, bottomRight);
-    }
-
     /**
      * Get a {@link GeoDistance} according to a given name. Valid values are
      *
      * <ul>
      *     <li><b>plane</b> for <code>GeoDistance.PLANE</code></li>
-     *     <li><b>sloppy_arc</b> for <code>GeoDistance.SLOPPY_ARC</code></li>
-     *     <li><b>factor</b> for <code>GeoDistance.FACTOR</code></li>
      *     <li><b>arc</b> for <code>GeoDistance.ARC</code></li>
      * </ul>
      *
@@ -218,222 +65,16 @@ public enum GeoDistance implements Writeable {
             return PLANE;
         } else if ("arc".equals(name)) {
             return ARC;
-        } else if ("sloppy_arc".equals(name)) {
-            return SLOPPY_ARC;
-        } else if ("factor".equals(name)) {
-            return FACTOR;
         }
         throw new IllegalArgumentException("No geo distance for [" + name + "]");
     }
 
-    public interface FixedSourceDistance {
-
-        double calculate(double targetLatitude, double targetLongitude);
-    }
-
-    public interface DistanceBoundingCheck {
-
-        boolean isWithin(double targetLatitude, double targetLongitude);
-
-        GeoPoint topLeft();
-
-        GeoPoint bottomRight();
-    }
-
-    public static final AlwaysDistanceBoundingCheck ALWAYS_INSTANCE = new AlwaysDistanceBoundingCheck();
-
-    private static class AlwaysDistanceBoundingCheck implements DistanceBoundingCheck {
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return true;
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return null;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return null;
-        }
-    }
-
-    public static class Meridian180DistanceBoundingCheck implements DistanceBoundingCheck {
-
-        private final GeoPoint topLeft;
-        private final GeoPoint bottomRight;
-
-        public Meridian180DistanceBoundingCheck(GeoPoint topLeft, GeoPoint bottomRight) {
-            this.topLeft = topLeft;
-            this.bottomRight = bottomRight;
-        }
-
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return (targetLatitude >= bottomRight.lat() && targetLatitude <= topLeft.lat()) &&
-                    (targetLongitude >= topLeft.lon() || targetLongitude <= bottomRight.lon());
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return topLeft;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return bottomRight;
-        }
-    }
-
-    public static class SimpleDistanceBoundingCheck implements DistanceBoundingCheck {
-        private final GeoPoint topLeft;
-        private final GeoPoint bottomRight;
-
-        public SimpleDistanceBoundingCheck(GeoPoint topLeft, GeoPoint bottomRight) {
-            this.topLeft = topLeft;
-            this.bottomRight = bottomRight;
-        }
-
-        @Override
-        public boolean isWithin(double targetLatitude, double targetLongitude) {
-            return (targetLatitude >= bottomRight.lat() && targetLatitude <= topLeft.lat()) &&
-                    (targetLongitude >= topLeft.lon() && targetLongitude <= bottomRight.lon());
-        }
-
-        @Override
-        public GeoPoint topLeft() {
-            return topLeft;
-        }
-
-        @Override
-        public GeoPoint bottomRight() {
-            return bottomRight;
-        }
-    }
-
-    public static class PlaneFixedSourceDistance implements FixedSourceDistance {
-
-        private final double sourceLatitude;
-        private final double sourceLongitude;
-        private final double distancePerDegree;
-
-        public PlaneFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            this.sourceLatitude = sourceLatitude;
-            this.sourceLongitude = sourceLongitude;
-            this.distancePerDegree = unit.getDistancePerDegree();
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            double px = targetLongitude - sourceLongitude;
-            double py = targetLatitude - sourceLatitude;
-            return Math.sqrt(px * px + py * py) * distancePerDegree;
-        }
-    }
-
-    public static class FactorFixedSourceDistance implements FixedSourceDistance {
-
-        private final double sourceLongitude;
-
-        private final double a;
-        private final double sinA;
-        private final double cosA;
-
-        public FactorFixedSourceDistance(double sourceLatitude, double sourceLongitude) {
-            this.sourceLongitude = sourceLongitude;
-            this.a = Math.toRadians(90D - sourceLatitude);
-            this.sinA = Math.sin(a);
-            this.cosA = Math.cos(a);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            double longitudeDifference = targetLongitude - sourceLongitude;
-            double c = Math.toRadians(90D - targetLatitude);
-            return (cosA * Math.cos(c)) + (sinA * Math.sin(c) * Math.cos(Math.toRadians(longitudeDifference)));
-        }
-    }
-
-    /**
-     * Basic implementation of {@link FixedSourceDistance}. This class keeps the basic parameters for a distance
-     * functions based on a fixed source. Namely latitude, longitude and unit.
-     */
-    public abstract static class FixedSourceDistanceBase implements FixedSourceDistance {
-        protected final double sourceLatitude;
-        protected final double sourceLongitude;
-        protected final DistanceUnit unit;
-
-        public FixedSourceDistanceBase(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            this.sourceLatitude = sourceLatitude;
-            this.sourceLongitude = sourceLongitude;
-            this.unit = unit;
-        }
-    }
-
-    public static class ArcFixedSourceDistance extends FixedSourceDistanceBase {
-
-        public ArcFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            super(sourceLatitude, sourceLongitude, unit);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            return ARC.calculate(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude, unit);
-        }
-
-    }
-
-    public static class SloppyArcFixedSourceDistance extends FixedSourceDistanceBase {
-
-        public SloppyArcFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            super(sourceLatitude, sourceLongitude, unit);
-        }
-
-        @Override
-        public double calculate(double targetLatitude, double targetLongitude) {
-            return SLOPPY_ARC.calculate(sourceLatitude, sourceLongitude, targetLatitude, targetLongitude, unit);
-        }
-    }
-
-
-    /**
-     * Return a {@link SortedNumericDoubleValues} instance that returns the distances to a list of geo-points for each document.
-     */
-    public static SortedNumericDoubleValues distanceValues(final MultiGeoPointValues geoPointValues, final FixedSourceDistance... distances) {
-        final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues);
-        if (singleValues != null && distances.length == 1) {
-            final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues);
-            return FieldData.singleton(new NumericDoubleValues() {
-
-                @Override
-                public double get(int docID) {
-                    if (docsWithField != null && !docsWithField.get(docID)) {
-                        return 0d;
-                    }
-                    final GeoPoint point = singleValues.get(docID);
-                    return distances[0].calculate(point.lat(), point.lon());
-                }
-
-            }, docsWithField);
-        } else {
-            return new SortingNumericDoubleValues() {
-
-                @Override
-                public void setDocument(int doc) {
-                    geoPointValues.setDocument(doc);
-                    resize(geoPointValues.count() * distances.length);
-                    int valueCounter = 0;
-                    for (FixedSourceDistance distance : distances) {
-                        for (int i = 0; i < geoPointValues.count(); ++i) {
-                            final GeoPoint point = geoPointValues.valueAt(i);
-                            values[valueCounter] = distance.calculate(point.lat(), point.lon());
-                            valueCounter++;
-                        }
-                    }
-                    sort();
-                }
-            };
+    /** compute the distance between two points using the selected algorithm (PLANE, ARC) */
+    public double calculate(double srcLat, double srcLon, double dstLat, double dstLon, DistanceUnit unit) {
+        if (this == PLANE) {
+            return DistanceUnit.convert(GeoUtils.planeDistance(srcLat, srcLon, dstLat, dstLon),
+                DistanceUnit.METERS, unit);
         }
+        return DistanceUnit.convert(GeoUtils.arcDistance(srcLat, srcLon, dstLat, dstLon), DistanceUnit.METERS, unit);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
index 2046b1a6e14..6e4033efd4c 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
@@ -19,13 +19,21 @@
 
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.geo.Rectangle;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.SloppyMath;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.GeoPointValues;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
 
 import java.io.IOException;
 
@@ -65,14 +73,6 @@ public class GeoUtils {
     /** rounding error for quantized latitude and longitude values */
     public static final double TOLERANCE = 1E-6;
 
-    /** Returns the minimum between the provided distance 'initialRadius' and the
-     * maximum distance/radius from the point 'center' before overlapping
-     **/
-    public static double maxRadialDistance(GeoPoint center, double initialRadius) {
-        final double maxRadius = maxRadialDistanceMeters(center.lat(), center.lon());
-        return Math.min(initialRadius, maxRadius);
-    }
-
     /** Returns true if latitude is actually a valid latitude value.*/
     public static boolean isValidLatitude(double latitude) {
         if (Double.isNaN(latitude) || Double.isInfinite(latitude) || latitude < GeoUtils.MIN_LAT || latitude > GeoUtils.MAX_LAT) {
@@ -481,7 +481,8 @@ public class GeoUtils {
 
     /**
      * Return the distance (in meters) between 2 lat,lon geo points using a simple tangential plane
-     * this provides a faster alternative to {@link GeoUtils#arcDistance} when points are within 5 km
+     * this provides a faster alternative to {@link GeoUtils#arcDistance} but is innaccurate for distances greater than
+     * 4 decimal degrees
      */
     public static double planeDistance(double lat1, double lon1, double lat2, double lon2) {
         double x = (lon2 - lon1) * SloppyMath.TO_RADIANS * Math.cos((lat2 + lat1) / 2.0 * SloppyMath.TO_RADIANS);
@@ -489,6 +490,61 @@ public class GeoUtils {
         return Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS;
     }
 
+    /** check if point is within a rectangle
+     * todo: move this to lucene Rectangle class
+     */
+    public static boolean rectangleContainsPoint(Rectangle r, double lat, double lon) {
+        if (lat >= r.minLat && lat <= r.maxLat) {
+            // if rectangle crosses the dateline we only check if the lon is >= min or max
+            return r.crossesDateline() ? lon >= r.minLon || lon <= r.maxLon : lon >= r.minLon && lon <= r.maxLon;
+        }
+        return false;
+    }
+
+    /**
+     * Return a {@link SortedNumericDoubleValues} instance that returns the distances to a list of geo-points
+     * for each document.
+     */
+    public static SortedNumericDoubleValues distanceValues(final GeoDistance distance,
+                                                           final DistanceUnit unit,
+                                                           final MultiGeoPointValues geoPointValues,
+                                                           final GeoPoint... fromPoints) {
+        final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues);
+        if (singleValues != null && fromPoints.length == 1) {
+            final Bits docsWithField = FieldData.unwrapSingletonBits(geoPointValues);
+            return FieldData.singleton(new NumericDoubleValues() {
+
+                @Override
+                public double get(int docID) {
+                    if (docsWithField != null && !docsWithField.get(docID)) {
+                        return 0d;
+                    }
+                    final GeoPoint to = singleValues.get(docID);
+                    final GeoPoint from = fromPoints[0];
+                    return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit);
+                }
+
+            }, docsWithField);
+        } else {
+            return new SortingNumericDoubleValues() {
+                @Override
+                public void setDocument(int doc) {
+                    geoPointValues.setDocument(doc);
+                    resize(geoPointValues.count() * fromPoints.length);
+                    int v = 0;
+                    for (GeoPoint from : fromPoints) {
+                        for (int i = 0; i < geoPointValues.count(); ++i) {
+                            final GeoPoint point = geoPointValues.valueAt(i);
+                            values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit);
+                            v++;
+                        }
+                    }
+                    sort();
+                }
+            };
+        }
+    }
+
     private GeoUtils() {
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java b/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
index 5a7c9e2a325..acac5fd6690 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/SpatialStrategy.java
@@ -31,7 +31,7 @@ public enum SpatialStrategy implements Writeable {
 
     private final String strategyName;
 
-    private SpatialStrategy(String strategyName) {
+    SpatialStrategy(String strategyName) {
         this.strategyName = strategyName;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
index cb2f8bb4e78..fbe47a17826 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilder.java
@@ -381,7 +381,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
         }
     }
 
-    public static enum Orientation {
+    public enum Orientation {
         LEFT,
         RIGHT;
 
@@ -427,7 +427,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
     /**
      * Enumeration that lists all {@link GeoShapeType}s that can be handled
      */
-    public static enum GeoShapeType {
+    public enum GeoShapeType {
         POINT("point"),
         MULTIPOINT("multipoint"),
         LINESTRING("linestring"),
@@ -440,7 +440,7 @@ public abstract class ShapeBuilder extends ToXContentToBytes implements NamedWri
 
         private final String shapename;
 
-        private GeoShapeType(String shapename) {
+        GeoShapeType(String shapename) {
             this.shapename = shapename;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java b/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
index 87bf31e911e..aa7029f2a9c 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ConstantFactory.java
@@ -30,7 +30,7 @@ class ConstantFactory<T> implements InternalFactory<T> {
 
     private final Initializable<T> initializable;
 
-    public ConstantFactory(Initializable<T> initializable) {
+    ConstantFactory(Initializable<T> initializable) {
         this.initializable = initializable;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java b/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
index 40d589f37dc..2bc8f770bf8 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/DeferredLookups.java
@@ -34,7 +34,7 @@ class DeferredLookups implements Lookups {
     private final InjectorImpl injector;
     private final List<Element> lookups = new ArrayList<>();
 
-    public DeferredLookups(InjectorImpl injector) {
+    DeferredLookups(InjectorImpl injector) {
         this.injector = injector;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
index 8b8b7b78218..ed49d28dea7 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/EncounterImpl.java
@@ -36,7 +36,7 @@ final class EncounterImpl<T> implements TypeEncounter<T> {
     private List<InjectionListener<? super T>> injectionListeners; // lazy
     private boolean valid = true;
 
-    public EncounterImpl(Errors errors, Lookups lookups) {
+    EncounterImpl(Errors errors, Lookups lookups) {
         this.errors = errors;
         this.lookups = lookups;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java b/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
index efc10b27e49..b3cf0a14b6b 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ExposedKeyFactory.java
@@ -33,7 +33,7 @@ class ExposedKeyFactory<T> implements InternalFactory<T>, BindingProcessor.Creat
     private final PrivateElements privateElements;
     private BindingImpl<T> delegate;
 
-    public ExposedKeyFactory(Key<T> key, PrivateElements privateElements) {
+    ExposedKeyFactory(Key<T> key, PrivateElements privateElements) {
         this.key = key;
         this.privateElements = privateElements;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Initializer.java b/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
index 1d68f163bfe..ce7d7765ce3 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Initializer.java
@@ -115,7 +115,7 @@ class Initializer {
         private final Object source;
         private MembersInjectorImpl<T> membersInjector;
 
-        public InjectableReference(InjectorImpl injector, T instance, Object source) {
+        InjectableReference(InjectorImpl injector, T instance, Object source) {
             this.injector = injector;
             this.instance = Objects.requireNonNull(instance, "instance");
             this.source = Objects.requireNonNull(source, "source");
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java b/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
index e8b38b51330..909779cb442 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InjectionRequestProcessor.java
@@ -86,7 +86,7 @@ class InjectionRequestProcessor extends AbstractProcessor {
         final StaticInjectionRequest request;
         List<SingleMemberInjector> memberInjectors;
 
-        public StaticInjection(InjectorImpl injector, StaticInjectionRequest request) {
+        StaticInjection(InjectorImpl injector, StaticInjectionRequest request) {
             this.injector = injector;
             this.source = request.getSource();
             this.request = request;
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java b/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
index 8739d9182d8..54fecae9ba9 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InternalFactoryToProviderAdapter.java
@@ -33,11 +33,11 @@ class InternalFactoryToProviderAdapter<T> implements InternalFactory<T> {
     private final Initializable<Provider<? extends T>> initializable;
     private final Object source;
 
-    public InternalFactoryToProviderAdapter(Initializable<Provider<? extends T>> initializable) {
+    InternalFactoryToProviderAdapter(Initializable<Provider<? extends T>> initializable) {
         this(initializable, SourceProvider.UNKNOWN_SOURCE);
     }
 
-    public InternalFactoryToProviderAdapter(
+    InternalFactoryToProviderAdapter(
             Initializable<Provider<? extends T>> initializable, Object source) {
         this.initializable = Objects.requireNonNull(initializable, "provider");
         this.source = Objects.requireNonNull(source, "source");
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Key.java b/core/src/main/java/org/elasticsearch/common/inject/Key.java
index 83ab440d23c..833aac2d3fc 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Key.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Key.java
@@ -380,7 +380,7 @@ public class Key<T> {
         }
     }
 
-    static enum NullAnnotationStrategy implements AnnotationStrategy {
+    enum NullAnnotationStrategy implements AnnotationStrategy {
         INSTANCE;
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java b/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
index d7b6afbe6da..f4437569088 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ProviderToInternalFactoryAdapter.java
@@ -30,7 +30,7 @@ class ProviderToInternalFactoryAdapter<T> implements Provider<T> {
     private final InjectorImpl injector;
     private final InternalFactory<? extends T> internalFactory;
 
-    public ProviderToInternalFactoryAdapter(InjectorImpl injector,
+    ProviderToInternalFactoryAdapter(InjectorImpl injector,
                                             InternalFactory<? extends T> internalFactory) {
         this.injector = injector;
         this.internalFactory = internalFactory;
diff --git a/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java b/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
index 10ba17d86cd..ce02a26ffd0 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/SingleFieldInjector.java
@@ -34,7 +34,7 @@ class SingleFieldInjector implements SingleMemberInjector {
     final Dependency<?> dependency;
     final InternalFactory<?> factory;
 
-    public SingleFieldInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
+    SingleFieldInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
             throws ErrorsException {
         this.injectionPoint = injectionPoint;
         this.field = (Field) injectionPoint.getMember();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java b/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
index 9c407791160..7330d05df3b 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/SingleMethodInjector.java
@@ -34,7 +34,7 @@ class SingleMethodInjector implements SingleMemberInjector {
     final SingleParameterInjector<?>[] parameterInjectors;
     final InjectionPoint injectionPoint;
 
-    public SingleMethodInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
+    SingleMethodInjector(InjectorImpl injector, InjectionPoint injectionPoint, Errors errors)
             throws ErrorsException {
         this.injectionPoint = injectionPoint;
         final Method method = (Method) injectionPoint.getMember();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
index 07166537081..edd45c290a5 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/AssistedConstructor.java
@@ -43,7 +43,7 @@ class AssistedConstructor<T> {
     private final List<Parameter> allParameters;
 
     @SuppressWarnings("unchecked")
-    public AssistedConstructor(Constructor<T> constructor, List<TypeLiteral<?>> parameterTypes) {
+    AssistedConstructor(Constructor<T> constructor, List<TypeLiteral<?>> parameterTypes) {
         this.constructor = constructor;
 
         Annotation[][] annotations = constructor.getParameterAnnotations();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
index 5ceb086db9f..a21dc3aa7f5 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
@@ -39,7 +39,7 @@ class Parameter {
     private final Annotation bindingAnnotation;
     private final boolean isProvider;
 
-    public Parameter(Type type, Annotation[] annotations) {
+    Parameter(Type type, Annotation[] annotations) {
         this.type = type;
         this.bindingAnnotation = getBindingAnnotation(annotations);
         this.isAssisted = hasAssistedAnnotation(annotations);
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
index fc2a96e19df..7967f473948 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/ParameterListKey.java
@@ -34,11 +34,11 @@ class ParameterListKey {
 
     private final List<Type> paramList;
 
-    public ParameterListKey(List<Type> paramList) {
+    ParameterListKey(List<Type> paramList) {
         this.paramList = new ArrayList<>(paramList);
     }
 
-    public ParameterListKey(Type[] types) {
+    ParameterListKey(Type[] types) {
         this(Arrays.asList(types));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java b/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
index cc5a23a786d..0fec6b5bac2 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/internal/NullOutputException.java
@@ -24,7 +24,7 @@ package org.elasticsearch.common.inject.internal;
  * @author Bob Lee
  */
 class NullOutputException extends NullPointerException {
-    public NullOutputException(String s) {
+    NullOutputException(String s) {
         super(s);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
index 931d290fc19..76df334e4e3 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/AbstractMatcher.java
@@ -36,7 +36,7 @@ public abstract class AbstractMatcher<T> implements Matcher<T> {
     private static class AndMatcher<T> extends AbstractMatcher<T> {
         private final Matcher<? super T> a, b;
 
-        public AndMatcher(Matcher<? super T> a, Matcher<? super T> b) {
+        AndMatcher(Matcher<? super T> a, Matcher<? super T> b) {
             this.a = a;
             this.b = b;
         }
@@ -67,7 +67,7 @@ public abstract class AbstractMatcher<T> implements Matcher<T> {
     private static class OrMatcher<T> extends AbstractMatcher<T> {
         private final Matcher<? super T> a, b;
 
-        public OrMatcher(Matcher<? super T> a, Matcher<? super T> b) {
+        OrMatcher(Matcher<? super T> a, Matcher<? super T> b) {
             this.a = a;
             this.b = b;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
index e2ced98034e..cc354145b11 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/matcher/Matchers.java
@@ -113,7 +113,7 @@ public class Matchers {
     private static class AnnotatedWithType extends AbstractMatcher<AnnotatedElement> {
         private final Class<? extends Annotation> annotationType;
 
-        public AnnotatedWithType(Class<? extends Annotation> annotationType) {
+        AnnotatedWithType(Class<? extends Annotation> annotationType) {
             this.annotationType = Objects.requireNonNull(annotationType, "annotation type");
             checkForRuntimeRetention(annotationType);
         }
@@ -152,7 +152,7 @@ public class Matchers {
     private static class AnnotatedWith extends AbstractMatcher<AnnotatedElement> {
         private final Annotation annotation;
 
-        public AnnotatedWith(Annotation annotation) {
+        AnnotatedWith(Annotation annotation) {
             this.annotation = Objects.requireNonNull(annotation, "annotation");
             checkForRuntimeRetention(annotation.annotationType());
         }
@@ -191,7 +191,7 @@ public class Matchers {
     private static class SubclassesOf extends AbstractMatcher<Class> {
         private final Class<?> superclass;
 
-        public SubclassesOf(Class<?> superclass) {
+        SubclassesOf(Class<?> superclass) {
             this.superclass = Objects.requireNonNull(superclass, "superclass");
         }
 
@@ -227,7 +227,7 @@ public class Matchers {
     private static class Only extends AbstractMatcher<Object> {
         private final Object value;
 
-        public Only(Object value) {
+        Only(Object value) {
             this.value = Objects.requireNonNull(value, "value");
         }
 
@@ -263,7 +263,7 @@ public class Matchers {
     private static class IdenticalTo extends AbstractMatcher<Object> {
         private final Object value;
 
-        public IdenticalTo(Object value) {
+        IdenticalTo(Object value) {
             this.value = Objects.requireNonNull(value, "value");
         }
 
@@ -301,7 +301,7 @@ public class Matchers {
         private final transient Package targetPackage;
         private final String packageName;
 
-        public InPackage(Package targetPackage) {
+        InPackage(Package targetPackage) {
             this.targetPackage = Objects.requireNonNull(targetPackage, "package");
             this.packageName = targetPackage.getName();
         }
@@ -345,7 +345,7 @@ public class Matchers {
     private static class InSubpackage extends AbstractMatcher<Class>  {
         private final String targetPackageName;
 
-        public InSubpackage(String targetPackageName) {
+        InSubpackage(String targetPackageName) {
             this.targetPackageName = targetPackageName;
         }
 
@@ -384,7 +384,7 @@ public class Matchers {
     private static class Returns extends AbstractMatcher<Method> {
         private final Matcher<? super Class<?>> returnType;
 
-        public Returns(Matcher<? super Class<?>> returnType) {
+        Returns(Matcher<? super Class<?>> returnType) {
             this.returnType = Objects.requireNonNull(returnType, "return type matcher");
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java b/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
index e4cc088e30a..eb3fb00a51a 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/name/NamedImpl.java
@@ -23,7 +23,7 @@ class NamedImpl implements Named {
 
     private final String value;
 
-    public NamedImpl(String value) {
+    NamedImpl(String value) {
         this.value = Objects.requireNonNull(value, "name");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
index 55da169e620..5db80a711ee 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NamedWriteableAwareStreamInput.java
@@ -36,14 +36,20 @@ public class NamedWriteableAwareStreamInput extends FilterStreamInput {
     @Override
     public <C extends NamedWriteable> C readNamedWriteable(Class<C> categoryClass) throws IOException {
         String name = readString();
+        return readNamedWriteable(categoryClass, name);
+    }
+
+    @Override
+    public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused") Class<C> categoryClass,
+                                                           @SuppressWarnings("unused") String name) throws IOException {
         Writeable.Reader<? extends C> reader = namedWriteableRegistry.getReader(categoryClass, name);
         C c = reader.read(this);
         if (c == null) {
             throw new IOException(
-                    "Writeable.Reader [" + reader + "] returned null which is not allowed and probably means it screwed up the stream.");
+                "Writeable.Reader [" + reader + "] returned null which is not allowed and probably means it screwed up the stream.");
         }
         assert name.equals(c.getWriteableName()) : c + " claims to have a different name [" + c.getWriteableName()
-                + "] than it was read from [" + name + "].";
+            + "] than it was read from [" + name + "].";
         return c;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
index b25d4bc9b72..fd4a215eabf 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
@@ -38,8 +38,7 @@ public final class NotSerializableExceptionWrapper extends ElasticsearchExceptio
     private final RestStatus status;
 
     public NotSerializableExceptionWrapper(Throwable other) {
-        super(ElasticsearchException.getExceptionName(other) +
-                        ": " + other.getMessage(), other.getCause());
+        super(ElasticsearchException.getExceptionName(other) + ": " + other.getMessage(), other.getCause());
         this.name = ElasticsearchException.getExceptionName(other);
         this.status = ExceptionsHelper.status(other);
         setStackTrace(other.getStackTrace());
@@ -51,6 +50,9 @@ public final class NotSerializableExceptionWrapper extends ElasticsearchExceptio
             for (String key : ex.getHeaderKeys()) {
                 this.addHeader(key, ex.getHeader(key));
             }
+            for (String key : ex.getMetadataKeys()) {
+                this.addMetadata(key, ex.getMetadata(key));
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index f7a8a2ff1a6..e33c3ed840a 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -214,9 +214,8 @@ public abstract class StreamInput extends InputStream {
     }
 
     /**
-     * Reads a long stored in variable-length format.  Reads between one and
-     * nine bytes.  Smaller values take fewer bytes.  Negative numbers are not
-     * supported.
+     * Reads a long stored in variable-length format. Reads between one and ten bytes. Smaller values take fewer bytes. Negative numbers
+     * are encoded in ten bytes so prefer {@link #readLong()} or {@link #readZLong()} for negative numbers.
      */
     public long readVLong() throws IOException {
         byte b = readByte();
@@ -260,8 +259,16 @@ public abstract class StreamInput extends InputStream {
             return i;
         }
         b = readByte();
-        assert (b & 0x80) == 0;
-        return i | ((b & 0x7FL) << 56);
+        i |= ((b & 0x7FL) << 56);
+        if ((b & 0x80) == 0) {
+            return i;
+        }
+        b = readByte();
+        if (b != 0 && b != 1) {
+            throw new IOException("Invalid vlong (" + Integer.toHexString(b) + " << 63) | " + Long.toHexString(i));
+        }
+        i |= ((long) b) << 63;
+        return i;
     }
 
     public long readZLong() throws IOException {
@@ -818,6 +825,22 @@ public abstract class StreamInput extends InputStream {
         throw new UnsupportedOperationException("can't read named writeable from StreamInput");
     }
 
+    /**
+     * Reads a {@link NamedWriteable} from the current stream with the given name. It is assumed that the caller obtained the name
+     * from other source, so it's not read from the stream. The name is used for looking for
+     * the corresponding entry in the registry by name, so that the proper object can be read and returned.
+     * Default implementation throws {@link UnsupportedOperationException} as StreamInput doesn't hold a registry.
+     * Use {@link FilterInputStream} instead which wraps a stream and supports a {@link NamedWriteableRegistry} too.
+     *
+     * Prefer {@link StreamInput#readNamedWriteable(Class)} and {@link StreamOutput#writeNamedWriteable(NamedWriteable)} unless you
+     * have a compelling reason to use this method instead.
+     */
+    @Nullable
+    public <C extends NamedWriteable> C readNamedWriteable(@SuppressWarnings("unused") Class<C> categoryClass,
+                                                           @SuppressWarnings("unused") String name) throws IOException {
+        throw new UnsupportedOperationException("can't read named writeable from StreamInput");
+    }
+
     /**
      * Reads an optional {@link NamedWriteable}.
      */
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 4fc253cf45d..4d57e7c1b88 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -210,12 +210,22 @@ public abstract class StreamOutput extends OutputStream {
     }
 
     /**
-     * Writes a non-negative long in a variable-length format.
-     * Writes between one and nine bytes. Smaller values take fewer bytes.
-     * Negative numbers are not supported.
+     * Writes a non-negative long in a variable-length format. Writes between one and ten bytes. Smaller values take fewer bytes. Negative
+     * numbers use ten bytes and trip assertions (if running in tests) so prefer {@link #writeLong(long)} or {@link #writeZLong(long)} for
+     * negative numbers.
      */
     public void writeVLong(long i) throws IOException {
-        assert i >= 0;
+        if (i < 0) {
+            throw new IllegalStateException("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + i + "]");
+        }
+        writeVLongNoCheck(i);
+    }
+
+    /**
+     * Writes a long in a variable-length format without first checking if it is negative. Package private for testing. Use
+     * {@link #writeVLong(long)} instead.
+     */
+    void writeVLongNoCheck(long i) throws IOException {
         while ((i & ~0x7F) != 0) {
             writeByte((byte) ((i & 0x7f) | 0x80));
             i >>>= 7;
@@ -328,7 +338,7 @@ public abstract class StreamOutput extends OutputStream {
             // make sure any possible char can fit into the buffer in any possible iteration
             // we need at most 3 bytes so we flush the buffer once we have less than 3 bytes
             // left before we start another iteration
-            if (offset > buffer.length-3) {
+            if (offset > buffer.length - 3) {
                 writeBytes(buffer, offset);
                 offset = 0;
             }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
index 30607f33759..9d645038d65 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
@@ -34,7 +34,7 @@ public interface Writeable {
     /**
      * Write this into the {@linkplain StreamOutput}.
      */
-    void writeTo(final StreamOutput out) throws IOException;
+    void writeTo(StreamOutput out) throws IOException;
 
     /**
      * Reference to a method that can write some object to a {@link StreamOutput}.
@@ -60,7 +60,7 @@ public interface Writeable {
          * @param out Output to write the {@code value} too
          * @param value The value to add
          */
-        void write(final StreamOutput out, final V value) throws IOException;
+        void write(StreamOutput out, V value) throws IOException;
 
     }
 
@@ -86,7 +86,7 @@ public interface Writeable {
          *
          * @param in Input to read the value from
          */
-        V read(final StreamInput in) throws IOException;
+        V read(StreamInput in) throws IOException;
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index ace0569a14a..39b882f1a1d 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -21,11 +21,11 @@ package org.elasticsearch.common.logging;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 
 import java.util.Iterator;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArraySet;
 
@@ -42,7 +42,7 @@ public class DeprecationLogger {
      *
      * https://tools.ietf.org/html/rfc7234#section-5.5
      */
-    public static final String DEPRECATION_HEADER = "Warning";
+    public static final String WARNING_HEADER = "Warning";
 
     /**
      * This is set once by the {@code Node} constructor, but it uses {@link CopyOnWriteArraySet} to ensure that tests can run in parallel.
@@ -64,7 +64,7 @@ public class DeprecationLogger {
      * @throws IllegalStateException if this {@code threadContext} has already been set
      */
     public static void setThreadContext(ThreadContext threadContext) {
-        assert threadContext != null;
+        Objects.requireNonNull(threadContext, "Cannot register a null ThreadContext");
 
         // add returning false means it _did_ have it already
         if (THREAD_CONTEXT.add(threadContext) == false) {
@@ -128,7 +128,7 @@ public class DeprecationLogger {
 
             while (iterator.hasNext()) {
                 try {
-                    iterator.next().addResponseHeader(DEPRECATION_HEADER, formattedMessage);
+                    iterator.next().addResponseHeader(WARNING_HEADER, formattedMessage);
                 } catch (IllegalStateException e) {
                     // ignored; it should be removed shortly
                 }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
index 548c1da5a8c..16f47f78ddb 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/ESLoggerFactory.java
@@ -38,8 +38,8 @@ public final class ESLoggerFactory {
     public static final Setting<Level> LOG_DEFAULT_LEVEL_SETTING =
         new Setting<>("logger.level", Level.INFO.name(), Level::valueOf, Property.NodeScope);
     public static final Setting<Level> LOG_LEVEL_SETTING =
-        Setting.prefixKeySetting("logger.", Level.INFO.name(), Level::valueOf,
-            Property.Dynamic, Property.NodeScope);
+        Setting.prefixKeySetting("logger.", (key) -> new Setting<>(key, Level.INFO.name(), Level::valueOf, Property.Dynamic,
+            Property.NodeScope));
 
     public static Logger getLogger(String prefix, String name) {
         return getLogger(prefix, LogManager.getLogger(name));
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index 428e3ce7964..5e20b6c37e3 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -36,6 +36,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
 
 import java.io.IOException;
 import java.nio.file.FileVisitOption;
@@ -97,7 +98,7 @@ public class LogConfigurator {
         final Set<FileVisitOption> options = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
         Files.walkFileTree(configsPath, options, Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
             @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+            public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
                 if (file.getFileName().toString().equals("log4j2.properties")) {
                     configurations.add((PropertiesConfiguration) factory.getConfiguration(context, file.toString(), file.toUri()));
                 }
@@ -122,23 +123,53 @@ public class LogConfigurator {
         Configurator.initialize(builder.build());
     }
 
-    private static void configureLoggerLevels(Settings settings) {
+    /**
+     * Configures the logging levels for loggers configured in the specified settings.
+     *
+     * @param settings the settings from which logger levels will be extracted
+     */
+    private static void configureLoggerLevels(final Settings settings) {
         if (ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.exists(settings)) {
             final Level level = ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.get(settings);
             Loggers.setLevel(ESLoggerFactory.getRootLogger(), level);
         }
 
         final Map<String, String> levels = settings.filter(ESLoggerFactory.LOG_LEVEL_SETTING::match).getAsMap();
-        for (String key : levels.keySet()) {
-            final Level level = ESLoggerFactory.LOG_LEVEL_SETTING.getConcreteSetting(key).get(settings);
-            Loggers.setLevel(ESLoggerFactory.getLogger(key.substring("logger.".length())), level);
+        for (final String key : levels.keySet()) {
+            // do not set a log level for a logger named level (from the default log setting)
+            if (!key.equals(ESLoggerFactory.LOG_DEFAULT_LEVEL_SETTING.getKey())) {
+                final Level level = ESLoggerFactory.LOG_LEVEL_SETTING.getConcreteSetting(key).get(settings);
+                Loggers.setLevel(ESLoggerFactory.getLogger(key.substring("logger.".length())), level);
+            }
         }
     }
 
-
+    /**
+     * Set system properties that can be used in configuration files to specify paths and file patterns for log files. We expose three
+     * properties here:
+     * <ul>
+     * <li>
+     * {@code es.logs.base_path} the base path containing the log files
+     * </li>
+     * <li>
+     * {@code es.logs.cluster_name} the cluster name, used as the prefix of log filenames in the default configuration
+     * </li>
+     * <li>
+     * {@code es.logs.node_name} the node name, can be used as part of log filenames (only exposed if {@link Node#NODE_NAME_SETTING} is
+     * explicitly set)
+     * </li>
+     * </ul>
+     *
+     * @param logsPath the path to the log files
+     * @param settings the settings to extract the cluster and node names
+     */
     @SuppressForbidden(reason = "sets system property for logging configuration")
     private static void setLogConfigurationSystemProperty(final Path logsPath, final Settings settings) {
-        System.setProperty("es.logs", logsPath.resolve(ClusterName.CLUSTER_NAME_SETTING.get(settings).value()).toString());
+        System.setProperty("es.logs.base_path", logsPath.toString());
+        System.setProperty("es.logs.cluster_name", ClusterName.CLUSTER_NAME_SETTING.get(settings).value());
+        if (Node.NODE_NAME_SETTING.exists(settings)) {
+            System.setProperty("es.logs.node_name", Node.NODE_NAME_SETTING.get(settings));
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index e3d3bf1c624..2a539a84981 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -56,6 +56,7 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopFieldDocs;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -280,31 +281,8 @@ public class Lucene {
     }
 
     public static TopDocs readTopDocs(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            int totalHits = in.readVInt();
-            float maxScore = in.readFloat();
-
-            SortField[] fields = new SortField[in.readVInt()];
-            for (int i = 0; i < fields.length; i++) {
-                String field = null;
-                if (in.readBoolean()) {
-                    field = in.readString();
-                }
-                SortField.Type sortType = readSortType(in);
-                Object missingValue = readMissingValue(in);
-                boolean reverse = in.readBoolean();
-                fields[i] = new SortField(field, sortType, reverse);
-                if (missingValue != null) {
-                    fields[i].setMissingValue(missingValue);
-                }
-            }
-
-            FieldDoc[] fieldDocs = new FieldDoc[in.readVInt()];
-            for (int i = 0; i < fieldDocs.length; i++) {
-                fieldDocs[i] = readFieldDoc(in);
-            }
-            return new TopFieldDocs(totalHits, fieldDocs, fields, maxScore);
-        } else {
+        byte type = in.readByte();
+        if (type == 0) {
             int totalHits = in.readVInt();
             float maxScore = in.readFloat();
 
@@ -313,6 +291,39 @@ public class Lucene {
                 scoreDocs[i] = new ScoreDoc(in.readVInt(), in.readFloat());
             }
             return new TopDocs(totalHits, scoreDocs, maxScore);
+        } else if (type == 1) {
+            int totalHits = in.readVInt();
+            float maxScore = in.readFloat();
+
+            SortField[] fields = new SortField[in.readVInt()];
+            for (int i = 0; i < fields.length; i++) {
+                fields[i] = readSortField(in);
+            }
+
+            FieldDoc[] fieldDocs = new FieldDoc[in.readVInt()];
+            for (int i = 0; i < fieldDocs.length; i++) {
+                fieldDocs[i] = readFieldDoc(in);
+            }
+            return new TopFieldDocs(totalHits, fieldDocs, fields, maxScore);
+        } else if (type == 2) {
+            int totalHits = in.readVInt();
+            float maxScore = in.readFloat();
+
+            String field = in.readString();
+            SortField[] fields = new SortField[in.readVInt()];
+            for (int i = 0; i < fields.length; i++) {
+               fields[i] = readSortField(in);
+            }
+            int size = in.readVInt();
+            Object[] collapseValues = new Object[size];
+            FieldDoc[] fieldDocs = new FieldDoc[size];
+            for (int i = 0; i < fieldDocs.length; i++) {
+                fieldDocs[i] = readFieldDoc(in);
+                collapseValues[i] = readSortValue(in);
+            }
+            return new CollapseTopFieldDocs(field, totalHits, fieldDocs, fields, collapseValues, maxScore);
+        } else {
+            throw new IllegalStateException("Unknown type " + type);
         }
     }
 
@@ -347,6 +358,33 @@ public class Lucene {
         return new FieldDoc(in.readVInt(), in.readFloat(), cFields);
     }
 
+    private static Comparable readSortValue(StreamInput in) throws IOException {
+        byte type = in.readByte();
+        if (type == 0) {
+            return null;
+        } else if (type == 1) {
+            return in.readString();
+        } else if (type == 2) {
+            return in.readInt();
+        } else if (type == 3) {
+            return in.readLong();
+        } else if (type == 4) {
+            return in.readFloat();
+        } else if (type == 5) {
+            return in.readDouble();
+        } else if (type == 6) {
+            return in.readByte();
+        } else if (type == 7) {
+            return in.readShort();
+        } else if (type == 8) {
+            return in.readBoolean();
+        } else if (type == 9) {
+            return in.readBytesRef();
+        } else {
+            throw new IOException("Can't match type [" + type + "]");
+        }
+    }
+
     public static ScoreDoc readScoreDoc(StreamInput in) throws IOException {
         return new ScoreDoc(in.readVInt(), in.readFloat());
     }
@@ -354,8 +392,28 @@ public class Lucene {
     private static final Class<?> GEO_DISTANCE_SORT_TYPE_CLASS = LatLonDocValuesField.newDistanceSort("some_geo_field", 0, 0).getClass();
 
     public static void writeTopDocs(StreamOutput out, TopDocs topDocs) throws IOException {
-        if (topDocs instanceof TopFieldDocs) {
-            out.writeBoolean(true);
+        if (topDocs instanceof CollapseTopFieldDocs) {
+            out.writeByte((byte) 2);
+            CollapseTopFieldDocs collapseDocs = (CollapseTopFieldDocs) topDocs;
+
+            out.writeVInt(topDocs.totalHits);
+            out.writeFloat(topDocs.getMaxScore());
+
+            out.writeString(collapseDocs.field);
+
+            out.writeVInt(collapseDocs.fields.length);
+            for (SortField sortField : collapseDocs.fields) {
+               writeSortField(out, sortField);
+            }
+
+            out.writeVInt(topDocs.scoreDocs.length);
+            for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+                ScoreDoc doc = collapseDocs.scoreDocs[i];
+                writeFieldDoc(out, (FieldDoc) doc);
+                writeSortValue(out, collapseDocs.collapseValues[i]);
+            }
+        } else if (topDocs instanceof TopFieldDocs) {
+            out.writeByte((byte) 1);
             TopFieldDocs topFieldDocs = (TopFieldDocs) topDocs;
 
             out.writeVInt(topDocs.totalHits);
@@ -363,31 +421,7 @@ public class Lucene {
 
             out.writeVInt(topFieldDocs.fields.length);
             for (SortField sortField : topFieldDocs.fields) {
-                if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) {
-                    // for geo sorting, we replace the SortField with a SortField that assumes a double field.
-                    // this works since the SortField is only used for merging top docs
-                    SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE);
-                    newSortField.setMissingValue(sortField.getMissingValue());
-                    sortField = newSortField;
-                }
-                if (sortField.getClass() != SortField.class) {
-                    throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]");
-                }
-                if (sortField.getField() == null) {
-                    out.writeBoolean(false);
-                } else {
-                    out.writeBoolean(true);
-                    out.writeString(sortField.getField());
-                }
-                if (sortField.getComparatorSource() != null) {
-                    IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
-                    writeSortType(out, comparatorSource.reducedType());
-                    writeMissingValue(out, comparatorSource.missingValue(sortField.getReverse()));
-                } else {
-                    writeSortType(out, sortField.getType());
-                    writeMissingValue(out, sortField.getMissingValue());
-                }
-                out.writeBoolean(sortField.getReverse());
+              writeSortField(out, sortField);
             }
 
             out.writeVInt(topDocs.scoreDocs.length);
@@ -395,7 +429,7 @@ public class Lucene {
                 writeFieldDoc(out, (FieldDoc) doc);
             }
         } else {
-            out.writeBoolean(false);
+            out.writeByte((byte) 0);
             out.writeVInt(topDocs.totalHits);
             out.writeFloat(topDocs.getMaxScore());
 
@@ -431,44 +465,49 @@ public class Lucene {
         }
     }
 
+
+    private static void writeSortValue(StreamOutput out, Object field) throws IOException {
+        if (field == null) {
+            out.writeByte((byte) 0);
+        } else {
+            Class type = field.getClass();
+            if (type == String.class) {
+                out.writeByte((byte) 1);
+                out.writeString((String) field);
+            } else if (type == Integer.class) {
+                out.writeByte((byte) 2);
+                out.writeInt((Integer) field);
+            } else if (type == Long.class) {
+                out.writeByte((byte) 3);
+                out.writeLong((Long) field);
+            } else if (type == Float.class) {
+                out.writeByte((byte) 4);
+                out.writeFloat((Float) field);
+            } else if (type == Double.class) {
+                out.writeByte((byte) 5);
+                out.writeDouble((Double) field);
+            } else if (type == Byte.class) {
+                out.writeByte((byte) 6);
+                out.writeByte((Byte) field);
+            } else if (type == Short.class) {
+                out.writeByte((byte) 7);
+                out.writeShort((Short) field);
+            } else if (type == Boolean.class) {
+                out.writeByte((byte) 8);
+                out.writeBoolean((Boolean) field);
+            } else if (type == BytesRef.class) {
+                out.writeByte((byte) 9);
+                out.writeBytesRef((BytesRef) field);
+            } else {
+                throw new IOException("Can't handle sort field value of type [" + type + "]");
+            }
+        }
+    }
+
     public static void writeFieldDoc(StreamOutput out, FieldDoc fieldDoc) throws IOException {
         out.writeVInt(fieldDoc.fields.length);
         for (Object field : fieldDoc.fields) {
-            if (field == null) {
-                out.writeByte((byte) 0);
-            } else {
-                Class type = field.getClass();
-                if (type == String.class) {
-                    out.writeByte((byte) 1);
-                    out.writeString((String) field);
-                } else if (type == Integer.class) {
-                    out.writeByte((byte) 2);
-                    out.writeInt((Integer) field);
-                } else if (type == Long.class) {
-                    out.writeByte((byte) 3);
-                    out.writeLong((Long) field);
-                } else if (type == Float.class) {
-                    out.writeByte((byte) 4);
-                    out.writeFloat((Float) field);
-                } else if (type == Double.class) {
-                    out.writeByte((byte) 5);
-                    out.writeDouble((Double) field);
-                } else if (type == Byte.class) {
-                    out.writeByte((byte) 6);
-                    out.writeByte((Byte) field);
-                } else if (type == Short.class) {
-                    out.writeByte((byte) 7);
-                    out.writeShort((Short) field);
-                } else if (type == Boolean.class) {
-                    out.writeByte((byte) 8);
-                    out.writeBoolean((Boolean) field);
-                } else if (type == BytesRef.class) {
-                    out.writeByte((byte) 9);
-                    out.writeBytesRef((BytesRef) field);
-                } else {
-                    throw new IOException("Can't handle sort field value of type [" + type + "]");
-                }
-            }
+            writeSortValue(out, field);
         }
         out.writeVInt(fieldDoc.doc);
         out.writeFloat(fieldDoc.score);
@@ -487,10 +526,53 @@ public class Lucene {
         return SortField.Type.values()[in.readVInt()];
     }
 
+    public static SortField readSortField(StreamInput in) throws IOException {
+        String field = null;
+        if (in.readBoolean()) {
+            field = in.readString();
+        }
+        SortField.Type sortType = readSortType(in);
+        Object missingValue = readMissingValue(in);
+        boolean reverse = in.readBoolean();
+        SortField sortField = new SortField(field, sortType, reverse);
+        if (missingValue != null) {
+            sortField.setMissingValue(missingValue);
+        }
+        return sortField;
+    }
+
     public static void writeSortType(StreamOutput out, SortField.Type sortType) throws IOException {
         out.writeVInt(sortType.ordinal());
     }
 
+    public static void writeSortField(StreamOutput out, SortField sortField) throws IOException {
+        if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) {
+            // for geo sorting, we replace the SortField with a SortField that assumes a double field.
+            // this works since the SortField is only used for merging top docs
+            SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE);
+            newSortField.setMissingValue(sortField.getMissingValue());
+            sortField = newSortField;
+        }
+        if (sortField.getClass() != SortField.class) {
+            throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]");
+        }
+        if (sortField.getField() == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            out.writeString(sortField.getField());
+        }
+        if (sortField.getComparatorSource() != null) {
+            IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
+            writeSortType(out, comparatorSource.reducedType());
+            writeMissingValue(out, comparatorSource.missingValue(sortField.getReverse()));
+        } else {
+            writeSortType(out, sortField.getType());
+            writeMissingValue(out, sortField.getMissingValue());
+        }
+        out.writeBoolean(sortField.getReverse());
+    }
+
     public static Explanation readExplanation(StreamInput in) throws IOException {
         boolean match = in.readBoolean();
         String description = in.readString();
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
index 146fb7ba05e..38e7691f398 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java
@@ -29,9 +29,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * A map between segment core cache keys and the shard that these segments
@@ -50,7 +50,7 @@ public final class ShardCoreKeyMap {
     private final Map<String, Set<Object>> indexToCoreKey;
 
     public ShardCoreKeyMap() {
-        coreKeyToShard = new IdentityHashMap<>();
+        coreKeyToShard = new ConcurrentHashMap<>();
         indexToCoreKey = new HashMap<>();
     }
 
@@ -64,9 +64,17 @@ public final class ShardCoreKeyMap {
             throw new IllegalArgumentException("Could not extract shard id from " + reader);
         }
         final Object coreKey = reader.getCoreCacheKey();
+
+        if (coreKeyToShard.containsKey(coreKey)) {
+            // Do this check before entering the synchronized block in order to
+            // avoid taking the mutex if possible (which should happen most of
+            // the time).
+            return;
+        }
+
         final String index = shardId.getIndexName();
         synchronized (this) {
-            if (coreKeyToShard.put(coreKey, shardId) == null) {
+            if (coreKeyToShard.containsKey(coreKey) == false) {
                 Set<Object> objects = indexToCoreKey.get(index);
                 if (objects == null) {
                     objects = new HashSet<>();
@@ -90,6 +98,14 @@ public final class ShardCoreKeyMap {
                 try {
                     reader.addCoreClosedListener(listener);
                     addedListener = true;
+
+                    // Only add the core key to the map as a last operation so that
+                    // if another thread sees that the core key is already in the
+                    // map (like the check just before this synchronized block),
+                    // then it means that the closed listener has already been
+                    // registered.
+                    ShardId previous = coreKeyToShard.put(coreKey, shardId);
+                    assert previous == null;
                 } finally {
                     if (false == addedListener) {
                         try {
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
index 269c1c55eec..5307a417e10 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
@@ -32,7 +32,6 @@ import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
@@ -87,21 +86,18 @@ public final class AllTermQuery extends Query {
         if (rewritten != this) {
             return rewritten;
         }
-        boolean fieldExists = false;
         boolean hasPayloads = false;
         for (LeafReaderContext context : reader.leaves()) {
             final Terms terms = context.reader().terms(term.field());
             if (terms != null) {
-                fieldExists = true;
                 if (terms.hasPayloads()) {
                     hasPayloads = true;
                     break;
                 }
             }
         }
-        if (fieldExists == false) {
-            return new MatchNoDocsQuery();
-        }
+        // if the terms does not exist we could return a MatchNoDocsQuery but this would break the unified highlighter
+        // which rewrites query with an empty reader.
         if (hasPayloads == false) {
             return new TermQuery(term);
         }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 87bfdacb1c7..a76428e829a 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -25,6 +25,8 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.Query;
@@ -115,6 +117,20 @@ public class MultiPhrasePrefixQuery extends Query {
         positions.add(position);
     }
 
+    /**
+     * Returns the terms for each position in this phrase
+     */
+    public Term[][] getTerms() {
+        Term[][] terms = new Term[termArrays.size()][];
+        for (int i = 0; i < termArrays.size(); i++) {
+            terms[i] = new Term[termArrays.get(i).length];
+            for (int j = 0; j < termArrays.get(i).length; j++) {
+                terms[i][j] = termArrays.get(i)[j];
+            }
+        }
+        return terms;
+    }
+
     /**
      * Returns the relative positions of terms in this phrase.
      */
@@ -150,7 +166,12 @@ public class MultiPhrasePrefixQuery extends Query {
             }
         }
         if (terms.isEmpty()) {
-            return Queries.newMatchNoDocsQuery("No terms supplied for " + MultiPhrasePrefixQuery.class.getName());
+            // if the terms does not exist we could return a MatchNoDocsQuery but this would break the unified highlighter
+            // which rewrites query with an empty reader.
+            return new BooleanQuery.Builder()
+                .add(query.build(), BooleanClause.Occur.MUST)
+                .add(Queries.newMatchNoDocsQuery("No terms supplied for " + MultiPhrasePrefixQuery.class.getName()),
+                    BooleanClause.Occur.MUST).build();
         }
         query.add(terms.toArray(Term.class), position);
         return query.build();
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
index 8933b56b124..a29b249375e 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.common.lucene.search;
 
 import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PrefixQuery;
@@ -135,6 +137,33 @@ public class Queries {
         }
     }
 
+    /**
+     * Potentially apply minimum should match value if we have a query that it can be applied to,
+     * otherwise return the original query.
+     */
+    public static Query maybeApplyMinimumShouldMatch(Query query, @Nullable String minimumShouldMatch) {
+        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
+        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
+        // and multiple variations of the same word in the query (synonyms for instance).
+        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
+            return applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
+        } else if (query instanceof ExtendedCommonTermsQuery) {
+            ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch);
+        } else if (query instanceof GraphQuery && ((GraphQuery) query).hasBoolean()) {
+            // we have a graph query that has at least one boolean sub-query
+            // re-build and set minimum should match value on all boolean queries
+            List<Query> oldQueries = ((GraphQuery) query).getQueries();
+            Query[] queries = new Query[oldQueries.size()];
+            for (int i = 0; i < queries.length; i++) {
+                queries[i] = maybeApplyMinimumShouldMatch(oldQueries.get(i), minimumShouldMatch);
+            }
+
+            return new GraphQuery(queries);
+        }
+
+        return query;
+    }
+
     private static Pattern spaceAroundLessThanPattern = Pattern.compile("(\\s+<\\s*)|(\\s*<\\s+)");
     private static Pattern spacePattern = Pattern.compile(" ");
     private static Pattern lessThanPattern = Pattern.compile("<");
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
index fd7c8f6c49d..fe12622748e 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
@@ -37,7 +37,6 @@ import org.elasticsearch.common.lucene.Lucene;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -157,7 +156,7 @@ public class FiltersFunctionScoreQuery extends Query {
         final Weight[] filterWeights;
         final boolean needsScores;
 
-        public CustomBoostFactorWeight(Query parent, Weight subQueryWeight, Weight[] filterWeights, boolean needsScores) throws IOException {
+        CustomBoostFactorWeight(Query parent, Weight subQueryWeight, Weight[] filterWeights, boolean needsScores) throws IOException {
             super(parent);
             this.subQueryWeight = subQueryWeight;
             this.filterWeights = filterWeights;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index 5e94e82021f..61de1ab303f 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -109,7 +109,7 @@ public class FunctionScoreQuery extends Query {
         final Weight subQueryWeight;
         final boolean needsScores;
 
-        public CustomBoostFactorWeight(Query parent, Weight subQueryWeight, boolean needsScores) throws IOException {
+        CustomBoostFactorWeight(Query parent, Weight subQueryWeight, boolean needsScores) throws IOException {
             super(parent);
             this.subQueryWeight = subQueryWeight;
             this.needsScores = needsScores;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
index 47b85250735..bee7087c1d5 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
@@ -38,7 +38,7 @@ public class ScriptScoreFunction extends ScoreFunction {
         protected int docid;
         protected float score;
 
-        public CannedScorer() {
+        CannedScorer() {
             super(null);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
index bfe36b0a060..44c6245a812 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/WeightFactorFunction.java
@@ -68,7 +68,7 @@ public class WeightFactorFunction extends ScoreFunction {
 
     @Override
     public boolean needsScores() {
-        return false;
+        return scoreFunction.needsScores();
     }
 
     public Explanation explainWeight() {
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
index 67f06c4f8d0..6e4e3e1923d 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
@@ -57,7 +57,7 @@ final class PerThreadIDAndVersionLookup {
     /**
      * Initialize lookup for the provided segment
      */
-    public PerThreadIDAndVersionLookup(LeafReader reader) throws IOException {
+    PerThreadIDAndVersionLookup(LeafReader reader) throws IOException {
         TermsEnum termsEnum = null;
         NumericDocValues versions = null;
 
diff --git a/core/src/main/java/org/elasticsearch/common/network/IfConfig.java b/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
index 7fd4cc6d2f3..8ad85150299 100644
--- a/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
+++ b/core/src/main/java/org/elasticsearch/common/network/IfConfig.java
@@ -34,17 +34,17 @@ import java.util.Locale;
 /**
  * Simple class to log {@code ifconfig}-style output at DEBUG logging.
  */
-final class IfConfig {
+public final class IfConfig {
 
     private static final Logger logger = Loggers.getLogger(IfConfig.class);
     private static final String INDENT = "        ";
 
     /** log interface configuration at debug level, if its enabled */
-    static void logIfNecessary() {
+    public static void logIfNecessary() {
         if (logger.isDebugEnabled()) {
             try {
                 doLogging();
-            } catch (IOException | SecurityException e) {
+            } catch (IOException e) {
                 logger.warn("unable to gather network information", e);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 530ecefd4cf..a8356bfe10f 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -24,17 +24,18 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimary
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.NetworkPlugin;
@@ -46,6 +47,7 @@ import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestHandler;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -75,12 +77,8 @@ public final class NetworkModule {
     private final Settings settings;
     private final boolean transportClient;
 
-    private static final AllocationCommandRegistry allocationCommandRegistry = new AllocationCommandRegistry();
     private static final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
-
-    private final Map<String, Supplier<Transport>> transportFactories = new HashMap<>();
-    private final Map<String, Supplier<HttpServerTransport>> transportHttpFactories = new HashMap<>();
-    private final List<TransportInterceptor> transportIntercetors = new ArrayList<>();
+    private static final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
 
     static {
         registerAllocationCommand(CancelAllocationCommand::new, CancelAllocationCommand::fromXContent,
@@ -98,6 +96,11 @@ public final class NetworkModule {
         namedWriteables.add(
             new NamedWriteableRegistry.Entry(Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new));
     }
+
+    private final Map<String, Supplier<Transport>> transportFactories = new HashMap<>();
+    private final Map<String, Supplier<HttpServerTransport>> transportHttpFactories = new HashMap<>();
+    private final List<TransportInterceptor> transportIntercetors = new ArrayList<>();
+
     /**
      * Creates a network module that custom networking classes can be plugged into.
      * @param settings The settings for the node
@@ -107,13 +110,14 @@ public final class NetworkModule {
                          BigArrays bigArrays,
                          CircuitBreakerService circuitBreakerService,
                          NamedWriteableRegistry namedWriteableRegistry,
-                         NetworkService networkService) {
+                         NamedXContentRegistry xContentRegistry,
+                         NetworkService networkService, HttpServerTransport.Dispatcher dispatcher) {
         this.settings = settings;
         this.transportClient = transportClient;
         for (NetworkPlugin plugin : plugins) {
             if (transportClient == false && HTTP_ENABLED.get(settings)) {
                 Map<String, Supplier<HttpServerTransport>> httpTransportFactory = plugin.getHttpTransports(settings, threadPool, bigArrays,
-                    circuitBreakerService, namedWriteableRegistry, networkService);
+                    circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, dispatcher);
                 for (Map.Entry<String, Supplier<HttpServerTransport>> entry : httpTransportFactory.entrySet()) {
                     registerHttpTransport(entry.getKey(), entry.getValue());
                 }
@@ -123,7 +127,8 @@ public final class NetworkModule {
             for (Map.Entry<String, Supplier<Transport>> entry : httpTransportFactory.entrySet()) {
                 registerTransport(entry.getKey(), entry.getValue());
             }
-            List<TransportInterceptor> transportInterceptors = plugin.getTransportInterceptors(namedWriteableRegistry);
+            List<TransportInterceptor> transportInterceptors = plugin.getTransportInterceptors(namedWriteableRegistry,
+                threadPool.getThreadContext());
             for (TransportInterceptor interceptor : transportInterceptors) {
                 registerTransportInterceptor(interceptor);
             }
@@ -163,22 +168,19 @@ public final class NetworkModule {
      *        it is the name under which the command's reader is registered.
      */
     private static <T extends AllocationCommand> void registerAllocationCommand(Writeable.Reader<T> reader,
-                                                                            AllocationCommand.Parser<T> parser, ParseField commandName) {
-        allocationCommandRegistry.register(parser, commandName);
-        namedWriteables.add(new Entry(AllocationCommand.class, commandName.getPreferredName(), reader));
+            CheckedFunction<XContentParser, T, IOException> parser, ParseField commandName) {
+        namedXContents.add(new NamedXContentRegistry.Entry(AllocationCommand.class, commandName, parser));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(AllocationCommand.class, commandName.getPreferredName(), reader));
     }
 
-    /**
-     * The registry of allocation command parsers.
-     */
-    public static AllocationCommandRegistry getAllocationCommandRegistry() {
-        return allocationCommandRegistry;
-    }
-
-    public static List<Entry> getNamedWriteables() {
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return Collections.unmodifiableList(namedWriteables);
     }
 
+    public static List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return Collections.unmodifiableList(namedXContents);
+    }
+
     public Supplier<HttpServerTransport> getHttpServerTransportSupplier() {
         final String name;
         if (HTTP_TYPE_SETTING.exists(settings)) {
@@ -235,9 +237,10 @@ public final class NetworkModule {
 
         @Override
         public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                        boolean forceExecution,
                                                                                         TransportRequestHandler<T> actualHandler) {
             for (TransportInterceptor interceptor : this.transportInterceptors) {
-                actualHandler = interceptor.interceptHandler(action, executor, actualHandler);
+                actualHandler = interceptor.interceptHandler(action, executor, forceExecution, actualHandler);
             }
             return actualHandler;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
index b72acf8064c..a9d3dc4a336 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkService.java
@@ -60,12 +60,6 @@ public class NetworkService extends AbstractComponent {
             Setting.byteSizeSetting("network.tcp.send_buffer_size", new ByteSizeValue(-1), Property.NodeScope);
         public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
             Setting.byteSizeSetting("network.tcp.receive_buffer_size", new ByteSizeValue(-1), Property.NodeScope);
-        public static final Setting<Boolean> TCP_BLOCKING =
-            Setting.boolSetting("network.tcp.blocking", false, Property.NodeScope);
-        public static final Setting<Boolean> TCP_BLOCKING_SERVER =
-            Setting.boolSetting("network.tcp.blocking_server", TCP_BLOCKING, Property.NodeScope);
-        public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
-            Setting.boolSetting("network.tcp.blocking_client", TCP_BLOCKING, Property.NodeScope);
         public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
             Setting.timeSetting("network.tcp.connect_timeout", new TimeValue(30, TimeUnit.SECONDS), Property.NodeScope);
     }
@@ -90,7 +84,6 @@ public class NetworkService extends AbstractComponent {
 
     public NetworkService(Settings settings, List<CustomNameResolver> customNameResolvers) {
         super(settings);
-        IfConfig.logIfNecessary();
         this.customNameResolvers = customNameResolvers;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
index c711bfa2a61..44accbf190a 100644
--- a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
+++ b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
@@ -111,7 +111,10 @@ public class PathTrie<T> {
                 // in case the target(last) node already exist but without a value
                 // than the value should be updated.
                 if (index == (path.length - 1)) {
-                    assert (node.value == null || node.value == value);
+                    if (node.value != null) {
+                        throw new IllegalArgumentException("Path [" + String.join("/", path)+ "] already has a value ["
+                                + node.value + "]");
+                    }
                     if (node.value == null) {
                         node.value = value;
                     }
@@ -190,6 +193,9 @@ public class PathTrie<T> {
     public void insert(String path, T value) {
         String[] strings = path.split(SEPARATOR);
         if (strings.length == 0) {
+            if (rootValue != null) {
+                throw new IllegalArgumentException("Path [/] already has a value [" + rootValue + "]");
+            }
             rootValue = value;
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java b/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
index 5bac8f7bcfd..f84441fbce4 100644
--- a/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
+++ b/core/src/main/java/org/elasticsearch/common/recycler/Recyclers.java
@@ -170,7 +170,7 @@ public enum Recyclers {
                 }
             }
 
-            final int slot() {
+            int slot() {
                 final long id = Thread.currentThread().getId();
                 // don't trust Thread.hashCode to have equiprobable low bits
                 int slot = (int) BitMixer.mix64(id);
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
index 3bf4f460d71..9a3e40ac844 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
@@ -40,7 +40,7 @@ public enum DateTimeUnit {
     private final byte id;
     private final Function<DateTimeZone, DateTimeField> fieldFunction;
 
-    private DateTimeUnit(byte id, Function<DateTimeZone, DateTimeField> fieldFunction) {
+    DateTimeUnit(byte id, Function<DateTimeZone, DateTimeField> fieldFunction) {
         this.id = id;
         this.fieldFunction = fieldFunction;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 3622623987b..95333a988cd 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -195,6 +195,19 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         addSettingsUpdater(setting.newUpdater(consumer, logger, validator));
     }
 
+    /**
+     * Adds a settings consumer for affix settings. Affix settings have a namespace associated to it that needs to be available to the
+     * consumer in order to be processed correctly.
+     */
+    public synchronized <T> void addAffixUpdateConsumer(Setting.AffixSetting<T> setting,  BiConsumer<String, T> consumer,
+                                                        BiConsumer<String, T> validator) {
+        final Setting<?> registeredSetting = this.complexMatchers.get(setting.getKey());
+        if (setting != registeredSetting) {
+            throw new IllegalArgumentException("Setting is not registered for key [" + setting.getKey() + "]");
+        }
+        addSettingsUpdater(setting.newAffixUpdater(consumer, logger, validator));
+    }
+
     synchronized void addSettingsUpdater(SettingUpdater<?> updater) {
         this.settingUpdaters.add(updater);
     }
@@ -239,11 +252,9 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
      */
     public final void validate(Settings settings) {
         List<RuntimeException> exceptions = new ArrayList<>();
-        // we want them sorted for deterministic error messages
-        SortedMap<String, String> sortedSettings = new TreeMap<>(settings.getAsMap());
-        for (Map.Entry<String, String> entry : sortedSettings.entrySet()) {
+        for (String key : settings.getAsMap().keySet()) { // settings iterate in deterministic fashion
             try {
-                validate(entry.getKey(), settings);
+                validate(key, settings);
             } catch (RuntimeException ex) {
                 exceptions.add(ex);
             }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
new file mode 100644
index 00000000000..741469caa58
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/AddStringKeyStoreCommand.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli which adds a string setting.
+ */
+class AddStringKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<Void> stdinOption;
+    private final OptionSpec<Void> forceOption;
+    private final OptionSpec<String> arguments;
+
+    AddStringKeyStoreCommand() {
+        super("Add a string setting to the keystore");
+        this.stdinOption = parser.acceptsAll(Arrays.asList("x", "stdin"), "Read setting value from stdin");
+        this.forceOption = parser.acceptsAll(Arrays.asList("f", "force"), "Overwrite existing setting without prompting");
+        this.arguments = parser.nonOptions("setting name");
+    }
+
+    // pkg private so tests can manipulate
+    InputStream getStdin() {
+        return System.in;
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        String setting = arguments.value(options);
+        if (setting == null) {
+            throw new UserException(ExitCodes.USAGE, "The setting name can not be null");
+        }
+        if (keystore.getSettings().contains(setting) && options.has(forceOption) == false) {
+            if (terminal.promptYesNo("Setting " + setting + " already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without modifying keystore.");
+                return;
+            }
+        }
+
+        final char[] value;
+        if (options.has(stdinOption)) {
+            BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            value = stdinReader.readLine().toCharArray();
+        } else {
+            value = terminal.readSecret("Enter value for " + setting + ": ");
+        }
+
+        try {
+            keystore.setString(setting, value);
+        } catch (IllegalArgumentException e) {
+            throw new UserException(ExitCodes.DATA_ERROR, "String value must contain only ASCII");
+        }
+        keystore.save(env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index c9106b8cdba..69adc5878ff 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.common.settings;
 
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
+import org.elasticsearch.action.search.RemoteClusterService;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
@@ -54,9 +55,9 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.ElectMasterService;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.discovery.zen.FaultDetection;
 import org.elasticsearch.discovery.zen.UnicastZenPing;
+import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.GatewayService;
@@ -80,7 +81,6 @@ import org.elasticsearch.monitor.process.ProcessService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
@@ -237,6 +237,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     HttpTransportSettings.SETTING_CORS_ALLOW_METHODS,
                     HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS,
                     HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED,
+                    HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED,
                     HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH,
                     HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE,
                     HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE,
@@ -253,6 +254,10 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     SearchService.DEFAULT_SEARCH_TIMEOUT_SETTING,
                     ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING,
                     TransportSearchAction.SHARD_COUNT_LIMIT_SETTING,
+                    RemoteClusterService.REMOTE_CLUSTERS_SEEDS,
+                    RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER,
+                    RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING,
+                    RemoteClusterService.REMOTE_NODE_ATTRIBUTE,
                     TransportService.TRACE_LOG_EXCLUDE_SETTING,
                     TransportService.TRACE_LOG_INCLUDE_SETTING,
                     TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING,
@@ -273,7 +278,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     TcpTransport.CONNECTIONS_PER_NODE_STATE,
                     TcpTransport.CONNECTIONS_PER_NODE_PING,
                     TcpTransport.PING_SCHEDULE,
-                    TcpTransport.TCP_BLOCKING_CLIENT,
                     TcpTransport.TCP_CONNECT_TIMEOUT,
                     NetworkService.NETWORK_SERVER,
                     TcpTransport.TCP_NO_DELAY,
@@ -281,7 +285,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     TcpTransport.TCP_REUSE_ADDRESS,
                     TcpTransport.TCP_SEND_BUFFER_SIZE,
                     TcpTransport.TCP_RECEIVE_BUFFER_SIZE,
-                    TcpTransport.TCP_BLOCKING_SERVER,
                     NetworkService.GLOBAL_NETWORK_HOST_SETTING,
                     NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
                     NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING,
@@ -290,9 +293,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     NetworkService.TcpSettings.TCP_REUSE_ADDRESS,
                     NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE,
                     NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
-                    NetworkService.TcpSettings.TCP_BLOCKING,
-                    NetworkService.TcpSettings.TCP_BLOCKING_SERVER,
-                    NetworkService.TcpSettings.TCP_BLOCKING_CLIENT,
                     NetworkService.TcpSettings.TCP_CONNECT_TIMEOUT,
                     IndexSettings.QUERY_STRING_ANALYZE_WILDCARD,
                     IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD,
@@ -348,9 +348,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Node.NODE_INGEST_SETTING,
                     Node.NODE_ATTRIBUTES,
                     Node.NODE_LOCAL_STORAGE_SETTING,
-                    URLRepository.ALLOWED_URLS_SETTING,
-                    URLRepository.REPOSITORIES_URL_SETTING,
-                    URLRepository.SUPPORTED_PROTOCOLS_SETTING,
                     TransportMasterNodeReadAction.FORCE_LOCAL_SETTING,
                     AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,
                     BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX,
diff --git a/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java
new file mode 100644
index 00000000000..08860cb5ea9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/CreateKeyStoreCommand.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to create a new keystore.
+ */
+class CreateKeyStoreCommand extends EnvironmentAwareCommand {
+
+    CreateKeyStoreCommand() {
+        super("Creates a new elasticsearch keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        if (Files.exists(keystoreFile)) {
+            if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) {
+                terminal.println("Exiting without creating keystore.");
+                return;
+            }
+        }
+
+
+        char[] password = new char[0];// terminal.readSecret("Enter passphrase (empty for no passphrase): ");
+        /* TODO: uncomment when entering passwords on startup is supported
+        char[] passwordRepeat = terminal.readSecret("Enter same passphrase again: ");
+        if (Arrays.equals(password, passwordRepeat) == false) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Passphrases are not equal, exiting.");
+        }*/
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password);
+        keystore.save(env.configFile());
+        terminal.println("Created elasticsearch keystore in " + env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 959fe1849ea..1ea02252641 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -36,7 +36,7 @@ import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.seqno.LocalCheckpointService;
+import org.elasticsearch.index.seqno.LocalCheckpointTracker;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.Store;
@@ -69,6 +69,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING,
         IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING,
         IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING,
+        IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING,
         IndexMetaData.INDEX_SHADOW_REPLICAS_SETTING,
         IndexMetaData.INDEX_SHARED_FILESYSTEM_SETTING,
         IndexMetaData.INDEX_READ_ONLY_SETTING,
@@ -107,13 +108,14 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
         IndexSettings.MAX_RESULT_WINDOW_SETTING,
         IndexSettings.MAX_RESCORE_WINDOW_SETTING,
+        IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
         IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
         IndexSettings.DEFAULT_FIELD_SETTING,
         IndexSettings.QUERY_STRING_LENIENT_SETTING,
         IndexSettings.ALLOW_UNMAPPED,
         IndexSettings.INDEX_CHECK_ON_STARTUP,
         IndexSettings.INDEX_SEQ_NO_CHECKPOINT_SYNC_INTERVAL,
-        LocalCheckpointService.SETTINGS_BIT_ARRAYS_SIZE,
+        LocalCheckpointTracker.SETTINGS_BIT_ARRAYS_SIZE,
         IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
         IndexSettings.MAX_SLICES_PER_SCROLL,
         ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java
similarity index 52%
rename from core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
rename to core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java
index 3f3f805d8f7..5bded392fdb 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggesters.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java
@@ -16,25 +16,26 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.suggest;
 
-import java.util.Map;
+package org.elasticsearch.common.settings;
+
+import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.Terminal;
 
 /**
- * Registry of Suggesters. This is only its own class to make Guice happy.
+ * A cli tool for managing secrets in the elasticsearch keystore.
  */
-public final class Suggesters {
-    private final Map<String, Suggester<?>> suggesters;
+public class KeyStoreCli extends MultiCommand {
 
-    public Suggesters(Map<String, Suggester<?>> suggesters) {
-        this.suggesters = suggesters;
+    private KeyStoreCli() {
+        super("A tool for managing settings stored in the elasticsearch keystore");
+        subcommands.put("create", new CreateKeyStoreCommand());
+        subcommands.put("list", new ListKeyStoreCommand());
+        subcommands.put("add", new AddStringKeyStoreCommand());
+        subcommands.put("remove", new RemoveSettingKeyStoreCommand());
     }
 
-    public Suggester<?> getSuggester(String suggesterName) {
-        Suggester<?> suggester = suggesters.get(suggesterName);
-        if (suggester == null) {
-            throw new IllegalArgumentException("suggester with name [" + suggesterName + "] not supported");
-        }
-        return suggester;
+    public static void main(String[] args) throws Exception {
+        exit(new KeyStoreCli().main(args, Terminal.DEFAULT));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
new file mode 100644
index 00000000000..f77b63d88fa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+import javax.security.auth.DestroyFailedException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.security.GeneralSecurityException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.ElasticsearchException;
+
+/**
+ * A wrapper around a Java KeyStore which provides supplements the keystore with extra metadata.
+ *
+ * Loading a keystore has 2 phases. First, call {@link #load(Path)}. Then call
+ * {@link #decrypt(char[])} with the keystore password, or an empty char array if
+ * {@link #hasPassword()} is {@code false}.  Loading and decrypting should happen
+ * in a single thread. Once decrypted, keys may be read with the wrapper in
+ * multiple threads.
+ */
+public class KeyStoreWrapper implements SecureSettings {
+
+    /** The name of the keystore file to read and write. */
+    private static final String KEYSTORE_FILENAME = "elasticsearch.keystore";
+
+    /** The version of the metadata written before the keystore data. */
+    private static final int FORMAT_VERSION = 1;
+
+    /** The keystore type for a newly created keystore. */
+    private static final String NEW_KEYSTORE_TYPE = "PKCS12";
+
+    /** The algorithm used to store password for a newly created keystore. */
+    private static final String NEW_KEYSTORE_SECRET_KEY_ALGO = "PBE";//"PBEWithHmacSHA256AndAES_128";
+
+    /** An encoder to check whether string values are ascii. */
+    private static final CharsetEncoder ASCII_ENCODER = StandardCharsets.US_ASCII.newEncoder();
+
+    /** True iff the keystore has a password needed to read. */
+    private final boolean hasPassword;
+
+    /** The type of the keystore, as passed to {@link java.security.KeyStore#getInstance(String)} */
+    private final String type;
+
+    /** A factory necessary for constructing instances of secrets in a {@link KeyStore}. */
+    private final SecretKeyFactory secretFactory;
+
+    /** The raw bytes of the encrypted keystore. */
+    private final byte[] keystoreBytes;
+
+    /** The loaded keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore> keystore = new SetOnce<>();
+
+    /** The password for the keystore. See {@link #decrypt(char[])}. */
+    private final SetOnce<KeyStore.PasswordProtection> keystorePassword = new SetOnce<>();
+
+    /** The setting names contained in the loaded keystore. */
+    private final Set<String> settingNames = new HashSet<>();
+
+    private KeyStoreWrapper(boolean hasPassword, String type, String secretKeyAlgo, byte[] keystoreBytes) {
+        this.hasPassword = hasPassword;
+        this.type = type;
+        try {
+            secretFactory = SecretKeyFactory.getInstance(secretKeyAlgo);
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        this.keystoreBytes = keystoreBytes;
+    }
+
+    /** Returns a path representing the ES keystore in the given config dir. */
+    static Path keystorePath(Path configDir) {
+        return configDir.resolve(KEYSTORE_FILENAME);
+    }
+
+    /** Constructs a new keystore with the given password. */
+    static KeyStoreWrapper create(char[] password) throws Exception {
+        KeyStoreWrapper wrapper = new KeyStoreWrapper(password.length != 0, NEW_KEYSTORE_TYPE, NEW_KEYSTORE_SECRET_KEY_ALGO, null);
+        KeyStore keyStore = KeyStore.getInstance(NEW_KEYSTORE_TYPE);
+        keyStore.load(null, null);
+        wrapper.keystore.set(keyStore);
+        wrapper.keystorePassword.set(new KeyStore.PasswordProtection(password));
+        return wrapper;
+    }
+
+    /**
+     * Loads information about the Elasticsearch keystore from the provided config directory.
+     *
+     * {@link #decrypt(char[])} must be called before reading or writing any entries.
+     * Returns {@code null} if no keystore exists.
+     */
+    public static KeyStoreWrapper load(Path configDir) throws IOException {
+        Path keystoreFile = keystorePath(configDir);
+        if (Files.exists(keystoreFile) == false) {
+            return null;
+        }
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
+            ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
+            CodecUtil.checkHeader(input, KEYSTORE_FILENAME, FORMAT_VERSION, FORMAT_VERSION);
+            byte hasPasswordByte = input.readByte();
+            boolean hasPassword = hasPasswordByte == 1;
+            if (hasPassword == false && hasPasswordByte != 0) {
+                throw new IllegalStateException("hasPassword boolean is corrupt: "
+                    + String.format(Locale.ROOT, "%02x", hasPasswordByte));
+            }
+            String type = input.readString();
+            String secretKeyAlgo = input.readString();
+            byte[] keystoreBytes = new byte[input.readInt()];
+            input.readBytes(keystoreBytes, 0, keystoreBytes.length);
+            CodecUtil.checkFooter(input);
+            return new KeyStoreWrapper(hasPassword, type, secretKeyAlgo, keystoreBytes);
+        }
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return keystore.get() != null;
+    }
+
+    /** Return true iff calling {@link #decrypt(char[])} requires a non-empty password. */
+    public boolean hasPassword() {
+        return hasPassword;
+    }
+
+    /**
+     * Decrypts the underlying java keystore.
+     *
+     * This may only be called once. The provided password will be zeroed out.
+     */
+    public void decrypt(char[] password) throws GeneralSecurityException, IOException {
+        if (keystore.get() != null) {
+            throw new IllegalStateException("Keystore has already been decrypted");
+        }
+        keystore.set(KeyStore.getInstance(type));
+        try (InputStream in = new ByteArrayInputStream(keystoreBytes)) {
+            keystore.get().load(in, password);
+        } finally {
+            Arrays.fill(keystoreBytes, (byte)0);
+        }
+
+        keystorePassword.set(new KeyStore.PasswordProtection(password));
+        Arrays.fill(password, '\0');
+
+        // convert keystore aliases enum into a set for easy lookup
+        Enumeration<String> aliases = keystore.get().aliases();
+        while (aliases.hasMoreElements()) {
+            settingNames.add(aliases.nextElement());
+        }
+    }
+
+    /** Write the keystore to the given config directory. */
+    void save(Path configDir) throws Exception {
+        char[] password = this.keystorePassword.get().getPassword();
+
+        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
+        // write to tmp file first, then overwrite
+        String tmpFile = KEYSTORE_FILENAME + ".tmp";
+        try (IndexOutput output = directory.createOutput(tmpFile, IOContext.DEFAULT)) {
+            CodecUtil.writeHeader(output, KEYSTORE_FILENAME, FORMAT_VERSION);
+            output.writeByte(password.length == 0 ? (byte)0 : (byte)1);
+            output.writeString(type);
+            output.writeString(secretFactory.getAlgorithm());
+
+            ByteArrayOutputStream keystoreBytesStream = new ByteArrayOutputStream();
+            keystore.get().store(keystoreBytesStream, password);
+            byte[] keystoreBytes = keystoreBytesStream.toByteArray();
+            output.writeInt(keystoreBytes.length);
+            output.writeBytes(keystoreBytes, keystoreBytes.length);
+            CodecUtil.writeFooter(output);
+        }
+
+        Path keystoreFile = keystorePath(configDir);
+        Files.move(configDir.resolve(tmpFile), keystoreFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+        PosixFileAttributeView attrs = Files.getFileAttributeView(keystoreFile, PosixFileAttributeView.class);
+        if (attrs != null) {
+            // don't rely on umask: ensure the keystore has minimal permissions
+            attrs.setPermissions(PosixFilePermissions.fromString("rw-------"));
+        }
+    }
+
+    public Set<String> getSettings() {
+        return settingNames;
+    }
+
+    @Override
+    public boolean hasSetting(String setting) {
+        return settingNames.contains(setting);
+    }
+
+    // TODO: make settings accessible only to code that registered the setting
+    /** Retrieve a string setting. The {@link SecureString} should be closed once it is used. */
+    @Override
+    public SecureString getString(String setting) throws GeneralSecurityException {
+        KeyStore.Entry entry = keystore.get().getEntry(setting, keystorePassword.get());
+        if (entry instanceof KeyStore.SecretKeyEntry == false) {
+            throw new IllegalStateException("Secret setting " + setting + " is not a string");
+        }
+        // TODO: only allow getting a setting once?
+        KeyStore.SecretKeyEntry secretKeyEntry = (KeyStore.SecretKeyEntry) entry;
+        PBEKeySpec keySpec = (PBEKeySpec) secretFactory.getKeySpec(secretKeyEntry.getSecretKey(), PBEKeySpec.class);
+        SecureString value = new SecureString(keySpec.getPassword());
+        keySpec.clearPassword();
+        return value;
+    }
+
+    /**
+     * Set a string setting.
+     *
+     * @throws IllegalArgumentException if the value is not ASCII
+     */
+    void setString(String setting, char[] value) throws GeneralSecurityException {
+        if (ASCII_ENCODER.canEncode(CharBuffer.wrap(value)) == false) {
+            throw new IllegalArgumentException("Value must be ascii");
+        }
+        SecretKey secretKey = secretFactory.generateSecret(new PBEKeySpec(value));
+        keystore.get().setEntry(setting, new KeyStore.SecretKeyEntry(secretKey), keystorePassword.get());
+        settingNames.add(setting);
+    }
+
+    /** Remove the given setting from the keystore. */
+    void remove(String setting) throws KeyStoreException {
+        keystore.get().deleteEntry(setting);
+        settingNames.remove(setting);
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            if (keystorePassword.get() != null) {
+                keystorePassword.get().destroy();
+            }
+        } catch (DestroyFailedException e) {
+            throw new IOException(e);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java
new file mode 100644
index 00000000000..b0484fb1526
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/ListKeyStoreCommand.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to list all settings in the keystore.
+ */
+class ListKeyStoreCommand extends EnvironmentAwareCommand {
+
+    ListKeyStoreCommand() {
+        super("List entries in the keystore");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        List<String> sortedEntries = new ArrayList<>(keystore.getSettings());
+        Collections.sort(sortedEntries);
+        for (String entry : sortedEntries) {
+            terminal.println(entry);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java b/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java
new file mode 100644
index 00000000000..e9089b85b56
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommand.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.List;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+/**
+ * A subcommand for the keystore cli to remove a setting.
+ */
+class RemoveSettingKeyStoreCommand extends EnvironmentAwareCommand {
+
+    private final OptionSpec<String> arguments;
+
+    RemoveSettingKeyStoreCommand() {
+        super("Remove a setting from the keystore");
+        arguments = parser.nonOptions("setting names");
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
+        List<String> settings = arguments.values(options);
+        if (settings.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "Must supply at least one setting to remove");
+        }
+
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        if (keystore == null) {
+            throw new UserException(ExitCodes.DATA_ERROR, "Elasticsearch keystore not found. Use 'create' command to create one.");
+        }
+
+        keystore.decrypt(new char[0] /* TODO: prompt for password when they are supported */);
+
+        for (String setting : arguments.values(options)) {
+            if (keystore.getSettings().contains(setting) == false) {
+                throw new UserException(ExitCodes.CONFIG, "Setting [" + setting + "] does not exist in the keystore.");
+            }
+            keystore.remove(setting);
+        }
+        keystore.save(env.configFile());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
new file mode 100644
index 00000000000..c06dad2a2ea
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSetting.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.util.ArrayUtils;
+
+
+/**
+ * A secure setting.
+ *
+ * This class allows access to settings from the Elasticsearch keystore.
+ */
+public abstract class SecureSetting<T> extends Setting<T> {
+    private static final Set<Property> ALLOWED_PROPERTIES = new HashSet<>(
+        Arrays.asList(Property.Deprecated, Property.Shared)
+    );
+
+    private static final Property[] FIXED_PROPERTIES = {
+        Property.NodeScope
+    };
+
+    private static final Property[] LEGACY_PROPERTIES = {
+        Property.NodeScope, Property.Deprecated, Property.Filtered
+    };
+
+    private SecureSetting(String key, Property... properties) {
+        super(key, (String)null, null, ArrayUtils.concat(properties, FIXED_PROPERTIES, Property.class));
+        assert assertAllowedProperties(properties);
+    }
+
+    private boolean assertAllowedProperties(Setting.Property... properties) {
+        for (Setting.Property property : properties) {
+            if (ALLOWED_PROPERTIES.contains(property) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public String getDefaultRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public T getDefault(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public String getRaw(Settings settings) {
+        throw new UnsupportedOperationException("secure settings are not strings");
+    }
+
+    @Override
+    public boolean exists(Settings settings) {
+        final SecureSettings secureSettings = settings.getSecureSettings();
+        return secureSettings != null && secureSettings.hasSetting(getKey());
+    }
+
+    @Override
+    public T get(Settings settings) {
+        checkDeprecation(settings);
+        final SecureSettings secureSettings = settings.getSecureSettings();
+        if (secureSettings == null || secureSettings.hasSetting(getKey()) == false) {
+            return getFallback(settings);
+        }
+        try {
+            return getSecret(secureSettings);
+        } catch (GeneralSecurityException e) {
+            throw new RuntimeException("failed to read secure setting " + getKey(), e);
+        }
+    }
+
+    /** Returns the secret setting from the keyStoreReader store. */
+    abstract T getSecret(SecureSettings secureSettings) throws GeneralSecurityException;
+
+    /** Returns the value from a fallback setting. Returns null if no fallback exists. */
+    abstract T getFallback(Settings settings);
+
+    // TODO: override toXContent
+
+    /**
+     * A setting which contains a sensitive string.
+     *
+     * This may be any sensitive string, e.g. a username, a password, an auth token, etc.
+     */
+    public static Setting<SecureString> secureString(String name, Setting<SecureString> fallback,
+                                                     boolean allowLegacy, Property... properties) {
+        final Setting<String> legacy;
+        if (allowLegacy) {
+            Property[] legacyProperties = ArrayUtils.concat(properties, LEGACY_PROPERTIES, Property.class);
+            legacy = Setting.simpleString(name, legacyProperties);
+        } else {
+            legacy = null;
+        }
+        return new SecureSetting<SecureString>(name, properties) {
+            @Override
+            protected SecureString getSecret(SecureSettings secureSettings) throws GeneralSecurityException {
+                return secureSettings.getString(getKey());
+            }
+            @Override
+            SecureString getFallback(Settings settings) {
+                if (legacy != null && legacy.exists(settings)) {
+                    return new SecureString(legacy.get(settings).toCharArray());
+                }
+                if (fallback != null) {
+                    return fallback.get(settings);
+                }
+                return new SecureString(new char[0]); // this means "setting does not exist"
+            }
+            @Override
+            protected void checkDeprecation(Settings settings) {
+                super.checkDeprecation(settings);
+                if (legacy != null) {
+                    legacy.checkDeprecation(settings);
+                }
+            }
+            @Override
+            public boolean exists(Settings settings) {
+                // handle legacy, which is internal to this setting
+                return super.exists(settings) || legacy != null && legacy.exists(settings);
+            }
+        };
+    }
+
+
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java b/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
new file mode 100644
index 00000000000..d810e1ec4da
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureSettings.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.Closeable;
+import java.security.GeneralSecurityException;
+
+/**
+ * An accessor for settings which are securely stored. See {@link SecureSetting}.
+ */
+public interface SecureSettings extends Closeable {
+
+    /** Returns true iff the settings are loaded and retrievable. */
+    boolean isLoaded();
+
+    /** Returns true iff the given setting exists in this secure settings. */
+    boolean hasSetting(String setting);
+
+    /** Return a string setting. The {@link SecureString} should be closed once it is used. */
+    SecureString getString(String setting) throws GeneralSecurityException;
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SecureString.java b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java
new file mode 100644
index 00000000000..36982ddde1c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/settings/SecureString.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A String implementations which allows clearing the underlying char array.
+ */
+public final class SecureString implements CharSequence, Closeable {
+
+    private char[] chars;
+
+    /**
+     * Constructs a new SecureString which controls the passed in char array.
+     *
+     * Note: When this instance is closed, the array will be zeroed out.
+     */
+    public SecureString(char[] chars) {
+        this.chars = Objects.requireNonNull(chars);
+    }
+
+    /**
+     * Constructs a new SecureString from an existing String.
+     *
+     * NOTE: This is not actually secure, since the provided String cannot be deallocated, but
+     * this constructor allows for easy compatibility between new and old apis.
+     *
+     * @deprecated Only use for compatibility between deprecated string settings and new secure strings
+     */
+    @Deprecated
+    public SecureString(String s) {
+        this(s.toCharArray());
+    }
+
+    /** Constant time equality to avoid potential timing attacks. */
+    @Override
+    public synchronized boolean equals(Object o) {
+        ensureNotClosed();
+        if (this == o) return true;
+        if (o == null || o instanceof CharSequence == false) return false;
+        CharSequence that = (CharSequence) o;
+        if (chars.length != that.length()) {
+            return false;
+        }
+
+        int equals = 0;
+        for (int i = 0; i < chars.length; i++) {
+            equals |= chars[i] ^ that.charAt(i);
+        }
+
+        return equals == 0;
+    }
+
+    @Override
+    public synchronized int hashCode() {
+        return Arrays.hashCode(chars);
+    }
+
+    @Override
+    public synchronized int length() {
+        ensureNotClosed();
+        return chars.length;
+    }
+
+    @Override
+    public synchronized char charAt(int index) {
+        ensureNotClosed();
+        return chars[index];
+    }
+
+    @Override
+    public SecureString subSequence(int start, int end) {
+        throw new UnsupportedOperationException("Cannot get subsequence of SecureString");
+    }
+
+    /**
+     * Convert to a {@link String}. This should only be used with APIs that do not take {@link CharSequence}.
+     */
+    @Override
+    public synchronized String toString() {
+        return new String(chars);
+    }
+
+    /**
+     * Closes the string by clearing the underlying char array.
+     */
+    @Override
+    public synchronized void close() {
+        Arrays.fill(chars, '\0');
+        chars = null;
+    }
+
+    /** Throw an exception if this string has been closed, indicating something is trying to access the data after being closed. */
+    private void ensureNotClosed() {
+        if (chars == null) {
+            throw new IllegalStateException("SecureString has already been closed");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 22c74afee7c..f85894790a8 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.MemorySizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -41,14 +42,18 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A setting. Encapsulates typical stuff like default value, parsing, and scope.
@@ -118,7 +123,8 @@ public class Setting<T> extends ToXContentToBytes {
 
     private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings, String> defaultValue, Function<String, T> parser,
             Property... properties) {
-        assert parser.apply(defaultValue.apply(Settings.EMPTY)) != null || this.isGroupSetting(): "parser returned null";
+        assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
+               : "parser returned null";
         this.key = key;
         this.fallbackSetting = fallbackSetting;
         this.defaultValue = defaultValue;
@@ -273,7 +279,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns the default value string representation for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final String getDefaultRaw(Settings settings) {
+    public String getDefaultRaw(Settings settings) {
         return defaultValue.apply(settings);
     }
 
@@ -281,7 +287,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns the default value for this setting.
      * @param settings a settings object for settings that has a default value depending on another setting if available
      */
-    public final T getDefault(Settings settings) {
+    public T getDefault(Settings settings) {
         return parser.apply(getDefaultRaw(settings));
     }
 
@@ -289,7 +295,7 @@ public class Setting<T> extends ToXContentToBytes {
      * Returns <code>true</code> iff this setting is present in the given settings object. Otherwise <code>false</code>
      */
     public boolean exists(Settings settings) {
-        return settings.get(getKey()) != null;
+        return settings.getAsMap().containsKey(getKey());
     }
 
     /**
@@ -329,14 +335,19 @@ public class Setting<T> extends ToXContentToBytes {
      * instead. This is useful if the value can't be parsed due to an invalid value to access the actual value.
      */
     public String getRaw(Settings settings) {
+        checkDeprecation(settings);
+        return settings.get(getKey(), defaultValue.apply(settings));
+    }
+
+    /** Logs a deprecation warning if the setting is deprecated and used. */
+    protected void checkDeprecation(Settings settings) {
         // They're using the setting, so we need to tell them to stop
         if (this.isDeprecated() && this.exists(settings)) {
             // It would be convenient to show its replacement key, but replacement is often not so simple
             final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(getClass()));
             deprecationLogger.deprecated("[{}] setting was deprecated in Elasticsearch and it will be removed in a future release! " +
-                    "See the breaking changes lists in the documentation for details", getKey());
+                "See the breaking changes lists in the documentation for details", getKey());
         }
-        return settings.get(getKey(), defaultValue.apply(settings));
     }
 
     /**
@@ -404,8 +415,8 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     /**
-     * Updates settings that depend on eachother. See {@link AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)}
-     * and its usage for details.
+     * Updates settings that depend on each other.
+     * See {@link AbstractScopedSettings#addSettingsUpdateConsumer(Setting, Setting, BiConsumer)} and its usage for details.
      */
     static <A, B> AbstractScopedSettings.SettingUpdater<Tuple<A, B>> compoundUpdater(final BiConsumer<A, B> consumer,
             final Setting<A> aSetting, final Setting<B> bSetting, Logger logger) {
@@ -440,13 +451,119 @@ public class Setting<T> extends ToXContentToBytes {
         };
     }
 
+    public static class AffixSetting<T> extends Setting<T> {
+        private final AffixKey key;
+        private final Function<String, Setting<T>> delegateFactory;
+
+        public AffixSetting(AffixKey key, Setting<T> delegate, Function<String, Setting<T>> delegateFactory) {
+            super(key, delegate.defaultValue, delegate.parser, delegate.properties.toArray(new Property[0]));
+            this.key = key;
+            this.delegateFactory = delegateFactory;
+        }
+
+        boolean isGroupSetting() {
+            return true;
+        }
+
+        private Stream<String> matchStream(Settings settings) {
+            return settings.getAsMap().keySet().stream().filter((key) -> match(key)).map(settingKey -> key.getConcreteString(settingKey));
+        }
+
+        AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>> newAffixUpdater(
+            BiConsumer<String, T> consumer, Logger logger, BiConsumer<String, T> validator) {
+            return new AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>>() {
+
+                @Override
+                public boolean hasChanged(Settings current, Settings previous) {
+                    return  Stream.concat(matchStream(current), matchStream(previous)).findAny().isPresent();
+                }
+
+                @Override
+                public Map<AbstractScopedSettings.SettingUpdater<T>, T> getValue(Settings current, Settings previous) {
+                    // we collect all concrete keys and then delegate to the actual setting for validation and settings extraction
+                    final Map<AbstractScopedSettings.SettingUpdater<T>, T> result = new IdentityHashMap<>();
+                    Stream.concat(matchStream(current), matchStream(previous)).distinct().forEach(aKey -> {
+                        String namespace = key.getNamespace(aKey);
+                        AbstractScopedSettings.SettingUpdater<T> updater =
+                            getConcreteSetting(aKey).newUpdater((v) -> consumer.accept(namespace, v), logger,
+                                (v) -> validator.accept(namespace, v));
+                        if (updater.hasChanged(current, previous)) {
+                            // only the ones that have changed otherwise we might get too many updates
+                            // the hasChanged above checks only if there are any changes
+                            T value = updater.getValue(current, previous);
+                            result.put(updater, value);
+                        }
+                    });
+                    return result;
+                }
+
+                @Override
+                public void apply(Map<AbstractScopedSettings.SettingUpdater<T>, T> value, Settings current, Settings previous) {
+                    for (Map.Entry<AbstractScopedSettings.SettingUpdater<T>, T> entry : value.entrySet()) {
+                        entry.getKey().apply(entry.getValue(), current, previous);
+                    }
+                }
+            };
+        }
+
+        @Override
+        public T get(Settings settings) {
+            throw new UnsupportedOperationException("affix settings can't return values" +
+                " use #getConcreteSetting to obtain a concrete setting");
+        }
+
+        @Override
+        public String getRaw(Settings settings) {
+            throw new UnsupportedOperationException("affix settings can't return values" +
+                " use #getConcreteSetting to obtain a concrete setting");
+        }
+
+        @Override
+        public Setting<T> getConcreteSetting(String key) {
+            if (match(key)) {
+                return delegateFactory.apply(key);
+            } else {
+                throw new IllegalArgumentException("key [" + key + "] must match [" + getKey() + "] but didn't.");
+            }
+        }
+
+        /**
+         * Get a setting with the given namespace filled in for prefix and suffix.
+         */
+        public Setting<T> getConcreteSettingForNamespace(String namespace) {
+            String fullKey = key.toConcreteKey(namespace).toString();
+            return getConcreteSetting(fullKey);
+        }
+
+        @Override
+        public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
+            matchStream(defaultSettings).forEach((key) -> getConcreteSetting(key).diff(builder, source, defaultSettings));
+        }
+
+        /**
+         * Returns the namespace for a concrete settting. Ie. an affix setting with prefix: <tt>search.</tt> and suffix: <tt>username</tt>
+         * will return <tt>remote</tt> as a namespace for the setting <tt>search.remote.username</tt>
+         */
+        public String getNamespace(Setting<T> concreteSetting) {
+            return key.getNamespace(concreteSetting.getKey());
+        }
+
+        /**
+         * Returns a stream of all concrete setting instances for the given settings. AffixSetting is only a specification, concrete
+         * settings depend on an actual set of setting keys.
+         */
+        public Stream<Setting<T>> getAllConcreteSettings(Settings settings) {
+            return matchStream(settings).distinct().map(this::getConcreteSetting);
+        }
+    }
+
 
     private final class Updater implements AbstractScopedSettings.SettingUpdater<T> {
         private final Consumer<T> consumer;
         private final Logger logger;
         private final Consumer<T> accept;
 
-        public Updater(Consumer<T> consumer, Logger logger, Consumer<T> accept) {
+        Updater(Consumer<T> consumer, Logger logger, Consumer<T> accept) {
             this.consumer = consumer;
             this.logger = logger;
             this.accept = accept;
@@ -559,15 +676,15 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     public static Setting<Boolean> boolSetting(String key, boolean defaultValue, Property... properties) {
-        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, (s) -> Boolean.toString(defaultValue), Booleans::parseBoolean, properties);
     }
 
     public static Setting<Boolean> boolSetting(String key, Setting<Boolean> fallbackSetting, Property... properties) {
-        return new Setting<>(key, fallbackSetting, Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, fallbackSetting, Booleans::parseBoolean, properties);
     }
 
     public static Setting<Boolean> boolSetting(String key, Function<Settings, String> defaultValueFn, Property... properties) {
-        return new Setting<>(key, defaultValueFn, Booleans::parseBooleanExact, properties);
+        return new Setting<>(key, defaultValueFn, Booleans::parseBoolean, properties);
     }
 
     public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, Property... properties) {
@@ -702,7 +819,8 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     private static List<String> parseableStringToList(String parsableString) {
-        try (XContentParser xContentParser = XContentType.JSON.xContent().createParser(parsableString)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser xContentParser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, parsableString)) {
             XContentParser.Token token = xContentParser.nextToken();
             if (token != XContentParser.Token.START_ARRAY) {
                 throw new IllegalArgumentException("expected START_ARRAY but got " + token);
@@ -720,7 +838,6 @@ public class Setting<T> extends ToXContentToBytes {
         }
     }
 
-
     private static String arrayToParsableString(String[] array) {
         try {
             XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent());
@@ -734,9 +851,11 @@ public class Setting<T> extends ToXContentToBytes {
             throw new ElasticsearchException(ex);
         }
     }
+
     public static Setting<Settings> groupSetting(String key, Property... properties) {
         return groupSetting(key, (s) -> {}, properties);
     }
+
     public static Setting<Settings> groupSetting(String key, Consumer<Settings> validator, Property... properties) {
         return new Setting<Settings>(new GroupKey(key), (s) -> "", (s) -> null, properties) {
             @Override
@@ -817,7 +936,9 @@ public class Setting<T> extends ToXContentToBytes {
 
                     @Override
                     public void apply(Settings value, Settings current, Settings previous) {
-                        logger.info("updating [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
+                        if (logger.isInfoEnabled()) { // getRaw can create quite some objects
+                            logger.info("updating [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
+                        }
                         consumer.accept(value);
                     }
 
@@ -885,59 +1006,24 @@ public class Setting<T> extends ToXContentToBytes {
      * can easily be added with this setting. Yet, prefix key settings don't support updaters out of the box unless
      * {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> prefixKeySetting(String prefix, String defaultValue, Function<String, T> parser,
-                                                  Property... properties) {
-        return affixKeySetting(AffixKey.withPrefix(prefix), (s) -> defaultValue, parser, properties);
+    public static <T> AffixSetting<T> prefixKeySetting(String prefix, Function<String, Setting<T>> delegateFactory) {
+        return affixKeySetting(new AffixKey(prefix), delegateFactory);
     }
 
     /**
      * This setting type allows to validate settings that have the same type and a common prefix and suffix. For instance
-     * storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, adfix key settings don't support updaters
+     * storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, affix key settings don't support updaters
      * out of the box unless {@link #getConcreteSetting(String)} is used to pull the updater.
      */
-    public static <T> Setting<T> affixKeySetting(String prefix, String suffix, Function<Settings, String> defaultValue,
-                                                 Function<String, T> parser, Property... properties) {
-        return affixKeySetting(AffixKey.withAffix(prefix, suffix), defaultValue, parser, properties);
+    public static <T> AffixSetting<T> affixKeySetting(String prefix, String suffix, Function<String, Setting<T>> delegateFactory) {
+        return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory);
     }
 
-    public static <T> Setting<T> affixKeySetting(String prefix, String suffix, String defaultValue, Function<String, T> parser,
-                                                 Property... properties) {
-        return affixKeySetting(prefix, suffix, (s) -> defaultValue, parser, properties);
-    }
+    private static <T> AffixSetting<T> affixKeySetting(AffixKey key, Function<String, Setting<T>> delegateFactory) {
+        Setting<T> delegate = delegateFactory.apply("_na_");
+        return new AffixSetting<>(key, delegate, delegateFactory);
+    };
 
-    public static <T> Setting<T> affixKeySetting(AffixKey key, Function<Settings, String> defaultValue, Function<String, T> parser,
-                                                 Property... properties) {
-        return new Setting<T>(key, defaultValue, parser, properties) {
-
-            @Override
-            boolean isGroupSetting() {
-                return true;
-            }
-
-            @Override
-            AbstractScopedSettings.SettingUpdater<T> newUpdater(Consumer<T> consumer, Logger logger, Consumer<T> validator) {
-                throw new UnsupportedOperationException("Affix settings can't be updated. Use #getConcreteSetting for updating.");
-            }
-
-            @Override
-            public Setting<T> getConcreteSetting(String key) {
-                if (match(key)) {
-                    return new Setting<>(key, defaultValue, parser, properties);
-                } else {
-                    throw new IllegalArgumentException("key [" + key + "] must match [" + getKey() + "] but didn't.");
-                }
-            }
-
-            @Override
-            public void diff(Settings.Builder builder, Settings source, Settings defaultSettings) {
-                for (Map.Entry<String, String> entry : defaultSettings.getAsMap().entrySet()) {
-                    if (match(entry.getKey())) {
-                        getConcreteSetting(entry.getKey()).diff(builder, source, defaultSettings);
-                    }
-                }
-            }
-        };
-    }
 
 
     public interface Key {
@@ -1003,37 +1089,60 @@ public class Setting<T> extends ToXContentToBytes {
         }
     }
 
+    /**
+     * A key that allows for static pre and suffix. This is used for settings
+     * that have dynamic namespaces like for different accounts etc.
+     */
     public static final class AffixKey implements Key {
-        public static AffixKey withPrefix(String prefix) {
-            return new AffixKey(prefix, null);
-        }
-
-        public static AffixKey withAffix(String prefix, String suffix) {
-            return new AffixKey(prefix, suffix);
-        }
-
+        private final Pattern pattern;
         private final String prefix;
         private final String suffix;
 
-        public AffixKey(String prefix, String suffix) {
+        AffixKey(String prefix) {
+            this(prefix, null);
+        }
+
+        AffixKey(String prefix, String suffix) {
             assert prefix != null || suffix != null: "Either prefix or suffix must be non-null";
+
             this.prefix = prefix;
             if (prefix.endsWith(".") == false) {
                 throw new IllegalArgumentException("prefix must end with a '.'");
             }
             this.suffix = suffix;
+            if (suffix == null) {
+                pattern = Pattern.compile("(" + Pattern.quote(prefix) + "((?:[-\\w]+[.])*[-\\w]+$))");
+            } else {
+                // the last part of this regexp is for lists since they are represented as x.${namespace}.y.1, x.${namespace}.y.2
+                pattern = Pattern.compile("(" + Pattern.quote(prefix) + "([-\\w]+)\\." + Pattern.quote(suffix) + ")(?:\\.\\d+)?");
+            }
         }
 
         @Override
         public boolean match(String key) {
-            boolean match = true;
-            if (prefix != null) {
-                match = key.startsWith(prefix);
+            return pattern.matcher(key).matches();
+        }
+
+        /**
+         * Returns a string representation of the concrete setting key
+         */
+        String getConcreteString(String key) {
+            Matcher matcher = pattern.matcher(key);
+            if (matcher.matches() == false) {
+                throw new IllegalStateException("can't get concrete string for key " + key + " key doesn't match");
             }
-            if (suffix != null) {
-                match = match && key.endsWith(suffix);
+            return matcher.group(1);
+        }
+
+        /**
+         * Returns a string representation of the concrete setting key
+         */
+        String getNamespace(String key) {
+            Matcher matcher = pattern.matcher(key);
+            if (matcher.matches() == false) {
+                throw new IllegalStateException("can't get concrete string for key " + key + " key doesn't match");
             }
-            return match;
+            return matcher.group(2);
         }
 
         public SimpleKey toConcreteKey(String missingPart) {
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 819edc246ac..579be7ce31f 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.common.settings;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.settings.loader.SettingsLoaderFactory;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -35,6 +37,7 @@ import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,6 +45,9 @@ import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.util.AbstractMap;
+import java.util.AbstractSet;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -52,16 +58,15 @@ import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Set;
-import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
 import static org.elasticsearch.common.unit.SizeValue.parseSizeValue;
@@ -75,11 +80,24 @@ public final class Settings implements ToXContent {
     public static final Settings EMPTY = new Builder().build();
     private static final Pattern ARRAY_PATTERN = Pattern.compile("(.*)\\.\\d+$");
 
-    private SortedMap<String, String> settings;
+    /** The raw settings from the full key to raw string value. */
+    private Map<String, String> settings;
 
-    Settings(Map<String, String> settings) {
+    /** The secure settings storage associated with these settings. */
+    private SecureSettings secureSettings;
+
+    Settings(Map<String, String> settings, SecureSettings secureSettings) {
         // we use a sorted map for consistent serialization when using getAsMap()
         this.settings = Collections.unmodifiableSortedMap(new TreeMap<>(settings));
+        this.secureSettings = secureSettings;
+    }
+
+    /**
+     * Retrieve the secure settings in these settings.
+     */
+    SecureSettings getSecureSettings() {
+        // pkg private so it can only be accessed by local subclasses of SecureSetting
+        return secureSettings;
     }
 
     /**
@@ -87,7 +105,8 @@ public final class Settings implements ToXContent {
      * @return an unmodifiable map of settings
      */
     public Map<String, String> getAsMap() {
-        return Collections.unmodifiableMap(this.settings);
+        // settings is always unmodifiable
+        return this.settings;
     }
 
     /**
@@ -186,30 +205,16 @@ public final class Settings implements ToXContent {
      * A settings that are filtered (and key is removed) with the specified prefix.
      */
     public Settings getByPrefix(String prefix) {
-        Builder builder = new Builder();
-        for (Map.Entry<String, String> entry : getAsMap().entrySet()) {
-            if (entry.getKey().startsWith(prefix)) {
-                if (entry.getKey().length() < prefix.length()) {
-                    // ignore this. one
-                    continue;
-                }
-                builder.put(entry.getKey().substring(prefix.length()), entry.getValue());
-            }
-        }
-        return builder.build();
+        return new Settings(new FilteredMap(this.settings, (k) -> k.startsWith(prefix), prefix),
+            secureSettings == null ? null : new PrefixedSecureSettings(secureSettings, prefix));
     }
 
     /**
      * Returns a new settings object that contains all setting of the current one filtered by the given settings key predicate.
+     * Secure settings may not be accessed through a filter.
      */
     public Settings filter(Predicate<String> predicate) {
-        Builder builder = new Builder();
-        for (Map.Entry<String, String> entry : getAsMap().entrySet()) {
-            if (predicate.test(entry.getKey())) {
-                builder.put(entry.getKey(), entry.getValue());
-            }
-        }
-        return builder.build();
+        return new Settings(new FilteredMap(this.settings, predicate, null), null);
     }
 
     /**
@@ -310,6 +315,36 @@ public final class Settings implements ToXContent {
         return Booleans.parseBoolean(get(setting), defaultValue);
     }
 
+    // TODO #22298: Delete this method and update call sites to <code>#getAsBoolean(String, Boolean)</code>.
+    /**
+     * Returns the setting value (as boolean) associated with the setting key. If it does not exist, returns the default value provided.
+     * If the index was created on Elasticsearch below 6.0, booleans will be parsed leniently otherwise they are parsed strictly.
+     *
+     * See {@link Booleans#isBooleanLenient(char[], int, int)} for the definition of a "lenient boolean"
+     * and {@link Booleans#isBoolean(char[], int, int)} for the definition of a "strict boolean".
+     *
+     * @deprecated Only used to provide automatic upgrades for pre 6.0 indices.
+     */
+    @Deprecated
+    public Boolean getAsBooleanLenientForPreEs6Indices(
+        final Version indexVersion,
+        final String setting,
+        final Boolean defaultValue,
+        final DeprecationLogger deprecationLogger) {
+        if (indexVersion.before(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            //Only emit a warning if the setting's value is not a proper boolean
+            final String value = get(setting, "false");
+            if (Booleans.isBoolean(value) == false) {
+                @SuppressWarnings("deprecation")
+                boolean convertedValue = Booleans.parseBooleanLenient(get(setting), defaultValue);
+                deprecationLogger.deprecated("The value [{}] of setting [{}] is not coerced into boolean anymore. Please change " +
+                    "this value to [{}].", value, setting, String.valueOf(convertedValue));
+                return convertedValue;
+            }
+        }
+        return getAsBoolean(setting, defaultValue);
+    }
+
     /**
      * Returns the setting value (as time) associated with the setting key. If it does not exists,
      * returns the default value provided.
@@ -443,6 +478,7 @@ public final class Settings implements ToXContent {
         }
         return getGroupsInternal(settingPrefix, ignoreNonGrouped);
     }
+
     private Map<String, Settings> getGroupsInternal(String settingPrefix, boolean ignoreNonGrouped) throws SettingsException {
         // we don't really care that it might happen twice
         Map<String, Map<String, String>> map = new LinkedHashMap<>();
@@ -470,7 +506,7 @@ public final class Settings implements ToXContent {
         }
         Map<String, Settings> retVal = new LinkedHashMap<>();
         for (Map.Entry<String, Map<String, String>> entry : map.entrySet()) {
-            retVal.put(entry.getKey(), new Settings(Collections.unmodifiableMap(entry.getValue())));
+            retVal.put(entry.getKey(), new Settings(Collections.unmodifiableMap(entry.getValue()), secureSettings));
         }
         return Collections.unmodifiableMap(retVal);
     }
@@ -553,7 +589,7 @@ public final class Settings implements ToXContent {
     }
 
     public static void writeSettingsToStream(Settings settings, StreamOutput out) throws IOException {
-        out.writeVInt(settings.getAsMap().size());
+        out.writeVInt(settings.size());
         for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
             out.writeString(entry.getKey());
             out.writeOptionalString(entry.getValue());
@@ -590,7 +626,12 @@ public final class Settings implements ToXContent {
      * @return <tt>true</tt> if this settings object contains no settings
      */
     public boolean isEmpty() {
-        return this.settings.isEmpty();
+        return this.settings.isEmpty(); // TODO: account for secure settings
+    }
+
+    /** Returns the number of settings in this settings object. */
+    public int size() {
+        return this.settings.size(); // TODO: account for secure settings
     }
 
     /**
@@ -602,7 +643,10 @@ public final class Settings implements ToXContent {
 
         public static final Settings EMPTY_SETTINGS = new Builder().build();
 
-        private final Map<String, String> map = new LinkedHashMap<>();
+        // we use a sorted map for consistent serialization when using getAsMap()
+        private final Map<String, String> map = new TreeMap<>();
+
+        private SetOnce<SecureSettings> secureSettings = new SetOnce<>();
 
         private Builder() {
 
@@ -626,6 +670,14 @@ public final class Settings implements ToXContent {
             return map.get(key);
         }
 
+        public Builder setSecureSettings(SecureSettings secureSettings) {
+            if (secureSettings.isLoaded() == false) {
+                throw new IllegalStateException("Secure settings must already be loaded");
+            }
+            this.secureSettings.set(secureSettings);
+            return this;
+        }
+
         /**
          * Puts tuples of key value pairs of settings. Simplified version instead of repeating calling
          * put for each one.
@@ -850,6 +902,9 @@ public final class Settings implements ToXContent {
         public Builder put(Settings settings) {
             removeNonArraysFieldsIfNewSettingsContainsFieldAsArray(settings.getAsMap());
             map.putAll(settings.getAsMap());
+            if (settings.getSecureSettings() != null) {
+                setSecureSettings(settings.getSecureSettings());
+            }
             return this;
         }
 
@@ -906,7 +961,9 @@ public final class Settings implements ToXContent {
         /**
          * Loads settings from the actual string content that represents them using the
          * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * @deprecated use {@link #loadFromSource(String, XContentType)} to avoid content type detection
          */
+        @Deprecated
         public Builder loadFromSource(String source) {
             SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromSource(source);
             try {
@@ -918,9 +975,24 @@ public final class Settings implements ToXContent {
             return this;
         }
 
+        /**
+         * Loads settings from the actual string content that represents them using the
+         * {@link SettingsLoaderFactory#loaderFromXContentType(XContentType)} method to obtain a loader
+         */
+        public Builder loadFromSource(String source, XContentType xContentType) {
+            SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromXContentType(xContentType);
+            try {
+                Map<String, String> loadedSettings = settingsLoader.load(source);
+                put(loadedSettings);
+            } catch (Exception e) {
+                throw new SettingsException("Failed to load settings from [" + source + "]", e);
+            }
+            return this;
+        }
+
         /**
          * Loads settings from a url that represents them using the
-         * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * {@link SettingsLoaderFactory#loaderFromResource(String)}.
          */
         public Builder loadFromPath(Path path) throws IOException {
             // NOTE: loadFromStream will close the input stream
@@ -929,7 +1001,7 @@ public final class Settings implements ToXContent {
 
         /**
          * Loads settings from a stream that represents them using the
-         * {@link SettingsLoaderFactory#loaderFromSource(String)}.
+         * {@link SettingsLoaderFactory#loaderFromResource(String)}.
          */
         public Builder loadFromStream(String resourceName, InputStream is) throws IOException {
             SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName);
@@ -1032,7 +1104,154 @@ public final class Settings implements ToXContent {
          * set on this builder.
          */
         public Settings build() {
-            return new Settings(Collections.unmodifiableMap(map));
+            return new Settings(map, secureSettings.get());
+        }
+    }
+
+    // TODO We could use an FST internally to make things even faster and more compact
+    private static final class FilteredMap extends AbstractMap<String, String> {
+        private final Map<String, String> delegate;
+        private final Predicate<String> filter;
+        private final String prefix;
+        // we cache that size since we have to iterate the entire set
+        // this is safe to do since this map is only used with unmodifiable maps
+        private int size = -1;
+        @Override
+        public Set<Entry<String, String>> entrySet() {
+            Set<Entry<String, String>> delegateSet = delegate.entrySet();
+            AbstractSet<Entry<String, String>> filterSet = new AbstractSet<Entry<String, String>>() {
+
+                @Override
+                public Iterator<Entry<String, String>> iterator() {
+                    Iterator<Entry<String, String>> iter = delegateSet.iterator();
+
+                    return new Iterator<Entry<String, String>>() {
+                        private int numIterated;
+                        private Entry<String, String> currentElement;
+                        @Override
+                        public boolean hasNext() {
+                            if (currentElement != null) {
+                                return true; // protect against calling hasNext twice
+                            } else {
+                                if (numIterated == size) { // early terminate
+                                    assert size != -1 : "size was never set: " + numIterated + " vs. " + size;
+                                    return false;
+                                }
+                                while (iter.hasNext()) {
+                                    if (filter.test((currentElement = iter.next()).getKey())) {
+                                        numIterated++;
+                                        return true;
+                                    }
+                                }
+                                // we didn't find anything
+                                currentElement = null;
+                                return false;
+                            }
+                        }
+
+                        @Override
+                        public Entry<String, String> next() {
+                            if (currentElement == null && hasNext() == false) { // protect against no #hasNext call or not respecting it
+
+                                throw new NoSuchElementException("make sure to call hasNext first");
+                            }
+                            final Entry<String, String> current = this.currentElement;
+                            this.currentElement = null;
+                            if (prefix == null) {
+                                return current;
+                            }
+                            return new Entry<String, String>() {
+                                @Override
+                                public String getKey() {
+                                    return current.getKey().substring(prefix.length());
+                                }
+
+                                @Override
+                                public String getValue() {
+                                    return current.getValue();
+                                }
+
+                                @Override
+                                public String setValue(String value) {
+                                    throw new UnsupportedOperationException();
+                                }
+                            };
+                        }
+                    };
+                }
+
+                @Override
+                public int size() {
+                    return FilteredMap.this.size();
+                }
+            };
+            return filterSet;
+        }
+
+        private FilteredMap(Map<String, String> delegate, Predicate<String> filter, String prefix) {
+            this.delegate = delegate;
+            this.filter = filter;
+            this.prefix = prefix;
+        }
+
+        @Override
+        public String get(Object key) {
+            if (key instanceof String) {
+                final String theKey = prefix == null ? (String)key : prefix + key;
+                if (filter.test(theKey)) {
+                    return delegate.get(theKey);
+                }
+            }
+            return null;
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            if (key instanceof String) {
+                final String theKey = prefix == null ? (String) key : prefix + key;
+                if (filter.test(theKey)) {
+                    return delegate.containsKey(theKey);
+                }
+            }
+            return false;
+        }
+
+        @Override
+        public int size() {
+            if (size == -1) {
+                size = Math.toIntExact(delegate.keySet().stream().filter((e) -> filter.test(e)).count());
+            }
+            return size;
+        }
+    }
+
+    private static class PrefixedSecureSettings implements SecureSettings {
+        private SecureSettings delegate;
+        private String prefix;
+
+        PrefixedSecureSettings(SecureSettings delegate, String prefix) {
+            this.delegate = delegate;
+            this.prefix = prefix;
+        }
+
+        @Override
+        public boolean isLoaded() {
+            return delegate.isLoaded();
+        }
+
+        @Override
+        public boolean hasSetting(String setting) {
+            return delegate.hasSetting(prefix + setting);
+        }
+
+        @Override
+        public SecureString getString(String setting) throws GeneralSecurityException{
+            return delegate.getString(prefix + setting);
+        }
+
+        @Override
+        public void close() throws IOException {
+            delegate.close();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 60276ce14f7..45a38ce34a5 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -54,6 +54,7 @@ public class SettingsModule implements Module {
     private final Logger logger;
     private final IndexScopedSettings indexScopedSettings;
     private final ClusterSettings clusterSettings;
+    private final SettingsFilter settingsFilter;
 
     public SettingsModule(Settings settings, Setting<?>... additionalSettings) {
         this(settings, Arrays.asList(additionalSettings), Collections.emptyList());
@@ -137,12 +138,13 @@ public class SettingsModule implements Module {
         final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));
         validateTribeSettings(settings, clusterSettings);
+        this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern);
      }
 
     @Override
     public void configure(Binder binder) {
         binder.bind(Settings.class).toInstance(settings);
-        binder.bind(SettingsFilter.class).toInstance(new SettingsFilter(settings, settingsFilterPattern));
+        binder.bind(SettingsFilter.class).toInstance(settingsFilter);
         binder.bind(ClusterSettings.class).toInstance(clusterSettings);
         binder.bind(IndexScopedSettings.class).toInstance(indexScopedSettings);
     }
@@ -218,4 +220,8 @@ public class SettingsModule implements Module {
     public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
+
+    public SettingsFilter getSettingsFilter() {
+        return settingsFilter;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
index 5f2da22c5f2..5d8cb4918b2 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoaderFactory.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common.settings.loader;
 
+import org.elasticsearch.common.xcontent.XContentType;
+
 /**
  * A class holding factory methods for settings loaders that attempts
  * to infer the type of the underlying settings content.
@@ -33,9 +35,7 @@ public final class SettingsLoaderFactory {
      * name. This factory method assumes that if the resource name ends
      * with ".json" then the content should be parsed as JSON, else if
      * the resource name ends with ".yml" or ".yaml" then the content
-     * should be parsed as YAML, else if the resource name ends with
-     * ".properties" then the content should be parsed as properties,
-     * otherwise default to attempting to parse as JSON. Note that the
+     * should be parsed as YAML, otherwise throws an exception. Note that the
      * parsers returned by this method will not accept null-valued
      * keys.
      *
@@ -59,13 +59,15 @@ public final class SettingsLoaderFactory {
      * contains an opening and closing brace ('{' and '}') then the
      * content should be parsed as JSON, else if the underlying content
      * fails this condition but contains a ':' then the content should
-     * be parsed as YAML, and otherwise should be parsed as properties.
+     * be parsed as YAML, and otherwise throws an exception.
      * Note that the JSON and YAML parsers returned by this method will
      * accept null-valued keys.
      *
      * @param source The underlying settings content.
      * @return A settings loader.
+     * @deprecated use {@link #loaderFromXContentType(XContentType)} instead
      */
+    @Deprecated
     public static SettingsLoader loaderFromSource(String source) {
         if (source.indexOf('{') != -1 && source.indexOf('}') != -1) {
             return new JsonSettingsLoader(true);
@@ -76,4 +78,20 @@ public final class SettingsLoaderFactory {
         }
     }
 
+    /**
+     * Returns a {@link SettingsLoader} based on the {@link XContentType}. Note only {@link XContentType#JSON} and
+     * {@link XContentType#YAML} are supported
+     *
+     * @param xContentType The content type
+     * @return A settings loader.
+     */
+    public static SettingsLoader loaderFromXContentType(XContentType xContentType) {
+        if (xContentType == XContentType.JSON) {
+            return new JsonSettingsLoader(true);
+        } else if (xContentType == XContentType.YAML) {
+            return new YamlSettingsLoader(true);
+        } else {
+            throw new IllegalArgumentException("unsupported content type [" + xContentType + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
index 30c62b91c79..d7eaa627a28 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/XContentSettingsLoader.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.settings.loader;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -46,14 +47,16 @@ public abstract class XContentSettingsLoader implements SettingsLoader {
 
     @Override
     public Map<String, String> load(String source) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(source)) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(NamedXContentRegistry.EMPTY, source)) {
             return load(parser);
         }
     }
 
     @Override
     public Map<String, String> load(byte[] source) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(source)) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(contentType()).createParser(NamedXContentRegistry.EMPTY, source)) {
             return load(parser);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/text/Text.java b/core/src/main/java/org/elasticsearch/common/text/Text.java
index 39eb817fe3c..d895b7c11b0 100644
--- a/core/src/main/java/org/elasticsearch/common/text/Text.java
+++ b/core/src/main/java/org/elasticsearch/common/text/Text.java
@@ -100,7 +100,10 @@ public final class Text implements Comparable<Text> {
 
     @Override
     public boolean equals(Object obj) {
-        if (obj == null) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
             return false;
         }
         return bytes().equals(((Text) obj).bytes());
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
index cac3132385f..02f2c627a4c 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigByteArray.java
@@ -36,7 +36,7 @@ final class BigByteArray extends AbstractBigArray implements ByteArray {
     private byte[][] pages;
 
     /** Constructor. */
-    public BigByteArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigByteArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(BYTE_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new byte[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
index 4aab593affe..a8b4503bda6 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java
@@ -35,7 +35,7 @@ final class BigDoubleArray extends AbstractBigArray implements DoubleArray {
     private long[][] pages;
 
     /** Constructor. */
-    public BigDoubleArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigDoubleArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(LONG_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new long[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
index 1fa79a9f3db..cf11eba37ae 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigFloatArray.java
@@ -35,7 +35,7 @@ final class BigFloatArray extends AbstractBigArray implements FloatArray {
     private int[][] pages;
 
     /** Constructor. */
-    public BigFloatArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigFloatArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(INT_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new int[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
index 4ce5fc7acee..16ca3ada24d 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigIntArray.java
@@ -35,7 +35,7 @@ final class BigIntArray extends AbstractBigArray implements IntArray {
     private int[][] pages;
 
     /** Constructor. */
-    public BigIntArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigIntArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(INT_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new int[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
index 2e3248143b4..cb9b9e6c332 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigLongArray.java
@@ -35,7 +35,7 @@ final class BigLongArray extends AbstractBigArray implements LongArray {
     private long[][] pages;
 
     /** Constructor. */
-    public BigLongArray(long size, BigArrays bigArrays, boolean clearOnResize) {
+    BigLongArray(long size, BigArrays bigArrays, boolean clearOnResize) {
         super(LONG_PAGE_SIZE, bigArrays, clearOnResize);
         this.size = size;
         pages = new long[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java b/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
index 19a41d3096d..023c710f3aa 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigObjectArray.java
@@ -35,7 +35,7 @@ final class BigObjectArray<T> extends AbstractBigArray implements ObjectArray<T>
     private Object[][] pages;
 
     /** Constructor. */
-    public BigObjectArray(long size, BigArrays bigArrays) {
+    BigObjectArray(long size, BigArrays bigArrays) {
         super(OBJECT_PAGE_SIZE, bigArrays, true);
         this.size = size;
         pages = new Object[numPages(size)][];
diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
index 5171be0ee91..54a49f7e4f2 100644
--- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
@@ -226,7 +226,7 @@ public class CollectionUtils {
         private final List<T> in;
         private final int distance;
 
-        public RotatedList(List<T> list, int distance) {
+        RotatedList(List<T> list, int distance) {
             if (distance < 0 || distance >= list.size()) {
                 throw new IllegalArgumentException();
             }
diff --git a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
index 2712aef8233..528982385ac 100644
--- a/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
+++ b/core/src/main/java/org/elasticsearch/common/util/IndexFolderUpgrader.java
@@ -26,6 +26,7 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -86,7 +87,7 @@ public class IndexFolderUpgrader {
     void upgrade(final String indexFolderName) throws IOException {
         for (NodeEnvironment.NodePath nodePath : nodeEnv.nodePaths()) {
             final Path indexFolderPath = nodePath.indicesPath.resolve(indexFolderName);
-            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, indexFolderPath);
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexFolderPath);
             if (indexMetaData != null) {
                 final Index index = indexMetaData.getIndex();
                 if (needsUpgrade(index, indexFolderName)) {
diff --git a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
index 4095f5d7014..a79e8d88be6 100644
--- a/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
+++ b/core/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java
@@ -161,7 +161,7 @@ public class LongObjectPagedHashMap<T> extends AbstractPagedHashMap implements I
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java
index d5b44ed4dfb..e0b8aea178c 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.common.util.concurrent;
 
 import org.apache.lucene.store.AlreadyClosedException;
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 2dad4663dc4..bd99f3b1a47 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -161,7 +161,7 @@ public class EsExecutors {
         final AtomicInteger threadNumber = new AtomicInteger(1);
         final String namePrefix;
 
-        public EsThreadFactory(String namePrefix) {
+        EsThreadFactory(String namePrefix) {
             this.namePrefix = namePrefix;
             SecurityManager s = System.getSecurityManager();
             group = (s != null) ? s.getThreadGroup() :
@@ -189,7 +189,7 @@ public class EsExecutors {
 
         ThreadPoolExecutor executor;
 
-        public ExecutorScalingQueue() {
+        ExecutorScalingQueue() {
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
index 813265f19c4..1b01455c1ca 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/PrioritizedEsThreadPoolExecutor.java
@@ -250,14 +250,14 @@ public class PrioritizedEsThreadPoolExecutor extends EsThreadPoolExecutor {
         final Priority priority;
         final long insertionOrder;
 
-        public PrioritizedFutureTask(Runnable runnable, Priority priority, T value, long insertionOrder) {
+        PrioritizedFutureTask(Runnable runnable, Priority priority, T value, long insertionOrder) {
             super(runnable, value);
             this.task = runnable;
             this.priority = priority;
             this.insertionOrder = insertionOrder;
         }
 
-        public PrioritizedFutureTask(PrioritizedCallable<T> callable, long insertionOrder) {
+        PrioritizedFutureTask(PrioritizedCallable<T> callable, long insertionOrder) {
             super(callable);
             this.task = callable;
             this.priority = callable.priority();
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index ca1d364ffd2..d439696b720 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.store.Store;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -34,6 +35,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A ThreadContext is a map of string headers and a transient map of keyed objects that are associated with
@@ -115,12 +119,57 @@ public final class ThreadContext implements Closeable, Writeable {
         return () -> threadLocal.set(context);
     }
 
+
     /**
      * Just like {@link #stashContext()} but no default context is set.
+     * @param preserveResponseHeaders if set to <code>true</code> the response headers of the restore thread will be preserved.
      */
-    public StoredContext newStoredContext() {
+    public StoredContext newStoredContext(boolean preserveResponseHeaders) {
         final ThreadContextStruct context = threadLocal.get();
-        return () -> threadLocal.set(context);
+        return ()  -> {
+            if (preserveResponseHeaders && threadLocal.get() != context) {
+                threadLocal.set(context.putResponseHeaders(threadLocal.get().responseHeaders));
+            } else {
+                threadLocal.set(context);
+            }
+        };
+    }
+
+    /**
+     * Returns a supplier that gathers a {@link #newStoredContext(boolean)} and restores it once the
+     * returned supplier is invoked. The context returned from the supplier is a stored version of the
+     * suppliers callers context that should be restored once the originally gathered context is not needed anymore.
+     * For instance this method should be used like this:
+     *
+     * <pre>
+     *     Supplier&lt;ThreadContext.StoredContext&gt; restorable = context.newRestorableContext(true);
+     *     new Thread() {
+     *         public void run() {
+     *             try (ThreadContext.StoredContext ctx = restorable.get()) {
+     *                 // execute with the parents context and restore the threads context afterwards
+     *             }
+     *         }
+     *
+     *     }.start();
+     * </pre>
+     *
+     * @param preserveResponseHeaders if set to <code>true</code> the response headers of the restore thread will be preserved.
+     * @return a restorable context supplier
+     */
+    public Supplier<StoredContext> newRestorableContext(boolean preserveResponseHeaders) {
+        return wrapRestorable(newStoredContext(preserveResponseHeaders));
+    }
+
+    /**
+     * Same as {@link #newRestorableContext(boolean)} but wraps an existing context to restore.
+     * @param storedContext the context to restore
+     */
+    public Supplier<StoredContext> wrapRestorable(StoredContext storedContext) {
+        return () -> {
+            StoredContext context = newStoredContext(false);
+            storedContext.restore();
+            return context;
+        };
     }
 
     @Override
@@ -327,6 +376,26 @@ public final class ThreadContext implements Closeable, Writeable {
             }
         }
 
+        private ThreadContextStruct putResponseHeaders(Map<String, List<String>> headers) {
+            assert headers != null;
+            if (headers.isEmpty()) {
+                return this;
+            }
+            final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders);
+            for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
+                String key = entry.getKey();
+                final List<String> existingValues = newResponseHeaders.get(key);
+                if (existingValues != null) {
+                    List<String> newValues = Stream.concat(entry.getValue().stream(),
+                        existingValues.stream()).distinct().collect(Collectors.toList());
+                    newResponseHeaders.put(key, Collections.unmodifiableList(newValues));
+                } else {
+                    newResponseHeaders.put(key, entry.getValue());
+                }
+            }
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders);
+        }
+
         private ThreadContextStruct putResponse(String key, String value) {
             assert value != null;
 
@@ -445,7 +514,7 @@ public final class ThreadContext implements Closeable, Writeable {
         private final ThreadContext.StoredContext ctx;
 
         private ContextPreservingRunnable(Runnable in) {
-            ctx = newStoredContext();
+            ctx = newStoredContext(false);
             this.in = in;
         }
 
@@ -487,7 +556,7 @@ public final class ThreadContext implements Closeable, Writeable {
         private ThreadContext.StoredContext threadsOriginalContext = null;
 
         private ContextPreservingAbstractRunnable(AbstractRunnable in) {
-            creatorsContext = newStoredContext();
+            creatorsContext = newStoredContext(false);
             this.in = in;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
index 64a5fa4c119..91acb267056 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -34,17 +34,9 @@ import java.util.function.BiFunction;
 /**
  * Superclass for {@link ObjectParser} and {@link ConstructingObjectParser}. Defines most of the "declare" methods so they can be shared.
  */
-public abstract class AbstractObjectParser<Value, Context extends ParseFieldMatcherSupplier>
+public abstract class AbstractObjectParser<Value, Context>
         implements BiFunction<XContentParser, Context, Value>, ContextParser<Context, Value> {
 
-    /**
-     * Reads an object right from the parser without any context.
-     */
-    @FunctionalInterface
-    public interface NoContextParser<T> {
-        T parse(XContentParser p) throws IOException;
-    }
-
     /**
      * Declare some field. Usually it is easier to use {@link #declareString(BiConsumer, ParseField)} or
      * {@link #declareObject(BiConsumer, ContextParser, ParseField)} rather than call this directly.
@@ -52,11 +44,12 @@ public abstract class AbstractObjectParser<Value, Context extends ParseFieldMatc
     public abstract <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField,
             ValueType type);
 
-    public <T> void declareField(BiConsumer<Value, T> consumer, NoContextParser<T> parser, ParseField parseField, ValueType type) {
+    public <T> void declareField(BiConsumer<Value, T> consumer, CheckedFunction<XContentParser, T, IOException> parser,
+            ParseField parseField, ValueType type) {
         if (parser == null) {
             throw new IllegalArgumentException("[parser] is required");
         }
-        declareField(consumer, (p, c) -> parser.parse(p), parseField, type);
+        declareField(consumer, (p, c) -> parser.apply(p), parseField, type);
     }
 
     public <T> void declareObject(BiConsumer<Value, T> consumer, ContextParser<Context, T> objectParser, ParseField field) {
@@ -122,7 +115,7 @@ public abstract class AbstractObjectParser<Value, Context extends ParseFieldMatc
     }
 
     public void declareRawObject(BiConsumer<Value, BytesReference> consumer, ParseField field) {
-        NoContextParser<BytesReference> bytesParser = p -> {
+        CheckedFunction<XContentParser, BytesReference, IOException> bytesParser = p -> {
             try (XContentBuilder builder = JsonXContent.contentBuilder()) {
                 builder.prettyPrint();
                 builder.copyCurrentStructure(p);
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
index 6e646094d06..82ee94550c1 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 
@@ -74,7 +73,7 @@ import java.util.function.Function;
  * Note: if optional constructor arguments aren't specified then the number of allocations is always the worst case.
  * </p>
  */
-public final class ConstructingObjectParser<Value, Context extends ParseFieldMatcherSupplier> extends AbstractObjectParser<Value, Context> {
+public final class ConstructingObjectParser<Value, Context> extends AbstractObjectParser<Value, Context> {
     /**
      * Consumer that marks a field as a required constructor argument instead of a real object field.
      */
@@ -236,7 +235,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
 
     /**
      * The target of the {@linkplain ConstructingObjectParser}. One of these is built every time you call
-     * {@linkplain ConstructingObjectParser#apply(XContentParser, ParseFieldMatcherSupplier)} Note that it is not static so it inherits
+     * {@linkplain ConstructingObjectParser#apply(XContentParser, Object)} Note that it is not static so it inherits
      * {@linkplain ConstructingObjectParser}'s type parameters.
      */
     private class Target {
@@ -268,7 +267,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
          */
         private Value targetObject;
 
-        public Target(XContentParser parser) {
+        Target(XContentParser parser) {
             this.parser = parser;
         }
 
@@ -361,7 +360,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
         final ParseField field;
         final boolean required;
 
-        public ConstructorArgInfo(ParseField field, boolean required) {
+        ConstructorArgInfo(ParseField field, boolean required) {
             this.field = field;
             this.required = required;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
new file mode 100644
index 00000000000..a565f1a3251
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableMap;
+import static java.util.Objects.requireNonNull;
+
+public class NamedXContentRegistry {
+    /**
+     * The empty {@link NamedXContentRegistry} for use when you are sure that you aren't going to call
+     * {@link XContentParser#namedObject(Class, String, Object)}. Be *very* careful with this singleton because a parser using it will fail
+     * every call to {@linkplain XContentParser#namedObject(Class, String, Object)}. Every non-test usage really should be checked thorowly
+     * and marked with a comment about how it was checked. That way anyone that sees code that uses it knows that it is potentially
+     * dangerous.
+     */
+    public static final NamedXContentRegistry EMPTY = new NamedXContentRegistry(emptyList());
+
+    /**
+     * An entry in the {@linkplain NamedXContentRegistry} containing the name of the object and the parser that can parse it.
+     */
+    public static class Entry {
+        /** The class that this entry can read. */
+        public final Class<?> categoryClass;
+
+        /** A name for the entry which is unique within the {@link #categoryClass}. */
+        public final ParseField name;
+
+        /** A parser capability of parser the entry's class. */
+        private final ContextParser<Object, ?> parser;
+
+        /** Creates a new entry which can be stored by the registry. */
+        public <T> Entry(Class<T> categoryClass, ParseField name, CheckedFunction<XContentParser, ? extends T, IOException> parser) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.parser = Objects.requireNonNull((p, c) -> parser.apply(p));
+        }
+        /**
+         * Creates a new entry which can be stored by the registry.
+         * @deprecated prefer {@link Entry#Entry(Class, ParseField, CheckedFunction)}. Contexts will be removed when possible
+         */
+        @Deprecated
+        public <T> Entry(Class<T> categoryClass, ParseField name, ContextParser<Object, ? extends T> parser) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.parser = Objects.requireNonNull(parser);
+        }
+    }
+
+    private final Map<Class<?>, Map<String, Entry>> registry;
+
+    public NamedXContentRegistry(List<Entry> entries) {
+        if (entries.isEmpty()) {
+            registry = emptyMap();
+            return;
+        }
+        entries = new ArrayList<>(entries);
+        entries.sort((e1, e2) -> e1.categoryClass.getName().compareTo(e2.categoryClass.getName()));
+
+        Map<Class<?>, Map<String, Entry>> registry = new HashMap<>();
+        Map<String, Entry> parsers = null;
+        Class<?> currentCategory = null;
+        for (Entry entry : entries) {
+            if (currentCategory != entry.categoryClass) {
+                if (currentCategory != null) {
+                    // we've seen the last of this category, put it into the big map
+                    registry.put(currentCategory, unmodifiableMap(parsers));
+                }
+                parsers = new HashMap<>();
+                currentCategory = entry.categoryClass;
+            }
+
+            for (String name : entry.name.getAllNamesIncludedDeprecated()) {
+                Object old = parsers.put(name, entry);
+                if (old != null) {
+                    throw new IllegalArgumentException("NamedXContent [" + currentCategory.getName() + "][" + entry.name + "]" +
+                        " is already registered for [" + old.getClass().getName() + "]," +
+                        " cannot register [" + entry.parser.getClass().getName() + "]");
+                }
+            }
+        }
+        // handle the last category
+        registry.put(currentCategory, unmodifiableMap(parsers));
+
+        this.registry = unmodifiableMap(registry);
+    }
+
+    /**
+     * Parse a named object, throwing an exception if the parser isn't found. Throws an {@link ElasticsearchException} if the
+     * {@code categoryClass} isn't registered because this is almost always a bug. Throws a {@link UnknownNamedObjectException} if the
+     * {@code categoryClass} is registered but the {@code name} isn't.
+     */
+    public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentParser parser, C context) throws IOException {
+        Map<String, Entry> parsers = registry.get(categoryClass);
+        if (parsers == null) {
+            if (registry.isEmpty()) {
+                // The "empty" registry will never work so we throw a better exception as a hint.
+                throw new ElasticsearchException("namedObject is not supported for this parser");
+            }
+            throw new ElasticsearchException("Unknown namedObject category [" + categoryClass.getName() + "]");
+        }
+        Entry entry = parsers.get(name);
+        if (entry == null) {
+            throw new UnknownNamedObjectException(parser.getTokenLocation(), categoryClass, name);
+        }
+        if (false == entry.name.match(name)) {
+            /* Note that this shouldn't happen because we already looked up the entry using the names but we need to call `match` anyway
+             * because it is responsible for logging deprecation warnings. */
+            throw new ParsingException(parser.getTokenLocation(),
+                    "Unknown " + categoryClass.getSimpleName() + " [" + name + "]: Parser didn't match");
+        }
+        return categoryClass.cast(entry.parser.parse(parser, context));
+    }
+
+    /**
+     * Thrown when {@link NamedXContentRegistry#parseNamedObject(Class, String, XContentParser, Object)} is called with an unregistered
+     * name. When this bubbles up to the rest layer it is converted into a response with {@code 400 BAD REQUEST} status.
+     */
+    public static class UnknownNamedObjectException extends ParsingException {
+        private final String categoryClass;
+        private final String name;
+
+        public UnknownNamedObjectException(XContentLocation contentLocation, Class<?> categoryClass,
+                String name) {
+            super(contentLocation, "Unknown " + categoryClass.getSimpleName() + " [" + name + "]");
+            this.categoryClass = requireNonNull(categoryClass, "categoryClass is required").getName();
+            this.name = requireNonNull(name, "name is required");
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public UnknownNamedObjectException(StreamInput in) throws IOException {
+            super(in);
+            categoryClass = in.readString();
+            name = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(categoryClass);
+            out.writeString(name);
+        }
+
+        /**
+         * Category class that was missing a parser. This is a String instead of a class because the class might not be on the classpath
+         * of all nodes or it might be exclusive to a plugin or something.
+         */
+        public String getCategoryClass() {
+            return categoryClass;
+        }
+
+        /**
+         * Name of the missing parser.
+         */
+        public String getName() {
+            return name;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 8bd3b634d69..0a7e71c6f06 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -20,8 +20,6 @@ package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 
 import java.io.IOException;
@@ -68,7 +66,7 @@ import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_STRIN
  * It's highly recommended to use the high level declare methods like {@link #declareString(BiConsumer, ParseField)} instead of
  * {@link #declareField} which can be used to implement exceptional parsing operations not covered by the high level methods.
  */
-public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier> extends AbstractObjectParser<Value, Context> {
+public final class ObjectParser<Value, Context> extends AbstractObjectParser<Value, Context> {
     /**
      * Adapts an array (or varags) setter into a list setter.
      */
@@ -122,7 +120,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
     /**
      * Parses a Value from the given {@link XContentParser}
      * @param parser the parser to build a value from
-     * @param context must at least provide a {@link ParseFieldMatcher}
+     * @param context context needed for parsing
      * @return a new value instance drawn from the provided value supplier on {@link #ObjectParser(String, Supplier)}
      * @throws IOException if an IOException occurs.
      */
@@ -153,7 +151,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
             }
         }
 
-        FieldParser<Value> fieldParser = null;
+        FieldParser fieldParser = null;
         String currentFieldName = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
@@ -167,7 +165,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
                     assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields";
                     parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array
                 } else {
-                    fieldParser.assertSupports(name, token, currentFieldName, context.getParseFieldMatcher());
+                    fieldParser.assertSupports(name, token, currentFieldName);
                     parseSub(parser, fieldParser, currentFieldName, value, context);
                 }
                 fieldParser = null;
@@ -357,13 +355,13 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
         return name;
     }
 
-    private void parseArray(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseArray(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         assert parser.currentToken() == XContentParser.Token.START_ARRAY : "Token was: " + parser.currentToken();
         parseValue(parser, fieldParser, currentFieldName, value, context);
     }
 
-    private void parseValue(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseValue(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         try {
             fieldParser.parser.parse(parser, value, context);
@@ -372,7 +370,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
         }
     }
 
-    private void parseSub(XContentParser parser, FieldParser<Value> fieldParser, String currentFieldName, Value value, Context context)
+    private void parseSub(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context)
             throws IOException {
         final XContentParser.Token token = parser.currentToken();
         switch (token) {
@@ -396,28 +394,28 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
     }
 
     private FieldParser getParser(String fieldName) {
-        FieldParser<Value> parser = fieldParserMap.get(fieldName);
+        FieldParser parser = fieldParserMap.get(fieldName);
         if (parser == null && false == ignoreUnknownFields) {
             throw new IllegalArgumentException("[" + name  + "] unknown field [" + fieldName + "], parser not found");
         }
         return parser;
     }
 
-    public static class FieldParser<T> {
-        private final Parser parser;
+    private class FieldParser {
+        private final Parser<Value, Context> parser;
         private final EnumSet<XContentParser.Token> supportedTokens;
         private final ParseField parseField;
         private final ValueType type;
 
-        public FieldParser(Parser parser, EnumSet<XContentParser.Token> supportedTokens, ParseField parseField, ValueType type) {
+        FieldParser(Parser<Value, Context> parser, EnumSet<XContentParser.Token> supportedTokens, ParseField parseField, ValueType type) {
             this.parser = parser;
             this.supportedTokens = supportedTokens;
             this.parseField = parseField;
             this.type = type;
         }
 
-        public void assertSupports(String parserName, XContentParser.Token token, String currentFieldName, ParseFieldMatcher matcher) {
-            if (matcher.match(currentFieldName, parseField) == false) {
+        void assertSupports(String parserName, XContentParser.Token token, String currentFieldName) {
+            if (parseField.match(currentFieldName) == false) {
                 throw new IllegalStateException("[" + parserName  + "] parsefield doesn't accept: " + currentFieldName);
             }
             if (supportedTokens.contains(token) == false) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java b/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
index 81f5b995c18..0282fba7646 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ParseFieldRegistry.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.collect.Tuple;
 
@@ -75,12 +74,11 @@ public class ParseFieldRegistry<T> {
      * Lookup a value from the registry by name while checking that the name matches the ParseField.
      *
      * @param name The name of the thing to look up.
-     * @param parseFieldMatcher to build nice error messages.
      * @return The value being looked up. Never null.
      * @throws ParsingException if the named thing isn't in the registry or the name was deprecated and deprecated names aren't supported.
      */
-    public T lookup(String name, ParseFieldMatcher parseFieldMatcher, XContentLocation xContentLocation) {
-        T value = lookupReturningNullIfNotFound(name, parseFieldMatcher);
+    public T lookup(String name, XContentLocation xContentLocation) {
+        T value = lookupReturningNullIfNotFound(name);
         if (value == null) {
             throw new ParsingException(xContentLocation, "no [" + registryName + "] registered for [" + name + "]");
         }
@@ -91,19 +89,17 @@ public class ParseFieldRegistry<T> {
      * Lookup a value from the registry by name while checking that the name matches the ParseField.
      *
      * @param name The name of the thing to look up.
-     * @param parseFieldMatcher The parseFieldMatcher. This is used to resolve the {@link ParseFieldMatcher} and to build nice
-     *        error messages.
      * @return The value being looked up or null if it wasn't found.
      * @throws ParsingException if the named thing isn't in the registry or the name was deprecated and deprecated names aren't supported.
      */
-    public T lookupReturningNullIfNotFound(String name, ParseFieldMatcher parseFieldMatcher) {
+    public T lookupReturningNullIfNotFound(String name) {
         Tuple<ParseField, T> parseFieldAndValue = registry.get(name);
         if (parseFieldAndValue == null) {
             return null;
         }
         ParseField parseField = parseFieldAndValue.v1();
         T value = parseFieldAndValue.v2();
-        boolean match = parseFieldMatcher.match(name, parseField);
+        boolean match = parseField.match(name);
         //this is always expected to match, ParseField is useful for deprecation warnings etc. here
         assert match : "ParseField did not match registered name [" + name + "][" + registryName + "]";
         return value;
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java
similarity index 94%
rename from core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java
rename to core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java
index f22aa39613f..ba6ccdfffad 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/StatusToXContentObject.java
@@ -24,7 +24,7 @@ import org.elasticsearch.rest.RestStatus;
  * Objects that can both render themselves in as json/yaml/etc and can provide a {@link RestStatus} for their response. Usually should be
  * implemented by top level responses sent back to users from REST endpoints.
  */
-public interface StatusToXContent extends ToXContent {
+public interface StatusToXContentObject extends ToXContentObject {
 
     /**
      * Returns the REST status to make sure it is returned correctly
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
index 01111fa940a..3006363a4dd 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
@@ -26,6 +26,8 @@ import java.util.Map;
 
 /**
  * An interface allowing to transfer an object to "XContent" using an {@link XContentBuilder}.
+ * The output may or may not be a value object. Objects implementing {@link ToXContentObject} output a valid value
+ * but those that don't may or may not require emitting a startObject and an endObject.
  */
 public interface ToXContent {
 
@@ -126,4 +128,8 @@ public interface ToXContent {
     }
 
     XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException;
+
+    default boolean isFragment() {
+        return true;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
similarity index 60%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
rename to core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
index 9dfc3be57e3..ed9aa304719 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContentObject.java
@@ -16,19 +16,19 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
 
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-
-import java.io.IOException;
+package org.elasticsearch.common.xcontent;
 
 /**
- * Parser for is_true assert sections
+ * An interface allowing to transfer an object to "XContent" using an {@link XContentBuilder}.
+ * The difference between {@link ToXContent} and {@link ToXContentObject} is that the former may output a fragment that
+ * requires to start and end a new anonymous object externally, while the latter guarantees that what gets printed
+ * out is fully valid syntax without any external addition.
  */
-public class IsTrueParser implements ClientYamlTestFragmentParser<IsTrueAssertion> {
+public interface ToXContentObject extends ToXContent {
 
     @Override
-    public IsTrueAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsTrueAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
+    default boolean isFragment() {
+        return false;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
index 72210f09d9b..879b9e9d723 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContent.java
@@ -52,7 +52,7 @@ public interface XContent {
      */
     static boolean isStrictDuplicateDetectionEnabled() {
         // Don't allow duplicate keys in JSON content by default but let the user opt out
-        return Booleans.parseBooleanExact(System.getProperty("es.xcontent.strict_duplicate_detection", "true"));
+        return Booleans.parseBoolean(System.getProperty("es.xcontent.strict_duplicate_detection", "true"));
     }
 
     /**
@@ -83,31 +83,31 @@ public interface XContent {
     /**
      * Creates a parser over the provided string content.
      */
-    XContentParser createParser(String content) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException;
 
     /**
      * Creates a parser over the provided input stream.
      */
-    XContentParser createParser(InputStream is) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(byte[] data) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(byte[] data, int offset, int length) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException;
 
     /**
      * Creates a parser over the provided bytes.
      */
-    XContentParser createParser(BytesReference bytes) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException;
 
     /**
      * Creates a parser over the provided reader.
      */
-    XContentParser createParser(Reader reader) throws IOException;
+    XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException;
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index df34ec726fd..189e9d3c8d5 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -964,21 +964,60 @@ public final class XContentBuilder implements BytesStream, Releasable, Flushable
     // Raw fields
     //////////////////////////////////
 
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     * @deprecated use {@link #rawField(String, InputStream, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     public XContentBuilder rawField(String name, InputStream value) throws IOException {
         generator.writeRawField(name, value);
         return this;
     }
 
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     */
+    public XContentBuilder rawField(String name, InputStream value, XContentType contentType) throws IOException {
+        generator.writeRawField(name, value, contentType);
+        return this;
+    }
+
+    /**
+     * Writes a raw field with the given bytes as the value
+     * @deprecated use {@link #rawField(String name, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     public XContentBuilder rawField(String name, BytesReference value) throws IOException {
         generator.writeRawField(name, value);
         return this;
     }
 
+    /**
+     * Writes a raw field with the given bytes as the value
+     */
+    public XContentBuilder rawField(String name, BytesReference value, XContentType contentType) throws IOException {
+        generator.writeRawField(name, value, contentType);
+        return this;
+    }
+
+    /**
+     * Writes a value with the source coming directly from the bytes
+     * @deprecated use {@link #rawValue(BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     public XContentBuilder rawValue(BytesReference value) throws IOException {
         generator.writeRawValue(value);
         return this;
     }
 
+    /**
+     * Writes a value with the source coming directly from the bytes
+     */
+    public XContentBuilder rawValue(BytesReference value, XContentType contentType) throws IOException {
+        generator.writeRawValue(value, contentType);
+        return this;
+    }
+
     public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException {
         generator.copyCurrentStructure(parser);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
index 478f3a8a08f..60a188ca6ce 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
@@ -86,12 +86,42 @@ public interface XContentGenerator extends Closeable, Flushable {
 
     void writeBinary(byte[] value, int offset, int length) throws IOException;
 
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     * @deprecated use {@link #writeRawField(String, InputStream, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     void writeRawField(String name, InputStream value) throws IOException;
 
+    /**
+     * Writes a raw field with the value taken from the bytes in the stream
+     */
+    void writeRawField(String name, InputStream value, XContentType xContentType) throws IOException;
+
+    /**
+     * Writes a raw field with the given bytes as the value
+     * @deprecated use {@link #writeRawField(String, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     void writeRawField(String name, BytesReference value) throws IOException;
 
+    /**
+     * Writes a raw field with the given bytes as the value
+     */
+    void writeRawField(String name, BytesReference value, XContentType xContentType) throws IOException;
+
+    /**
+     * Writes a value with the source coming directly from the bytes
+     * @deprecated use {@link #writeRawValue(BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
     void writeRawValue(BytesReference value) throws IOException;
 
+    /**
+     * Writes a value with the source coming directly from the bytes
+     */
+    void writeRawValue(BytesReference value, XContentType xContentType) throws IOException;
+
     void copyCurrentStructure(XContentParser parser) throws IOException;
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index 1625289e528..dd7508280d9 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -41,23 +41,61 @@ import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 @SuppressWarnings("unchecked")
 public class XContentHelper {
 
-    public static XContentParser createParser(BytesReference bytes) throws IOException {
+    /**
+     * Creates a parser based on the bytes provided
+     * @deprecated use {@link #createParser(NamedXContentRegistry, BytesReference, XContentType)} to avoid content type auto-detection
+     */
+    @Deprecated
+    public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
         Compressor compressor = CompressorFactory.compressor(bytes);
         if (compressor != null) {
             InputStream compressedInput = compressor.streamInput(bytes.streamInput());
             if (compressedInput.markSupported() == false) {
                 compressedInput = new BufferedInputStream(compressedInput);
             }
-            XContentType contentType = XContentFactory.xContentType(compressedInput);
-            return XContentFactory.xContent(contentType).createParser(compressedInput);
+            final XContentType contentType = XContentFactory.xContentType(compressedInput);
+            return XContentFactory.xContent(contentType).createParser(xContentRegistry, compressedInput);
         } else {
-            return XContentFactory.xContent(bytes).createParser(bytes.streamInput());
+            return XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes.streamInput());
         }
     }
 
-    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered) throws ElasticsearchParseException {
+    /**
+     * Creates a parser for the bytes using the supplied content-type
+     */
+    public static XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes,
+                                              XContentType xContentType) throws IOException {
+        Objects.requireNonNull(xContentType);
+        Compressor compressor = CompressorFactory.compressor(bytes);
+        if (compressor != null) {
+            InputStream compressedInput = compressor.streamInput(bytes.streamInput());
+            if (compressedInput.markSupported() == false) {
+                compressedInput = new BufferedInputStream(compressedInput);
+            }
+            return XContentFactory.xContent(xContentType).createParser(xContentRegistry, compressedInput);
+        } else {
+            return xContentType.xContent().createParser(xContentRegistry, bytes.streamInput());
+        }
+    }
+
+    /**
+     * Converts the given bytes into a map that is optionally ordered.
+     * @deprecated this method relies on auto-detection of content type. Use {@link #convertToMap(BytesReference, boolean, XContentType)}
+     *             instead with the proper {@link XContentType}
+     */
+    @Deprecated
+    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered)
+            throws ElasticsearchParseException {
+        return convertToMap(bytes, ordered, null);
+    }
+
+    /**
+     * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null.
+     */
+    public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType)
+        throws ElasticsearchParseException {
         try {
-            XContentType contentType;
+            final XContentType contentType;
             InputStream input;
             Compressor compressor = CompressorFactory.compressor(bytes);
             if (compressor != null) {
@@ -65,34 +103,68 @@ public class XContentHelper {
                 if (compressedStreamInput.markSupported() == false) {
                     compressedStreamInput = new BufferedInputStream(compressedStreamInput);
                 }
-                contentType = XContentFactory.xContentType(compressedStreamInput);
                 input = compressedStreamInput;
             } else {
-                contentType = XContentFactory.xContentType(bytes);
                 input = bytes.streamInput();
             }
-            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(input)) {
-                if (ordered) {
-                    return Tuple.tuple(contentType, parser.mapOrdered());
-                } else {
-                    return Tuple.tuple(contentType, parser.map());
-                }
-            }
+            contentType = xContentType != null ? xContentType : XContentFactory.xContentType(input);
+            return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), input, ordered));
         } catch (IOException e) {
             throw new ElasticsearchParseException("Failed to parse content to map", e);
         }
     }
 
+    /**
+     * Convert a string in some {@link XContent} format to a {@link Map}. Throws an {@link ElasticsearchParseException} if there is any
+     * error.
+     */
+    public static Map<String, Object> convertToMap(XContent xContent, String string, boolean ordered) throws ElasticsearchParseException {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, string)) {
+            return ordered ? parser.mapOrdered() : parser.map();
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("Failed to parse content to map", e);
+        }
+    }
+
+    /**
+     * Convert a string in some {@link XContent} format to a {@link Map}. Throws an {@link ElasticsearchParseException} if there is any
+     * error. Note that unlike {@link #convertToMap(BytesReference, boolean)}, this doesn't automatically uncompress the input.
+     */
+    public static Map<String, Object> convertToMap(XContent xContent, InputStream input, boolean ordered)
+            throws ElasticsearchParseException {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
+            return ordered ? parser.mapOrdered() : parser.map();
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("Failed to parse content to map", e);
+        }
+    }
+
+    @Deprecated
     public static String convertToJson(BytesReference bytes, boolean reformatJson) throws IOException {
         return convertToJson(bytes, reformatJson, false);
     }
 
+    @Deprecated
     public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException {
-        XContentType xContentType = XContentFactory.xContentType(bytes);
+        return convertToJson(bytes, reformatJson, prettyPrint, XContentFactory.xContentType(bytes));
+    }
+
+    public static String convertToJson(BytesReference bytes, boolean reformatJson, XContentType xContentType) throws IOException {
+        return convertToJson(bytes, reformatJson, false, xContentType);
+    }
+
+    public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint, XContentType xContentType)
+        throws IOException {
+        Objects.requireNonNull(xContentType);
         if (xContentType == XContentType.JSON && !reformatJson) {
             return bytes.utf8ToString();
         }
-        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) {
+
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(NamedXContentRegistry.EMPTY,
+                bytes.streamInput())) {
             parser.nextToken();
             XContentBuilder builder = XContentFactory.jsonBuilder();
             if (prettyPrint) {
@@ -191,7 +263,6 @@ public class XContentHelper {
      * Merges the defaults provided as the second parameter into the content of the first. Only does recursive merge
      * for inner maps.
      */
-    @SuppressWarnings({"unchecked"})
     public static void mergeDefaults(Map<String, Object> content, Map<String, Object> defaults) {
         for (Map.Entry<String, Object> defaultEntry : defaults.entrySet()) {
             if (!content.containsKey(defaultEntry.getKey())) {
@@ -255,33 +326,36 @@ public class XContentHelper {
         return true;
     }
 
-    public static void copyCurrentStructure(XContentGenerator generator, XContentParser parser) throws IOException {
+    /**
+     * Low level implementation detail of {@link XContentGenerator#copyCurrentStructure(XContentParser)}.
+     */
+    public static void copyCurrentStructure(XContentGenerator destination, XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
 
         // Let's handle field-name separately first
         if (token == XContentParser.Token.FIELD_NAME) {
-            generator.writeFieldName(parser.currentName());
+            destination.writeFieldName(parser.currentName());
             token = parser.nextToken();
             // fall-through to copy the associated value
         }
 
         switch (token) {
             case START_ARRAY:
-                generator.writeStartArray();
+                destination.writeStartArray();
                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                    copyCurrentStructure(generator, parser);
+                    copyCurrentStructure(destination, parser);
                 }
-                generator.writeEndArray();
+                destination.writeEndArray();
                 break;
             case START_OBJECT:
-                generator.writeStartObject();
+                destination.writeStartObject();
                 while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                    copyCurrentStructure(generator, parser);
+                    copyCurrentStructure(destination, parser);
                 }
-                generator.writeEndObject();
+                destination.writeEndObject();
                 break;
             default: // others are simple:
-                copyCurrentEvent(generator, parser);
+                copyCurrentEvent(destination, parser);
         }
     }
 
@@ -339,7 +413,10 @@ public class XContentHelper {
     /**
      * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
      * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference)}.
+     * @deprecated use {@link #writeRawField(String, BytesReference, XContentType, XContentBuilder, Params)} to avoid content type
+     * auto-detection
      */
+    @Deprecated
     public static void writeRawField(String field, BytesReference source, XContentBuilder builder, ToXContent.Params params) throws IOException {
         Compressor compressor = CompressorFactory.compressor(source);
         if (compressor != null) {
@@ -349,4 +426,38 @@ public class XContentHelper {
             builder.rawField(field, source);
         }
     }
+
+    /**
+     * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
+     * {@link XContentBuilder#rawField(String, org.elasticsearch.common.bytes.BytesReference, XContentType)}.
+     */
+    public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder,
+                                     ToXContent.Params params) throws IOException {
+        Objects.requireNonNull(xContentType);
+        Compressor compressor = CompressorFactory.compressor(source);
+        if (compressor != null) {
+            InputStream compressedStreamInput = compressor.streamInput(source.streamInput());
+            builder.rawField(field, compressedStreamInput, xContentType);
+        } else {
+            builder.rawField(field, source, xContentType);
+        }
+    }
+
+    /**
+     * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
+     * {@link XContentType}. Wraps the output into a new anonymous object.
+     */
+    public static BytesReference toXContent(ToXContent toXContent, XContentType xContentType, boolean humanReadable) throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            builder.humanReadable(humanReadable);
+            if (toXContent.isFragment()) {
+                builder.startObject();
+            }
+            toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            if (toXContent.isFragment()) {
+                builder.endObject();
+            }
+            return builder.bytes();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
index e5ab2a9f4c7..fc0b5c0f4f2 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java
@@ -33,7 +33,7 @@ import java.util.Map;
  *
  * <pre>
  *     XContentType xContentType = XContentType.JSON;
- *     XContentParser parser = xContentType.xContent().createParser("{\"key\" : \"value\"}");
+ *     XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, "{\"key\" : \"value\"}");
  * </pre>
  */
 public interface XContentParser extends Releasable {
@@ -201,16 +201,32 @@ public interface XContentParser extends Releasable {
 
     double doubleValue() throws IOException;
 
+    /**
+     * @return true iff the current value is either boolean (<code>true</code> or <code>false</code>) or one of "false", "true".
+     */
+    boolean isBooleanValue() throws IOException;
+
+    boolean booleanValue() throws IOException;
+
+    // TODO #22298: Remove this method and replace all call sites with #isBooleanValue()
     /**
      * returns true if the current value is boolean in nature.
      * values that are considered booleans:
      * - boolean value (true/false)
      * - numeric integers (=0 is considered as false, !=0 is true)
      * - one of the following strings: "true","false","on","off","yes","no","1","0"
+     *
+     * @deprecated Just present for providing backwards compatibility. Use {@link #isBooleanValue()} instead.
      */
-    boolean isBooleanValue() throws IOException;
+    @Deprecated
+    boolean isBooleanValueLenient() throws IOException;
 
-    boolean booleanValue() throws IOException;
+    // TODO #22298: Remove this method and replace all call sites with #booleanValue()
+    /**
+     * @deprecated Just present for providing backwards compatibility. Use {@link #booleanValue()} instead.
+     */
+    @Deprecated
+    boolean booleanValueLenient() throws IOException;
 
     /**
      * Reads a plain binary value that was written via one of the following methods:
@@ -249,5 +265,16 @@ public interface XContentParser extends Releasable {
      */
     XContentLocation getTokenLocation();
 
+    // TODO remove context entirely when it isn't needed
+    /**
+     * Parse an object by name.
+     */
+    <T> T namedObject(Class<T> categoryClass, String name, Object context) throws IOException;
+
+    /**
+     * The registry used to resolve {@link #namedObject(Class, String, Object)}. Use this when building a sub-parser from this parser.
+     */
+    NamedXContentRegistry getXContentRegistry();
+
     boolean isClosed();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
index a2180152444..fec83eefbdf 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentParserUtils.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 
 import java.io.IOException;
@@ -35,32 +36,17 @@ public final class XContentParserUtils {
     private XContentParserUtils() {
     }
 
-    /**
-     * Makes sure that current token is of type {@link XContentParser.Token#FIELD_NAME}
-     *
-     * @return the token
-     * @throws ParsingException if the token is not of type {@link XContentParser.Token#FIELD_NAME}
-     */
-    public static Token ensureFieldName(Token token, Supplier<XContentLocation> location) throws IOException {
-        return ensureType(Token.FIELD_NAME, token, location);
-    }
-
     /**
      * Makes sure that current token is of type {@link XContentParser.Token#FIELD_NAME} and the the field name is equal to the provided one
-     *
-     * @return the token
-     * @throws ParsingException if the token is not of type {@link XContentParser.Token#FIELD_NAME} or is not equal to the given
-     *                          field name
+     * @throws ParsingException if the token is not of type {@link XContentParser.Token#FIELD_NAME} or is not equal to the given field name
      */
-    public static Token ensureFieldName(XContentParser parser, Token token, String fieldName) throws IOException {
-        Token t = ensureType(Token.FIELD_NAME, token, parser::getTokenLocation);
-
-        String current = parser.currentName() != null ? parser.currentName() : "<null>";
-        if (current.equals(fieldName) == false) {
+    public static void ensureFieldName(XContentParser parser, Token token, String fieldName) throws IOException {
+        ensureExpectedToken(Token.FIELD_NAME, token, parser::getTokenLocation);
+        String currentName = parser.currentName();
+        if (currentName.equals(fieldName) == false) {
             String message = "Failed to parse object: expecting field with name [%s] but found [%s]";
-            throw new ParsingException(parser.getTokenLocation(), String.format(Locale.ROOT, message, fieldName, current));
+            throw new ParsingException(parser.getTokenLocation(), String.format(Locale.ROOT, message, fieldName, currentName));
         }
-        return t;
     }
 
     /**
@@ -72,16 +58,53 @@ public final class XContentParserUtils {
     }
 
     /**
-     * Makes sure that current token is of the expected type
+     * @throws ParsingException with a "unknown token found" reason
+     */
+    public static void throwUnknownToken(XContentParser.Token token, XContentLocation location) {
+        String message = "Failed to parse object: unexpected token [%s] found";
+        throw new ParsingException(location, String.format(Locale.ROOT, message, token));
+    }
+
+    /**
+     * Makes sure that provided token is of the expected type
      *
-     * @return the token
      * @throws ParsingException if the token is not equal to the expected type
      */
-    private static Token ensureType(Token expected, Token current, Supplier<XContentLocation> location) {
-        if (current != expected) {
+    public static void ensureExpectedToken(Token expected, Token actual, Supplier<XContentLocation> location) {
+        if (actual != expected) {
             String message = "Failed to parse object: expecting token of type [%s] but found [%s]";
-            throw new ParsingException(location.get(), String.format(Locale.ROOT, message, expected, current));
+            throw new ParsingException(location.get(), String.format(Locale.ROOT, message, expected, actual));
         }
-        return current;
+    }
+
+    /**
+     * Parse the current token depending on its token type. The following token types will be
+     * parsed by the corresponding parser methods:
+     * <ul>
+     *    <li>XContentParser.Token.VALUE_STRING: parser.text()</li>
+     *    <li>XContentParser.Token.VALUE_NUMBER: parser.numberValue()</li>
+     *    <li>XContentParser.Token.VALUE_BOOLEAN: parser.booleanValue()</li>
+     *    <li>XContentParser.Token.VALUE_EMBEDDED_OBJECT: parser.binaryValue()</li>
+     * </ul>
+     *
+     * @throws ParsingException if the token none of the allowed values
+     */
+    public static Object parseStoredFieldsValue(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        Object value = null;
+        if (token == XContentParser.Token.VALUE_STRING) {
+            //binary values will be parsed back and returned as base64 strings when reading from json and yaml
+            value = parser.text();
+        } else if (token == XContentParser.Token.VALUE_NUMBER) {
+            value = parser.numberValue();
+        } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
+            value = parser.booleanValue();
+        } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+            //binary values will be parsed back and returned as BytesArray when reading from cbor and smile
+            value = new BytesArray(parser.binaryValue());
+        } else {
+            throwUnknownToken(token, parser.getTokenLocation());
+        }
+        return value;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
index ddd736e0d00..8e3c2982704 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentType.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 
 import java.io.IOException;
 import java.util.Locale;
+import java.util.Objects;
 
 /**
  * The content type of {@link org.elasticsearch.common.xcontent.XContent}.
@@ -40,7 +41,7 @@ public enum XContentType implements Writeable {
      */
     JSON(0) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/json";
         }
 
@@ -64,7 +65,7 @@ public enum XContentType implements Writeable {
      */
     SMILE(1) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/smile";
         }
 
@@ -83,7 +84,7 @@ public enum XContentType implements Writeable {
      */
     YAML(2) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/yaml";
         }
 
@@ -102,7 +103,7 @@ public enum XContentType implements Writeable {
      */
     CBOR(3) {
         @Override
-        protected String mediaTypeWithoutParameters() {
+        public String mediaTypeWithoutParameters() {
             return "application/cbor";
         }
 
@@ -117,12 +118,18 @@ public enum XContentType implements Writeable {
         }
     };
 
+    /**
+     * Accepts either a format string, which is equivalent to {@link XContentType#shortName()} or a media type that optionally has
+     * parameters and attempts to match the value to an {@link XContentType}. The comparisons are done in lower case format and this method
+     * also supports a wildcard accept for {@code application/*}. This method can be used to parse the {@code Accept} HTTP header or a
+     * format query string parameter. This method will return {@code null} if no match is found
+     */
     public static XContentType fromMediaTypeOrFormat(String mediaType) {
         if (mediaType == null) {
             return null;
         }
         for (XContentType type : values()) {
-            if (isSameMediaTypeAs(mediaType, type)) {
+            if (isSameMediaTypeOrFormatAs(mediaType, type)) {
                 return type;
             }
         }
@@ -133,7 +140,22 @@ public enum XContentType implements Writeable {
         return null;
     }
 
-    private static boolean isSameMediaTypeAs(String stringType, XContentType type) {
+    /**
+     * Attempts to match the given media type with the known {@link XContentType} values. This match is done in a case-insensitive manner.
+     * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type}
+     * HTTP header. This method will return {@code null} if no match is found
+     */
+    public static XContentType fromMediaType(String mediaType) {
+        final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT);
+        for (XContentType type : values()) {
+            if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) {
+                return type;
+            }
+        }
+        return null;
+    }
+
+    private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) {
         return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) ||
                 stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") ||
                 type.shortName().equalsIgnoreCase(stringType);
@@ -157,7 +179,7 @@ public enum XContentType implements Writeable {
 
     public abstract XContent xContent();
 
-    protected abstract String mediaTypeWithoutParameters();
+    public abstract String mediaTypeWithoutParameters();
 
     public static XContentType readFrom(StreamInput in) throws IOException {
         int index = in.readVInt();
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
index d79173cfc2b..56435fd364b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -78,33 +79,33 @@ public class CborXContent implements XContent {
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new CborXContentParser(cborFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new CborXContentParser(xContentRegistry, cborFactory.createParser(reader));
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
index e63a928109d..119cb5c98c6 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class CborXContentGenerator extends JsonXContentGenerator {
 
-    public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public CborXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
index 772a5322cc7..61b4886420f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java
@@ -20,13 +20,15 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
 public class CborXContentParser extends JsonXContentParser {
 
-    public CborXContentParser(JsonParser parser) {
-        super(parser);
+    public CborXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
index 1b0b351e6ef..2e4393723e0 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -79,32 +80,32 @@ public class JsonXContent implements XContent {
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new JsonXContentParser(jsonFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new JsonXContentParser(xContentRegistry, jsonFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index 763fac4c6a3..1e09f8334f7 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -31,6 +31,7 @@ import com.fasterxml.jackson.core.util.JsonGeneratorDelegate;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -43,7 +44,6 @@ import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Objects;
 import java.util.Set;
 
@@ -72,10 +72,6 @@ public class JsonXContentGenerator implements XContentGenerator {
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
     private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         Objects.requireNonNull(includes, "Including filters must not be null");
         Objects.requireNonNull(excludes, "Excluding filters must not be null");
@@ -311,8 +307,14 @@ public class JsonXContentGenerator implements XContentGenerator {
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawField(name, content, contentType);
+    }
+
+    @Override
+    public void writeRawField(String name, InputStream content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
-            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(content)) {
+            // EMPTY is safe here because we never call namedObject when writing raw data
+            try (XContentParser parser = XContentFactory.xContent(contentType).createParser(NamedXContentRegistry.EMPTY, content)) {
                 parser.nextToken();
                 writeFieldName(name);
                 copyCurrentStructure(parser);
@@ -331,6 +333,11 @@ public class JsonXContentGenerator implements XContentGenerator {
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawField(name, content, contentType);
+    }
+
+    @Override
+    public final void writeRawField(String name, BytesReference content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
             writeFieldName(name);
             copyRawValue(content, contentType.xContent());
@@ -348,6 +355,11 @@ public class JsonXContentGenerator implements XContentGenerator {
         if (contentType == null) {
             throw new IllegalArgumentException("Can't write raw bytes whose xcontent-type can't be guessed");
         }
+        writeRawValue(content, contentType);
+    }
+
+    @Override
+    public final void writeRawValue(BytesReference content, XContentType contentType) throws IOException {
         if (mayWriteRawData(contentType) == false) {
             copyRawValue(content, contentType.xContent());
         } else {
@@ -378,8 +390,9 @@ public class JsonXContentGenerator implements XContentGenerator {
     }
 
     protected void copyRawValue(BytesReference content, XContent xContent) throws IOException {
+        // EMPTY is safe here because we never call namedObject
         try (StreamInput input = content.streamInput();
-             XContentParser parser = xContent.createParser(input)) {
+             XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
             copyCurrentStructure(parser);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
index f7ed46a6496..e5c30208ed6 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java
@@ -22,8 +22,10 @@ package org.elasticsearch.common.xcontent.json;
 import com.fasterxml.jackson.core.JsonLocation;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
+
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.AbstractXContentParser;
@@ -35,7 +37,8 @@ public class JsonXContentParser extends AbstractXContentParser {
 
     final JsonParser parser;
 
-    public JsonXContentParser(JsonParser parser) {
+    public JsonXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry);
         this.parser = parser;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
index 643326cd82f..b43a13a9193 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
@@ -26,6 +26,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -79,32 +80,32 @@ public class SmileXContent implements XContent {
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new SmileXContentParser(smileFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new SmileXContentParser(xContentRegistry, smileFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
index afa420805f7..f368c0e383f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class SmileXContentGenerator extends JsonXContentGenerator {
 
-    public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public SmileXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
index ad8e12e70bf..c7b4b8c000c 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java
@@ -20,13 +20,15 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
 public class SmileXContentParser extends JsonXContentParser {
 
-    public SmileXContentParser(JsonParser parser) {
-        super(parser);
+    public SmileXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
index d13dcbd9c93..95fe08d96c3 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
@@ -22,6 +22,7 @@ package org.elasticsearch.common.xcontent.support;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -49,7 +50,11 @@ public abstract class AbstractXContentParser implements XContentParser {
         }
     }
 
+    private final NamedXContentRegistry xContentRegistry;
 
+    public AbstractXContentParser(NamedXContentRegistry xContentRegistry) {
+        this.xContentRegistry = xContentRegistry;
+    }
 
     // The 3rd party parsers we rely on are known to silently truncate fractions: see
     //   http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue()
@@ -72,9 +77,6 @@ public abstract class AbstractXContentParser implements XContentParser {
         switch (currentToken()) {
             case VALUE_BOOLEAN:
                 return true;
-            case VALUE_NUMBER:
-                NumberType numberType = numberType();
-                return numberType == NumberType.LONG || numberType == NumberType.INT;
             case VALUE_STRING:
                 return Booleans.isBoolean(textCharacters(), textOffset(), textLength());
             default:
@@ -84,11 +86,37 @@ public abstract class AbstractXContentParser implements XContentParser {
 
     @Override
     public boolean booleanValue() throws IOException {
+        Token token = currentToken();
+        if (token == Token.VALUE_STRING) {
+            return Booleans.parseBoolean(textCharacters(), textOffset(), textLength(), false /* irrelevant */);
+        }
+        return doBooleanValue();
+    }
+
+    @Override
+    @Deprecated
+    public boolean isBooleanValueLenient() throws IOException {
+        switch (currentToken()) {
+            case VALUE_BOOLEAN:
+                return true;
+            case VALUE_NUMBER:
+                NumberType numberType = numberType();
+                return numberType == NumberType.LONG || numberType == NumberType.INT;
+            case VALUE_STRING:
+                return Booleans.isBooleanLenient(textCharacters(), textOffset(), textLength());
+            default:
+                return false;
+        }
+    }
+
+    @Override
+    @Deprecated
+    public boolean booleanValueLenient() throws IOException {
         Token token = currentToken();
         if (token == Token.VALUE_NUMBER) {
             return intValue() != 0;
         } else if (token == Token.VALUE_STRING) {
-            return Booleans.parseBoolean(textCharacters(), textOffset(), textLength(), false /* irrelevant */);
+            return Booleans.parseBooleanLenient(textCharacters(), textOffset(), textLength(), false /* irrelevant */);
         }
         return doBooleanValue();
     }
@@ -356,6 +384,16 @@ public abstract class AbstractXContentParser implements XContentParser {
         return null;
     }
 
+    @Override
+    public <T> T namedObject(Class<T> categoryClass, String name, Object context) throws IOException {
+        return xContentRegistry.parseNamedObject(categoryClass, name, this, context);
+    }
+
+    @Override
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
+
     @Override
     public abstract boolean isClosed();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
index a1affb4fe57..36eacb81f83 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
@@ -24,6 +24,7 @@ import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
@@ -263,7 +264,7 @@ public class XContentMapValues {
 
                 List<Object> filteredValue = filter((Iterable<?>) value,
                         subIncludeAutomaton, subIncludeState, excludeAutomaton, excludeState, matchAllAutomaton);
-                if (includeAutomaton.isAccept(includeState) || filteredValue.isEmpty() == false) {
+                if (filteredValue.isEmpty() == false) {
                     filtered.put(key, filteredValue);
                 }
 
@@ -286,6 +287,7 @@ public class XContentMapValues {
             CharacterRunAutomaton excludeAutomaton, int initialExcludeState,
             CharacterRunAutomaton matchAllAutomaton) {
         List<Object> filtered = new ArrayList<>();
+        boolean isInclude = includeAutomaton.isAccept(initialIncludeState);
         for (Object value : iterable) {
             if (value instanceof Map) {
                 int includeState = includeAutomaton.step(initialIncludeState, '.');
@@ -304,9 +306,8 @@ public class XContentMapValues {
                 if (filteredValue.isEmpty() == false) {
                     filtered.add(filteredValue);
                 }
-            } else {
-                // TODO: we have tests relying on this behavior on arrays even
-                // if the path does not match, but this looks like a bug?
+            } else if (isInclude) {
+                // #22557: only accept this array value if the key we are on is accepted:
                 filtered.add(value);
             }
         }
@@ -412,39 +413,29 @@ public class XContentMapValues {
         return Long.parseLong(node.toString());
     }
 
-    /**
-     * This method is very lenient, use {@link #nodeBooleanValue} instead.
-     */
-    public static boolean lenientNodeBooleanValue(Object node, boolean defaultValue) {
-        if (node == null) {
-            return defaultValue;
+    public static boolean nodeBooleanValue(Object node, String name, boolean defaultValue) {
+        try {
+            return nodeBooleanValue(node, defaultValue);
+        } catch (IllegalArgumentException ex) {
+            throw new IllegalArgumentException("Could not convert [" + name + "] to boolean", ex);
         }
-        return lenientNodeBooleanValue(node);
     }
 
-    /**
-     * This method is very lenient, use {@link #nodeBooleanValue} instead.
-     */
-    public static boolean lenientNodeBooleanValue(Object node) {
-        if (node instanceof Boolean) {
-            return (Boolean) node;
+    public static boolean nodeBooleanValue(Object node, boolean defaultValue) {
+        String nodeValue = node == null ? null : node.toString();
+        return Booleans.parseBoolean(nodeValue, defaultValue);
+    }
+
+    public static boolean nodeBooleanValue(Object node, String name) {
+        try {
+            return nodeBooleanValue(node);
+        } catch (IllegalArgumentException ex) {
+            throw new IllegalArgumentException("Could not convert [" + name + "] to boolean", ex);
         }
-        if (node instanceof Number) {
-            return ((Number) node).intValue() != 0;
-        }
-        String value = node.toString();
-        return !(value.equals("false") || value.equals("0") || value.equals("off"));
     }
 
     public static boolean nodeBooleanValue(Object node) {
-        switch (node.toString()) {
-        case "true":
-            return true;
-        case "false":
-            return false;
-        default:
-            throw new IllegalArgumentException("Can't parse boolean value [" + node + "], expected [true] or [false]");
-        }
+        return Booleans.parseBoolean(node.toString());
     }
 
     public static TimeValue nodeTimeValue(Object node, TimeValue defaultValue) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
index 7413f05f583..56dda843c45 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FastStringReader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentGenerator;
@@ -74,32 +75,32 @@ public class YamlXContent implements XContent {
     }
 
     @Override
-    public XContentParser createParser(String content) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(new FastStringReader(content)));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, String content) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(new FastStringReader(content)));
     }
 
     @Override
-    public XContentParser createParser(InputStream is) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(is));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, InputStream is) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(is));
     }
 
     @Override
-    public XContentParser createParser(byte[] data) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(data));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(data));
     }
 
     @Override
-    public XContentParser createParser(byte[] data, int offset, int length) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(data, offset, length));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, byte[] data, int offset, int length) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(data, offset, length));
     }
 
     @Override
-    public XContentParser createParser(BytesReference bytes) throws IOException {
-        return createParser(bytes.streamInput());
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, BytesReference bytes) throws IOException {
+        return createParser(xContentRegistry, bytes.streamInput());
     }
 
     @Override
-    public XContentParser createParser(Reader reader) throws IOException {
-        return new YamlXContentParser(yamlFactory.createParser(reader));
+    public XContentParser createParser(NamedXContentRegistry xContentRegistry, Reader reader) throws IOException {
+        return new YamlXContentParser(xContentRegistry, yamlFactory.createParser(reader));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
index d2c53c8a020..0d969c21a0f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentGenerator.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentGenerator;
 
 import java.io.OutputStream;
-import java.util.Collections;
 import java.util.Set;
 
 public class YamlXContentGenerator extends JsonXContentGenerator {
 
-    public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os) {
-        this(jsonGenerator, os, Collections.emptySet(), Collections.emptySet());
-    }
-
     public YamlXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, Set<String> includes, Set<String> excludes) {
         super(jsonGenerator, os, includes, excludes);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
index 5efceac7dcf..c2fdcfa740b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java
@@ -20,13 +20,15 @@
 package org.elasticsearch.common.xcontent.yaml;
 
 import com.fasterxml.jackson.core.JsonParser;
+
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 
 public class YamlXContentParser extends JsonXContentParser {
 
-    public YamlXContentParser(JsonParser parser) {
-        super(parser);
+    public YamlXContentParser(NamedXContentRegistry xContentRegistry, JsonParser parser) {
+        super(xContentRegistry, parser);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
index 98ce54428c7..ea3ae0c919b 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
@@ -30,6 +30,7 @@ import java.util.function.Supplier;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -54,8 +55,9 @@ public class DiscoveryModule {
 
     private final Discovery discovery;
 
-    public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportService transportService, NetworkService networkService,
-                           ClusterService clusterService, List<DiscoveryPlugin> plugins) {
+    public DiscoveryModule(Settings settings, ThreadPool threadPool, TransportService transportService,
+                           NamedWriteableRegistry namedWriteableRegistry, NetworkService networkService, ClusterService clusterService,
+                           List<DiscoveryPlugin> plugins) {
         final UnicastHostsProvider hostsProvider;
 
         Map<String, Supplier<UnicastHostsProvider>> hostProviders = new HashMap<>();
@@ -78,10 +80,12 @@ public class DiscoveryModule {
         }
 
         Map<String, Supplier<Discovery>> discoveryTypes = new HashMap<>();
-        discoveryTypes.put("zen", () -> new ZenDiscovery(settings, threadPool, transportService, clusterService, hostsProvider));
+        discoveryTypes.put("zen",
+            () -> new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
         discoveryTypes.put("none", () -> new NoneDiscovery(settings, clusterService, clusterService.getClusterSettings()));
         for (DiscoveryPlugin plugin : plugins) {
-            plugin.getDiscoveryTypes(threadPool, transportService, clusterService, hostsProvider).entrySet().forEach(entry -> {
+            plugin.getDiscoveryTypes(threadPool, transportService, namedWriteableRegistry,
+                clusterService, hostsProvider).entrySet().forEach(entry -> {
                 if (discoveryTypes.put(entry.getKey(), entry.getValue()) != null) {
                     throw new IllegalArgumentException("Cannot register discovery type [" + entry.getKey() + "] twice");
                 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
index 7116597bdaf..f0de15da9ae 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ElectMasterService.java
@@ -24,7 +24,6 @@ import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -174,7 +173,7 @@ public class ElectMasterService extends AbstractComponent {
      * Returns the given nodes sorted by likelihood of being elected as master, most likely first.
      * Non-master nodes are not removed but are rather put in the end
      */
-    public static List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) {
+    static List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) {
         ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes);
         CollectionUtil.introSort(sortedNodes, ElectMasterService::compareNodes);
         return sortedNodes;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
index c8527168198..c65542093d3 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java
@@ -170,7 +170,7 @@ public class MembershipAction extends AbstractComponent {
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            this.state = ClusterState.Builder.readFrom(in, localNode.get());
+            this.state = ClusterState.readFrom(in, localNode.get());
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 74dbf835b58..2d84f5f863d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -24,12 +24,11 @@ import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -59,7 +58,6 @@ public class NodeJoinController extends AbstractComponent {
     private final ClusterService clusterService;
     private final AllocationService allocationService;
     private final ElectMasterService electMaster;
-    private final DiscoverySettings discoverySettings;
     private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
 
     // this is set while trying to become a master
@@ -68,12 +66,11 @@ public class NodeJoinController extends AbstractComponent {
 
 
     public NodeJoinController(ClusterService clusterService, AllocationService allocationService, ElectMasterService electMaster,
-                              DiscoverySettings discoverySettings, Settings settings) {
+                              Settings settings) {
         super(settings);
         this.clusterService = clusterService;
         this.allocationService = allocationService;
         this.electMaster = electMaster;
-        this.discoverySettings = discoverySettings;
     }
 
     /**
@@ -408,8 +405,9 @@ public class NodeJoinController extends AbstractComponent {
     class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
 
         @Override
-        public BatchResult<DiscoveryNode> execute(ClusterState currentState, List<DiscoveryNode> joiningNodes) throws Exception {
-            final BatchResult.Builder<DiscoveryNode> results = BatchResult.builder();
+        public ClusterTasksResult<DiscoveryNode> execute(ClusterState currentState, List<DiscoveryNode> joiningNodes) throws Exception {
+            final ClusterTasksResult.Builder<DiscoveryNode> results = ClusterTasksResult.builder();
+
             final DiscoveryNodes currentNodes = currentState.nodes();
             boolean nodesChanged = false;
             ClusterState.Builder newState;
@@ -471,8 +469,6 @@ public class NodeJoinController extends AbstractComponent {
             DiscoveryNodes currentNodes = currentState.nodes();
             DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes);
             nodesBuilder.masterNodeId(currentState.nodes().getLocalNodeId());
-            ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
-                .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
             for (final DiscoveryNode joiningNode : joiningNodes) {
                 final DiscoveryNode nodeWithSameId = nodesBuilder.get(joiningNode.getId());
                 if (nodeWithSameId != null && nodeWithSameId.equals(joiningNode) == false) {
@@ -490,7 +486,7 @@ public class NodeJoinController extends AbstractComponent {
 
             // now trim any left over dead nodes - either left there when the previous master stepped down
             // or removed by us above
-            ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).blocks(clusterBlocks).build();
+            ClusterState tmpState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
             return ClusterState.builder(allocationService.deassociateDeadNodes(tmpState, false,
                 "removed dead nodes on election"));
         }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
index 9f0d3576c4b..4150783a8fd 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.discovery.zen;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -34,6 +35,8 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -79,6 +82,7 @@ public class PublishClusterStateAction extends AbstractComponent {
     }
 
     private final TransportService transportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
     private final Supplier<ClusterState> clusterStateSupplier;
     private final NewPendingClusterStateListener newPendingClusterStatelistener;
     private final DiscoverySettings discoverySettings;
@@ -88,12 +92,14 @@ public class PublishClusterStateAction extends AbstractComponent {
     public PublishClusterStateAction(
             Settings settings,
             TransportService transportService,
+            NamedWriteableRegistry namedWriteableRegistry,
             Supplier<ClusterState> clusterStateSupplier,
             NewPendingClusterStateListener listener,
             DiscoverySettings discoverySettings,
             ClusterName clusterName) {
         super(settings);
         this.transportService = transportService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
         this.clusterStateSupplier = clusterStateSupplier;
         this.newPendingClusterStatelistener = listener;
         this.discoverySettings = discoverySettings;
@@ -370,33 +376,37 @@ public class PublishClusterStateAction extends AbstractComponent {
 
     protected void handleIncomingClusterStateRequest(BytesTransportRequest request, TransportChannel channel) throws IOException {
         Compressor compressor = CompressorFactory.compressor(request.bytes());
-        StreamInput in;
-        if (compressor != null) {
-            in = compressor.streamInput(request.bytes().streamInput());
-        } else {
-            in = request.bytes().streamInput();
-        }
-        in.setVersion(request.version());
-        synchronized (lastSeenClusterStateMutex) {
-            final ClusterState incomingState;
-            // If true we received full cluster state - otherwise diffs
-            if (in.readBoolean()) {
-                incomingState = ClusterState.Builder.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
-                logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length());
-            } else if (lastSeenClusterState != null) {
-                Diff<ClusterState> diff = lastSeenClusterState.readDiffFrom(in);
-                incomingState = diff.apply(lastSeenClusterState);
-                logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]",
-                    incomingState.version(), incomingState.stateUUID(), request.bytes().length());
-            } else {
-                logger.debug("received diff for but don't have any local cluster state - requesting full state");
-                throw new IncompatibleClusterStateVersionException("have no local cluster state");
+        StreamInput in = request.bytes().streamInput();
+        try {
+            if (compressor != null) {
+                in = compressor.streamInput(in);
             }
-            // sanity check incoming state
-            validateIncomingState(incomingState, lastSeenClusterState);
+            in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
+            in.setVersion(request.version());
+            synchronized (lastSeenClusterStateMutex) {
+                final ClusterState incomingState;
+                // If true we received full cluster state - otherwise diffs
+                if (in.readBoolean()) {
+                    incomingState = ClusterState.readFrom(in, clusterStateSupplier.get().nodes().getLocalNode());
+                    logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(),
+                        request.bytes().length());
+                } else if (lastSeenClusterState != null) {
+                    Diff<ClusterState> diff = ClusterState.readDiffFrom(in, lastSeenClusterState.nodes().getLocalNode());
+                    incomingState = diff.apply(lastSeenClusterState);
+                    logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]",
+                        incomingState.version(), incomingState.stateUUID(), request.bytes().length());
+                } else {
+                    logger.debug("received diff for but don't have any local cluster state - requesting full state");
+                    throw new IncompatibleClusterStateVersionException("have no local cluster state");
+                }
+                // sanity check incoming state
+                validateIncomingState(incomingState, lastSeenClusterState);
 
-            pendingStatesQueue.addPending(incomingState);
-            lastSeenClusterState = incomingState;
+                pendingStatesQueue.addPending(incomingState);
+                lastSeenClusterState = incomingState;
+            }
+        } finally {
+            IOUtils.close(in);
         }
         channel.sendResponse(TransportResponse.Empty.INSTANCE);
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
index 4a6006c1a04..6658913cce6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
@@ -23,13 +23,12 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.ElasticsearchException;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -44,10 +43,14 @@ import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.ConnectionProfile;
+import org.elasticsearch.transport.NodeNotConnectedException;
 import org.elasticsearch.transport.RemoteTransportException;
+import org.elasticsearch.transport.Transport.Connection;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -60,8 +63,8 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
@@ -70,18 +73,17 @@ import java.util.Objects;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -116,22 +118,19 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
 
     private volatile PingContextProvider contextProvider;
 
-    private final AtomicInteger pingHandlerIdGenerator = new AtomicInteger();
+    private final AtomicInteger pingingRoundIdGenerator = new AtomicInteger();
 
-    // used to generate unique ids for nodes/address we temporarily connect to
-    private final AtomicInteger unicastNodeIdGenerator = new AtomicInteger();
-
-    // used as a node id prefix for nodes/address we temporarily connect to
+    // used as a node id prefix for configured unicast host nodes/address
     private static final String UNICAST_NODE_PREFIX = "#zen_unicast_";
 
-    private final Map<Integer, SendPingsHandler> receivedResponses = newConcurrentMap();
+    private final Map<Integer, PingingRound> activePingingRounds = newConcurrentMap();
 
     // a list of temporal responses a node will return for a request (holds responses from other nodes)
     private final Queue<PingResponse> temporalResponses = ConcurrentCollections.newQueue();
 
     private final UnicastHostsProvider hostsProvider;
 
-    private final ExecutorService unicastZenPingExecutorService;
+    protected final EsThreadPoolExecutor unicastZenPingExecutorService;
 
     private final TimeValue resolveTimeout;
 
@@ -146,15 +145,14 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
         this.hostsProvider = unicastHostsProvider;
 
         this.concurrentConnects = DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
-        final List<String> hosts = DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.get(settings);
-        if (hosts.isEmpty()) {
+        if (DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.exists(settings)) {
+            configuredHosts = DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING.get(settings);
+            // we only limit to 1 addresses, makes no sense to ping 100 ports
+            limitPortCounts = LIMIT_FOREIGN_PORTS_COUNT;
+        } else {
             // if unicast hosts are not specified, fill with simple defaults on the local machine
             configuredHosts = transportService.getLocalAddresses();
             limitPortCounts = LIMIT_LOCAL_PORTS_COUNT;
-        } else {
-            configuredHosts = hosts;
-            // we only limit to 1 addresses, makes no sense to ping 100 ports
-            limitPortCounts = LIMIT_FOREIGN_PORTS_COUNT;
         }
         resolveTimeout = DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT.get(settings);
         logger.debug(
@@ -164,7 +162,7 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
             resolveTimeout);
 
         transportService.registerRequestHandler(ACTION_NAME, UnicastPingRequest::new, ThreadPool.Names.SAME,
-                new UnicastPingRequestHandler());
+            new UnicastPingRequestHandler());
 
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[unicast_connect]");
         unicastZenPingExecutorService = EsExecutors.newScaling(
@@ -186,23 +184,23 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
      * @param hosts            the hosts to resolve
      * @param limitPortCounts  the number of ports to resolve (should be 1 for non-local transport)
      * @param transportService the transport service
-     * @param idGenerator      the generator to supply unique ids for each discovery node
+     * @param nodeId_prefix    a prefix to use for node ids
      * @param resolveTimeout   the timeout before returning from hostname lookups
      * @return a list of discovery nodes with resolved transport addresses
      */
-    public static List<DiscoveryNode> resolveDiscoveryNodes(
+    public static List<DiscoveryNode> resolveHostsLists(
         final ExecutorService executorService,
         final Logger logger,
         final List<String> hosts,
         final int limitPortCounts,
         final TransportService transportService,
-        final Supplier<String> idGenerator,
+        final String nodeId_prefix,
         final TimeValue resolveTimeout) throws InterruptedException {
         Objects.requireNonNull(executorService);
         Objects.requireNonNull(logger);
         Objects.requireNonNull(hosts);
         Objects.requireNonNull(transportService);
-        Objects.requireNonNull(idGenerator);
+        Objects.requireNonNull(nodeId_prefix);
         Objects.requireNonNull(resolveTimeout);
         if (resolveTimeout.nanos() < 0) {
             throw new IllegalArgumentException("resolve timeout must be non-negative but was [" + resolveTimeout + "]");
@@ -211,7 +209,7 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
         final List<Callable<TransportAddress[]>> callables =
             hosts
                 .stream()
-                .map(hn -> (Callable<TransportAddress[]>)() -> transportService.addressesFromString(hn, limitPortCounts))
+                .map(hn -> (Callable<TransportAddress[]>) () -> transportService.addressesFromString(hn, limitPortCounts))
                 .collect(Collectors.toList());
         final List<Future<TransportAddress[]>> futures =
             executorService.invokeAll(callables, resolveTimeout.nanos(), TimeUnit.NANOSECONDS);
@@ -226,11 +224,11 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
                 try {
                     final TransportAddress[] addresses = future.get();
                     logger.trace("resolved host [{}] to {}", hostname, addresses);
-                    for (final TransportAddress address : addresses) {
+                    for (int addressId = 0; addressId < addresses.length; addressId++) {
                         discoveryNodes.add(
                             new DiscoveryNode(
-                                idGenerator.get(),
-                                address,
+                                nodeId_prefix + hostname + "_" + addressId + "#",
+                                addresses[addressId],
                                 emptyMap(),
                                 emptySet(),
                                 Version.CURRENT.minimumCompatibilityVersion()));
@@ -249,8 +247,8 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
 
     @Override
     public void close() {
-        ThreadPool.terminate(unicastZenPingExecutorService, 0, TimeUnit.SECONDS);
-        Releasables.close(receivedResponses.values());
+        ThreadPool.terminate(unicastZenPingExecutorService, 10, TimeUnit.SECONDS);
+        Releasables.close(activePingingRounds.values());
         closed = true;
     }
 
@@ -266,106 +264,106 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
         temporalResponses.clear();
     }
 
-    // test only
-    Collection<PingResponse> pingAndWait(TimeValue duration) {
-        final AtomicReference<Collection<PingResponse>> response = new AtomicReference<>();
-        final CountDownLatch latch = new CountDownLatch(1);
-        ping(pings -> {
-            response.set(pings);
-            latch.countDown();
-        }, duration);
-        try {
-            latch.await();
-            return response.get();
-        } catch (InterruptedException e) {
-            return null;
-        }
-    }
-
     /**
-     * Sends three rounds of pings notifying the specified {@link PingListener} when pinging is complete. Pings are sent after resolving
+     * Sends three rounds of pings notifying the specified {@link Consumer} when pinging is complete. Pings are sent after resolving
      * configured unicast hosts to their IP address (subject to DNS caching within the JVM). A batch of pings is sent, then another batch
      * of pings is sent at half the specified {@link TimeValue}, and then another batch of pings is sent at the specified {@link TimeValue}.
      * The pings that are sent carry a timeout of 1.25 times the specified {@link TimeValue}. When pinging each node, a connection and
      * handshake is performed, with a connection timeout of the specified {@link TimeValue}.
      *
-     * @param listener the callback when pinging is complete
-     * @param duration the timeout for various components of the pings
+     * @param resultsConsumer the callback when pinging is complete
+     * @param duration        the timeout for various components of the pings
      */
     @Override
-    public void ping(final PingListener listener, final TimeValue duration) {
-        final List<DiscoveryNode> resolvedDiscoveryNodes;
+    public void ping(final Consumer<PingCollection> resultsConsumer, final TimeValue duration) {
+        ping(resultsConsumer, duration, duration);
+    }
+
+    /**
+     * a variant of {@link #ping(Consumer, TimeValue)}, but allows separating the scheduling duration
+     * from the duration used for request level time outs. This is useful for testing
+     */
+    protected void ping(final Consumer<PingCollection> resultsConsumer,
+                        final TimeValue scheduleDuration,
+                        final TimeValue requestDuration) {
+        final List<DiscoveryNode> seedNodes;
         try {
-            resolvedDiscoveryNodes = resolveDiscoveryNodes(
+            seedNodes = resolveHostsLists(
                 unicastZenPingExecutorService,
                 logger,
                 configuredHosts,
                 limitPortCounts,
                 transportService,
-                () -> UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "#",
+                UNICAST_NODE_PREFIX,
                 resolveTimeout);
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
-        final SendPingsHandler sendPingsHandler = new SendPingsHandler(pingHandlerIdGenerator.incrementAndGet());
-        try {
-            receivedResponses.put(sendPingsHandler.id(), sendPingsHandler);
-            try {
-                sendPings(duration, null, sendPingsHandler, resolvedDiscoveryNodes);
-            } catch (RejectedExecutionException e) {
-                logger.debug("Ping execution rejected", e);
-                // The RejectedExecutionException can come from the fact unicastZenPingExecutorService is at its max down in sendPings
-                // But don't bail here, we can retry later on after the send ping has been scheduled.
+        seedNodes.addAll(hostsProvider.buildDynamicNodes());
+        final DiscoveryNodes nodes = contextProvider.nodes();
+        // add all possible master nodes that were active in the last known cluster configuration
+        for (ObjectCursor<DiscoveryNode> masterNode : nodes.getMasterNodes().values()) {
+            seedNodes.add(masterNode.value);
+        }
+
+        final ConnectionProfile connectionProfile =
+            ConnectionProfile.buildSingleChannelProfile(TransportRequestOptions.Type.REG, requestDuration, requestDuration);
+        final PingingRound pingingRound = new PingingRound(pingingRoundIdGenerator.incrementAndGet(), seedNodes, resultsConsumer,
+            nodes.getLocalNode(), connectionProfile);
+        activePingingRounds.put(pingingRound.id(), pingingRound);
+        final AbstractRunnable pingSender = new AbstractRunnable() {
+            @Override
+            public void onFailure(Exception e) {
+                if (e instanceof AlreadyClosedException == false) {
+                    logger.warn("unexpected error while pinging", e);
+                }
             }
 
-            threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
-                @Override
-                protected void doRun() {
-                    sendPings(duration, null, sendPingsHandler, resolvedDiscoveryNodes);
-                    threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
-                        @Override
-                        protected void doRun() throws Exception {
-                            sendPings(duration, TimeValue.timeValueMillis(duration.millis() / 2), sendPingsHandler, resolvedDiscoveryNodes);
-                            sendPingsHandler.close();
-                            listener.onPing(sendPingsHandler.pingCollection().toList());
-                            for (DiscoveryNode node : sendPingsHandler.nodeToDisconnect) {
-                                logger.trace("[{}] disconnecting from {}", sendPingsHandler.id(), node);
-                                transportService.disconnectFromNode(node);
-                            }
-                        }
+            @Override
+            protected void doRun() throws Exception {
+                sendPings(requestDuration, pingingRound);
+            }
+        };
+        threadPool.generic().execute(pingSender);
+        threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3), ThreadPool.Names.GENERIC, pingSender);
+        threadPool.schedule(TimeValue.timeValueMillis(scheduleDuration.millis() / 3 * 2), ThreadPool.Names.GENERIC, pingSender);
+        threadPool.schedule(scheduleDuration, ThreadPool.Names.GENERIC, new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                finishPingingRound(pingingRound);
+            }
 
-                        @Override
-                        public void onFailure(Exception e) {
-                            logger.debug("Ping execution failed", e);
-                            sendPingsHandler.close();
-                        }
-                    });
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    logger.debug("Ping execution failed", e);
-                    sendPingsHandler.close();
-                }
-            });
-        } catch (EsRejectedExecutionException ex) { // TODO: remove this once ScheduledExecutor has support for AbstractRunnable
-            sendPingsHandler.close();
-            // we are shutting down
-        } catch (Exception e) {
-            sendPingsHandler.close();
-            throw new ElasticsearchException("Ping execution failed", e);
-        }
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("unexpected error while finishing pinging round", e);
+            }
+        });
     }
 
-    class SendPingsHandler implements Releasable {
+    // for testing
+    protected void finishPingingRound(PingingRound pingingRound) {
+        pingingRound.close();
+    }
+
+    protected class PingingRound implements Releasable {
         private final int id;
-        private final Set<DiscoveryNode> nodeToDisconnect = ConcurrentCollections.newConcurrentSet();
+        private final Map<TransportAddress, Connection> tempConnections = new HashMap<>();
+        private final KeyedLock<TransportAddress> connectionLock = new KeyedLock<>(true);
         private final PingCollection pingCollection;
+        private final List<DiscoveryNode> seedNodes;
+        private final Consumer<PingCollection> pingListener;
+        private final DiscoveryNode localNode;
+        private final ConnectionProfile connectionProfile;
 
         private AtomicBoolean closed = new AtomicBoolean(false);
 
-        SendPingsHandler(int id) {
+        PingingRound(int id, List<DiscoveryNode> seedNodes, Consumer<PingCollection> resultsConsumer, DiscoveryNode localNode,
+                     ConnectionProfile connectionProfile) {
             this.id = id;
+            this.seedNodes = Collections.unmodifiableList(new ArrayList<>(seedNodes));
+            this.pingListener = resultsConsumer;
+            this.localNode = localNode;
+            this.connectionProfile = connectionProfile;
             this.pingCollection = new PingCollection();
         }
 
@@ -377,154 +375,174 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
             return this.closed.get();
         }
 
-        public PingCollection pingCollection() {
-            return pingCollection;
+        public List<DiscoveryNode> getSeedNodes() {
+            ensureOpen();
+            return seedNodes;
+        }
+
+        public Connection getOrConnect(DiscoveryNode node) throws IOException {
+            Connection result;
+            try (Releasable ignore = connectionLock.acquire(node.getAddress())) {
+                result = tempConnections.get(node.getAddress());
+                if (result == null) {
+                    ensureOpen();
+                    boolean success = false;
+                    logger.trace("[{}] opening connection to [{}]", id(), node);
+                    result = transportService.openConnection(node, connectionProfile);
+                    try {
+                        transportService.handshake(result, connectionProfile.getHandshakeTimeout().millis());
+                        synchronized (this) {
+                            // acquire lock and check if closed, to prevent leaving an open connection after closing
+                            ensureOpen();
+                            Connection existing = tempConnections.put(node.getAddress(), result);
+                            assert existing == null;
+                            success = true;
+                        }
+                    } finally {
+                        if (success == false) {
+                            logger.trace("[{}] closing connection to [{}] due to failure", id(), node);
+                            IOUtils.closeWhileHandlingException(result);
+                        }
+                    }
+                }
+            }
+            return result;
+        }
+
+        private void ensureOpen() {
+            if (isClosed()) {
+                throw new AlreadyClosedException("pinging round [" + id + "] is finished");
+            }
+        }
+
+        public void addPingResponseToCollection(PingResponse pingResponse) {
+            if (localNode.equals(pingResponse.node()) == false) {
+                pingCollection.addPing(pingResponse);
+            }
         }
 
         @Override
         public void close() {
-            if (closed.compareAndSet(false, true)) {
-                receivedResponses.remove(id);
+            List<Connection> toClose = null;
+            synchronized (this) {
+                if (closed.compareAndSet(false, true)) {
+                    activePingingRounds.remove(id);
+                    toClose = new ArrayList<>(tempConnections.values());
+                    tempConnections.clear();
+                }
             }
+            if (toClose != null) {
+                // we actually closed
+                try {
+                    pingListener.accept(pingCollection);
+                } finally {
+                    IOUtils.closeWhileHandlingException(toClose);
+                }
+            }
+        }
+
+        public ConnectionProfile getConnectionProfile() {
+            return connectionProfile;
         }
     }
 
 
-    void sendPings(
-        final TimeValue timeout,
-        @Nullable TimeValue waitTime,
-        final SendPingsHandler sendPingsHandler,
-        final List<DiscoveryNode> resolvedDiscoveryNodes) {
+    protected void sendPings(final TimeValue timeout, final PingingRound pingingRound) {
         final UnicastPingRequest pingRequest = new UnicastPingRequest();
-        pingRequest.id = sendPingsHandler.id();
+        pingRequest.id = pingingRound.id();
         pingRequest.timeout = timeout;
         DiscoveryNodes discoNodes = contextProvider.nodes();
 
         pingRequest.pingResponse = createPingResponse(discoNodes);
 
-        HashSet<DiscoveryNode> nodesToPingSet = new HashSet<>();
-        for (PingResponse temporalResponse : temporalResponses) {
-            // Only send pings to nodes that have the same cluster name.
-            if (clusterName.equals(temporalResponse.clusterName())) {
-                nodesToPingSet.add(temporalResponse.node());
-            }
-        }
-        nodesToPingSet.addAll(hostsProvider.buildDynamicNodes());
+        Set<DiscoveryNode> nodesFromResponses = temporalResponses.stream().map(pingResponse -> {
+            assert clusterName.equals(pingResponse.clusterName()) :
+                "got a ping request from a different cluster. expected " + clusterName + " got " + pingResponse.clusterName();
+            return pingResponse.node();
+        }).collect(Collectors.toSet());
 
-        // add all possible master nodes that were active in the last known cluster configuration
-        for (ObjectCursor<DiscoveryNode> masterNode : discoNodes.getMasterNodes().values()) {
-            nodesToPingSet.add(masterNode.value);
-        }
+        // dedup by address
+        final Map<TransportAddress, DiscoveryNode> uniqueNodesByAddress =
+            Stream.concat(pingingRound.getSeedNodes().stream(), nodesFromResponses.stream())
+                .collect(Collectors.toMap(DiscoveryNode::getAddress, Function.identity(), (n1, n2) -> n1));
 
-        // sort the nodes by likelihood of being an active master
-        List<DiscoveryNode> sortedNodesToPing = ElectMasterService.sortByMasterLikelihood(nodesToPingSet);
 
-        // add the configured hosts first
-        final List<DiscoveryNode> nodesToPing = new ArrayList<>(resolvedDiscoveryNodes.size() + sortedNodesToPing.size());
-        nodesToPing.addAll(resolvedDiscoveryNodes);
-        nodesToPing.addAll(sortedNodesToPing);
-
-        final CountDownLatch latch = new CountDownLatch(nodesToPing.size());
-        for (final DiscoveryNode node : nodesToPing) {
-            // make sure we are connected
-            final boolean nodeFoundByAddress;
-            DiscoveryNode nodeToSend = discoNodes.findByAddress(node.getAddress());
-            if (nodeToSend != null) {
-                nodeFoundByAddress = true;
-            } else {
-                nodeToSend = node;
-                nodeFoundByAddress = false;
-            }
-
-            if (!transportService.nodeConnected(nodeToSend)) {
-                if (sendPingsHandler.isClosed()) {
-                    return;
+        // resolve what we can via the latest cluster state
+        final Set<DiscoveryNode> nodesToPing = uniqueNodesByAddress.values().stream()
+            .map(node -> {
+                DiscoveryNode foundNode = discoNodes.findByAddress(node.getAddress());
+                if (foundNode == null) {
+                    return node;
+                } else {
+                    return foundNode;
                 }
-                // if we find on the disco nodes a matching node by address, we are going to restore the connection
-                // anyhow down the line if its not connected...
-                // if we can't resolve the node, we don't know and we have to clean up after pinging. We do have
-                // to make sure we don't disconnect a true node which was temporarily removed from the DiscoveryNodes
-                // but will be added again during the pinging. We therefore create a new temporary node
-                if (!nodeFoundByAddress) {
-                    if (!nodeToSend.getId().startsWith(UNICAST_NODE_PREFIX)) {
-                        DiscoveryNode tempNode = new DiscoveryNode("",
-                            UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "_" + nodeToSend.getId() + "#",
-                            UUIDs.randomBase64UUID(), nodeToSend.getHostName(), nodeToSend.getHostAddress(), nodeToSend.getAddress(),
-                            nodeToSend.getAttributes(), nodeToSend.getRoles(), nodeToSend.getVersion());
+            }).collect(Collectors.toSet());
 
-                        logger.trace("replacing {} with temp node {}", nodeToSend, tempNode);
-                        nodeToSend = tempNode;
-                    }
-                    sendPingsHandler.nodeToDisconnect.add(nodeToSend);
-                }
-                // fork the connection to another thread
-                final DiscoveryNode finalNodeToSend = nodeToSend;
-                unicastZenPingExecutorService.execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        if (sendPingsHandler.isClosed()) {
-                            return;
-                        }
-                        boolean success = false;
-                        try {
-                            // connect to the node, see if we manage to do it, if not, bail
-                            if (!nodeFoundByAddress) {
-                                logger.trace("[{}] connecting (light) to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNodeAndHandshake(finalNodeToSend, timeout.getMillis());
-                            } else {
-                                logger.trace("[{}] connecting to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNode(finalNodeToSend);
-                            }
-                            logger.trace("[{}] connected to {}", sendPingsHandler.id(), node);
-                            if (receivedResponses.containsKey(sendPingsHandler.id())) {
-                                // we are connected and still in progress, send the ping request
-                                sendPingRequestToNode(sendPingsHandler.id(), timeout, pingRequest, latch, node, finalNodeToSend);
-                            } else {
-                                // connect took too long, just log it and bail
-                                latch.countDown();
-                                logger.trace("[{}] connect to {} was too long outside of ping window, bailing",
-                                        sendPingsHandler.id(), node);
-                            }
-                            success = true;
-                        } catch (ConnectTransportException e) {
-                            // can't connect to the node - this is a more common path!
-                            logger.trace(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] failed to connect to {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } catch (RemoteTransportException e) {
-                            // something went wrong on the other side
-                            logger.debug(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] received a remote error as a response to ping {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } catch (Exception e) {
-                            logger.warn(
-                                (Supplier<?>) () -> new ParameterizedMessage(
-                                    "[{}] failed send ping to {}", sendPingsHandler.id(), finalNodeToSend), e);
-                        } finally {
-                            if (!success) {
-                                latch.countDown();
-                            }
-                        }
-                    }
-                });
-            } else {
-                sendPingRequestToNode(sendPingsHandler.id(), timeout, pingRequest, latch, node, nodeToSend);
-            }
-        }
-        if (waitTime != null) {
-            try {
-                latch.await(waitTime.millis(), TimeUnit.MILLISECONDS);
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
+        nodesToPing.forEach(node -> sendPingRequestToNode(node, timeout, pingingRound, pingRequest));
     }
 
-    private void sendPingRequestToNode(final int id, final TimeValue timeout, final UnicastPingRequest pingRequest,
-                                       final CountDownLatch latch, final DiscoveryNode node, final DiscoveryNode nodeToSend) {
-        logger.trace("[{}] sending to {}", id, nodeToSend);
-        transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.builder()
-                .withTimeout((long) (timeout.millis() * 1.25)).build(), new TransportResponseHandler<UnicastPingResponse>() {
+    private void sendPingRequestToNode(final DiscoveryNode node, TimeValue timeout, final PingingRound pingingRound,
+                                       final UnicastPingRequest pingRequest) {
+        submitToExecutor(new AbstractRunnable() {
+            @Override
+            protected void doRun() throws Exception {
+                Connection connection = null;
+                if (transportService.nodeConnected(node)) {
+                    try {
+                        // concurrency can still cause disconnects
+                        connection = transportService.getConnection(node);
+                    } catch (NodeNotConnectedException e) {
+                        logger.trace("[{}] node [{}] just disconnected, will create a temp connection", pingingRound.id(), node);
+                    }
+                }
+
+                if (connection == null) {
+                    connection = pingingRound.getOrConnect(node);
+                }
+
+                logger.trace("[{}] sending to {}", pingingRound.id(), node);
+                transportService.sendRequest(connection, ACTION_NAME, pingRequest,
+                    TransportRequestOptions.builder().withTimeout((long) (timeout.millis() * 1.25)).build(),
+                    getPingResponseHandler(pingingRound, node));
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (e instanceof ConnectTransportException || e instanceof AlreadyClosedException) {
+                    // can't connect to the node - this is more common path!
+                    logger.trace(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] failed to ping {}", pingingRound.id(), node), e);
+                } else if (e instanceof RemoteTransportException) {
+                    // something went wrong on the other side
+                    logger.debug(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] received a remote error as a response to ping {}", pingingRound.id(), node), e);
+                } else {
+                    logger.warn(
+                        (Supplier<?>) () -> new ParameterizedMessage(
+                            "[{}] failed send ping to {}", pingingRound.id(), node), e);
+                }
+            }
+
+            @Override
+            public void onRejection(Exception e) {
+                // The RejectedExecutionException can come from the fact unicastZenPingExecutorService is at its max down in sendPings
+                // But don't bail here, we can retry later on after the send ping has been scheduled.
+                logger.debug("Ping execution rejected", e);
+            }
+        });
+    }
+
+    // for testing
+    protected void submitToExecutor(AbstractRunnable abstractRunnable) {
+        unicastZenPingExecutorService.execute(abstractRunnable);
+    }
+
+    // for testing
+    protected TransportResponseHandler<UnicastPingResponse> getPingResponseHandler(final PingingRound pingingRound,
+                                                                                   final DiscoveryNode node) {
+        return new TransportResponseHandler<UnicastPingResponse>() {
 
             @Override
             public UnicastPingResponse newInstance() {
@@ -538,50 +556,36 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
 
             @Override
             public void handleResponse(UnicastPingResponse response) {
-                logger.trace("[{}] received response from {}: {}", id, nodeToSend, Arrays.toString(response.pingResponses));
-                try {
-                    DiscoveryNodes discoveryNodes = contextProvider.nodes();
-                    for (PingResponse pingResponse : response.pingResponses) {
-                        if (pingResponse.node().equals(discoveryNodes.getLocalNode())) {
-                            // that's us, ignore
-                            continue;
-                        }
-                        SendPingsHandler sendPingsHandler = receivedResponses.get(response.id);
-                        if (sendPingsHandler == null) {
-                            if (!closed) {
-                                // Only log when we're not closing the node. Having no send ping handler is then expected
-                                logger.warn("received ping response {} with no matching handler id [{}]", pingResponse, response.id);
-                            }
-                        } else {
-                            sendPingsHandler.pingCollection().addPing(pingResponse);
-                        }
+                logger.trace("[{}] received response from {}: {}", pingingRound.id(), node, Arrays.toString(response.pingResponses));
+                if (pingingRound.isClosed()) {
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("[{}] skipping received response from {}. already closed", pingingRound.id(), node);
                     }
-                } finally {
-                    latch.countDown();
+                } else {
+                    Stream.of(response.pingResponses).forEach(pingingRound::addPingResponseToCollection);
                 }
             }
 
             @Override
             public void handleException(TransportException exp) {
-                latch.countDown();
-                if (exp instanceof ConnectTransportException) {
+                if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
                     // ok, not connected...
-                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", nodeToSend), exp);
-                } else {
+                    logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", node), exp);
+                } else if (closed == false) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send ping to [{}]", node), exp);
                 }
             }
-        });
+        };
     }
 
     private UnicastPingResponse handlePingRequest(final UnicastPingRequest request) {
+        assert clusterName.equals(request.pingResponse.clusterName()) :
+            "got a ping request from a different cluster. expected " + clusterName + " got " + request.pingResponse.clusterName();
         temporalResponses.add(request.pingResponse);
-        threadPool.schedule(TimeValue.timeValueMillis(request.timeout.millis() * 2), ThreadPool.Names.SAME, new Runnable() {
-            @Override
-            public void run() {
-                temporalResponses.remove(request.pingResponse);
-            }
-        });
+        // add to any ongoing pinging
+        activePingingRounds.values().forEach(p -> p.addPingResponseToCollection(request.pingResponse));
+        threadPool.schedule(TimeValue.timeValueMillis(request.timeout.millis() * 2), ThreadPool.Names.SAME,
+            () -> temporalResponses.remove(request.pingResponse));
 
         List<PingResponse> pingResponses = CollectionUtils.iterableAsArrayList(temporalResponses);
         pingResponses.add(createPingResponse(contextProvider.nodes()));
@@ -601,11 +605,11 @@ public class UnicastZenPing extends AbstractComponent implements ZenPing {
                 channel.sendResponse(handlePingRequest(request));
             } else {
                 throw new IllegalStateException(
-                        String.format(
-                                Locale.ROOT,
-                                "mismatched cluster names; request: [%s], local: [%s]",
-                                request.pingResponse.clusterName().value(),
-                                clusterName.value()));
+                    String.format(
+                        Locale.ROOT,
+                        "mismatched cluster names; request: [%s], local: [%s]",
+                        request.pingResponse.clusterName().value(),
+                        clusterName.value()));
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 16ca16d4a90..effc92a0c67 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -27,13 +27,12 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
-import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.NotMasterException;
-import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -44,6 +43,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasables;
@@ -67,11 +67,11 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
-import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
@@ -103,6 +103,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
 
     private final TransportService transportService;
+    private final NamedWriteableRegistry namedWriteableRegistry;
     private final ClusterService clusterService;
     private AllocationService allocationService;
     private final ClusterName clusterName;
@@ -139,11 +140,13 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     private volatile NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor;
 
     public ZenDiscovery(Settings settings, ThreadPool threadPool, TransportService transportService,
+                        NamedWriteableRegistry namedWriteableRegistry,
                         ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         super(settings);
         this.clusterService = clusterService;
         this.clusterName = clusterService.getClusterName();
         this.transportService = transportService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
         this.discoverySettings = new DiscoverySettings(settings, clusterService.getClusterSettings());
         this.zenPing = newZenPing(settings, threadPool, transportService, hostsProvider);
         this.electMaster = new ElectMasterService(settings);
@@ -180,6 +183,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 new PublishClusterStateAction(
                         settings,
                         transportService,
+                        namedWriteableRegistry,
                         clusterService::state,
                         new NewPendingClusterStateListener(),
                         discoverySettings,
@@ -207,25 +211,20 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
         zenPing.start(this);
-        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, discoverySettings, settings);
+        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, settings);
         this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, electMaster, this::submitRejoin, logger);
     }
 
     @Override
     public void startInitialJoin() {
         // start the join thread from a cluster state update. See {@link JoinThreadControl} for details.
-        clusterService.submitStateUpdateTask("initial_join", new ClusterStateUpdateTask() {
+        clusterService.submitStateUpdateTask("initial_join", new LocalClusterUpdateTask() {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 // do the join on a different thread, the DiscoveryService waits for 30s anyhow till it is discovered
                 joinThreadControl.startNewThreadIfNotRunning();
-                return currentState;
+                return unchanged();
             }
 
             @Override
@@ -352,7 +351,6 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         return joinThreadControl.joinThreadActive();
     }
 
-
     // used for testing
     public ClusterState[] pendingClusterStates() {
         return publishClusterState.pendingStatesQueue().pendingClusterStates();
@@ -408,18 +406,13 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
             // finalize join through the cluster state update thread
             final DiscoveryNode finalMasterNode = masterNode;
-            clusterService.submitStateUpdateTask("finalize_join (" + masterNode + ")", new ClusterStateUpdateTask() {
+            clusterService.submitStateUpdateTask("finalize_join (" + masterNode + ")", new LocalClusterUpdateTask() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                     if (!success) {
                         // failed to join. Try again...
                         joinThreadControl.markThreadAsDoneAndStartNew(currentThread);
-                        return currentState;
+                        return unchanged();
                     }
 
                     if (currentState.getNodes().getMasterNode() == null) {
@@ -427,7 +420,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                         // a valid master.
                         logger.debug("no master node is set, despite of join request completing. retrying pings.");
                         joinThreadControl.markThreadAsDoneAndStartNew(currentThread);
-                        return currentState;
+                        return unchanged();
                     }
 
                     if (!currentState.getNodes().getMasterNode().equals(finalMasterNode)) {
@@ -437,7 +430,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                     // Note: we do not have to start master fault detection here because it's set at {@link #processNextPendingClusterState }
                     // when the first cluster state arrives.
                     joinThreadControl.markThreadAsDone(currentThread);
-                    return currentState;
+                    return unchanged();
                 }
 
                 @Override
@@ -496,9 +489,9 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     }
 
     private void submitRejoin(String source) {
-        clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask(source, new LocalClusterUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 return rejoin(currentState, source);
             }
 
@@ -523,7 +516,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             private final DiscoveryNode node;
             private final String reason;
 
-            public Task(final DiscoveryNode node, final String reason) {
+            Task(final DiscoveryNode node, final String reason) {
                 this.node = node;
                 this.reason = reason;
             }
@@ -554,7 +547,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         }
 
         @Override
-        public BatchResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
+        public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
             final DiscoveryNodes.Builder remainingNodesBuilder = DiscoveryNodes.builder(currentState.nodes());
             boolean removed = false;
             for (final Task task : tasks) {
@@ -568,12 +561,12 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
             if (!removed) {
                 // no nodes to remove, keep the current cluster state
-                return BatchResult.<Task>builder().successes(tasks).build(currentState);
+                return ClusterTasksResult.<Task>builder().successes(tasks).build(currentState);
             }
 
             final ClusterState remainingNodesClusterState = remainingNodesClusterState(currentState, remainingNodesBuilder);
 
-            final BatchResult.Builder<Task> resultBuilder = BatchResult.<Task>builder().successes(tasks);
+            final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
             if (!electMasterService.hasEnoughMasterNodes(remainingNodesClusterState.nodes())) {
                 rejoin.accept("not enough master nodes");
                 return resultBuilder.build(currentState);
@@ -645,14 +638,14 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             // We only set the new value. If the master doesn't see enough nodes it will revoke it's mastership.
             return;
         }
-        clusterService.submitStateUpdateTask("zen-disco-mini-master-nodes-changed", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("zen-disco-min-master-nodes-changed", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 // check if we have enough master nodes, if not, we need to move into joining the cluster again
                 if (!electMaster.hasEnoughMasterNodes(currentState.nodes())) {
                     return rejoin(currentState, "not enough master nodes on change of minimum_master_nodes from [" + prevMinimumMasterNode + "] to [" + minimumMasterNodes + "]");
                 }
-                return currentState;
+                return unchanged();
             }
 
 
@@ -685,18 +678,13 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
         logger.info((Supplier<?>) () -> new ParameterizedMessage("master_left [{}], reason [{}]", masterNode, reason), cause);
 
-        clusterService.submitStateUpdateTask("master_failed (" + masterNode + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("master_failed (" + masterNode + ")", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 if (!masterNode.equals(currentState.nodes().getMasterNode())) {
                     // master got switched on us, no need to send anything
-                    return currentState;
+                    return unchanged();
                 }
 
                 // flush any pending cluster states from old master, so it will not be set as master again
@@ -710,29 +698,20 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure during [{}]", source), e);
             }
 
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
-
         });
     }
 
     void processNextPendingClusterState(String reason) {
-        clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", new ClusterStateUpdateTask(Priority.URGENT) {
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
+        clusterService.submitStateUpdateTask("zen-disco-receive(from master [" + reason + "])", new LocalClusterUpdateTask(Priority.URGENT) {
             ClusterState newClusterState = null;
 
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                 newClusterState = publishClusterState.pendingStatesQueue().getNextClusterStateToProcess();
 
                 // all pending states have been processed
                 if (newClusterState == null) {
-                    return currentState;
+                    return unchanged();
                 }
 
                 assert newClusterState.nodes().getMasterNode() != null : "received a cluster state without a master";
@@ -743,7 +722,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 if (shouldIgnoreOrRejectNewClusterState(logger, currentState, newClusterState)) {
-                    return currentState;
+                    return unchanged();
                 }
 
                 // check to see that we monitor the correct master of the cluster
@@ -754,7 +733,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 if (currentState.blocks().hasGlobalBlock(discoverySettings.getNoMasterBlock())) {
                     // its a fresh update from the master as we transition from a start of not having a master to having one
                     logger.debug("got first state from fresh master [{}]", newClusterState.nodes().getMasterNodeId());
-                    return newClusterState;
+                    return newState(newClusterState);
                 }
 
 
@@ -784,7 +763,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                     builder.metaData(metaDataBuilder);
                 }
 
-                return builder.build();
+                return newState(builder.build());
             }
 
             @Override
@@ -962,7 +941,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         return pingResponses;
     }
 
-    protected ClusterState rejoin(ClusterState clusterState, String reason) {
+    protected ClusterStateTaskExecutor.ClusterTasksResult<LocalClusterUpdateTask> rejoin(ClusterState clusterState, String reason) {
 
         // *** called from within an cluster state update task *** //
         assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME);
@@ -971,29 +950,17 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         nodesFD.stop();
         masterFD.stop(reason);
 
-
-        ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(clusterState.blocks())
-                .addGlobalBlock(discoverySettings.getNoMasterBlock())
-                .build();
-
-        // clean the nodes, we are now not connected to anybody, since we try and reform the cluster
-        DiscoveryNodes discoveryNodes = new DiscoveryNodes.Builder(clusterState.nodes()).masterNodeId(null).build();
-
         // TODO: do we want to force a new thread if we actively removed the master? this is to give a full pinging cycle
         // before a decision is made.
         joinThreadControl.startNewThreadIfNotRunning();
-
-        return ClusterState.builder(clusterState)
-                .blocks(clusterBlocks)
-                .nodes(discoveryNodes)
-                .build();
+        return LocalClusterUpdateTask.noMaster();
     }
 
     private boolean localNodeMaster() {
         return nodes().isLocalNodeElectedMaster();
     }
 
-    private ClusterState handleAnotherMaster(ClusterState localClusterState, final DiscoveryNode otherMaster, long otherClusterStateVersion, String reason) {
+    private ClusterStateTaskExecutor.ClusterTasksResult handleAnotherMaster(ClusterState localClusterState, final DiscoveryNode otherMaster, long otherClusterStateVersion, String reason) {
         assert localClusterState.nodes().isLocalNodeElectedMaster() : "handleAnotherMaster called but current node is not a master";
         assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME) : "not called from the cluster state update thread";
 
@@ -1016,29 +983,27 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to send rejoin request to [{}]", otherMaster), e);
             }
-            return localClusterState;
+            return LocalClusterUpdateTask.unchanged();
         }
     }
 
     private ZenPing.PingCollection pingAndWait(TimeValue timeout) {
-        final ZenPing.PingCollection response = new ZenPing.PingCollection();
-        final CountDownLatch latch = new CountDownLatch(1);
+        final CompletableFuture<ZenPing.PingCollection> response = new CompletableFuture<>();
         try {
-            zenPing.ping(pings -> {
-                response.addPings(pings);
-                latch.countDown();
-            }, timeout);
+            zenPing.ping(response::complete, timeout);
         } catch (Exception ex) {
-            logger.warn("Ping execution failed", ex);
-            latch.countDown();
+            // logged later
+            response.completeExceptionally(ex);
         }
 
         try {
-            latch.await();
-            return response;
+            return response.get();
         } catch (InterruptedException e) {
             logger.trace("pingAndWait interrupted");
-            return response;
+            return new ZenPing.PingCollection();
+        } catch (ExecutionException e) {
+            logger.warn("Ping execution failed", e);
+            return new ZenPing.PingCollection();
         }
     }
 
@@ -1085,12 +1050,16 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 return;
             }
             logger.debug("got a ping from another master {}. resolving who should rejoin. current ping count: [{}]", pingRequest.masterNode(), pingsWhileMaster.get());
-            clusterService.submitStateUpdateTask("ping from another master", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            clusterService.submitStateUpdateTask("ping from another master", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
                 @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
-                    pingsWhileMaster.set(0);
-                    return handleAnotherMaster(currentState, pingRequest.masterNode(), pingRequest.clusterStateVersion(), "node fd ping");
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
+                    if (currentState.nodes().isLocalNodeElectedMaster()) {
+                        pingsWhileMaster.set(0);
+                        return handleAnotherMaster(currentState, pingRequest.masterNode(), pingRequest.clusterStateVersion(), "node fd ping");
+                    } else {
+                        return unchanged();
+                    }
                 }
 
                 @Override
@@ -1136,15 +1105,10 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     class RejoinClusterRequestHandler implements TransportRequestHandler<RejoinClusterRequest> {
         @Override
         public void messageReceived(final RejoinClusterRequest request, final TransportChannel channel) throws Exception {
-            clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            clusterService.submitStateUpdateTask("received a request to rejoin the cluster from [" + request.fromNodeId + "]", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) {
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) {
                     try {
                         channel.sendResponse(TransportResponse.Empty.INSTANCE);
                     } catch (Exception e) {
@@ -1172,7 +1136,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         private final AtomicBoolean running = new AtomicBoolean(false);
         private final AtomicReference<Thread> currentJoinThread = new AtomicReference<>();
 
-        public JoinThreadControl(ThreadPool threadPool) {
+        JoinThreadControl(ThreadPool threadPool) {
             this.threadPool = threadPool;
         }
 
@@ -1188,7 +1152,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         }
 
         /** cleans any running joining thread and calls {@link #rejoin} */
-        public ClusterState stopRunningThreadAndRejoin(ClusterState clusterState, String reason) {
+        public ClusterStateTaskExecutor.ClusterTasksResult<LocalClusterUpdateTask> stopRunningThreadAndRejoin(ClusterState clusterState, String reason) {
             ClusterService.assertClusterStateThread();
             currentJoinThread.set(null);
             return rejoin(clusterState, reason);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java
index 75ea701dc99..622c4649db2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenPing.java
@@ -30,11 +30,11 @@ import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Consumer;
 
 import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 
@@ -42,17 +42,7 @@ public interface ZenPing extends Releasable {
 
     void start(PingContextProvider contextProvider);
 
-    void ping(PingListener listener, TimeValue timeout);
-
-    interface PingListener {
-
-        /**
-         * called when pinging is done.
-         *
-         * @param pings ping result *must
-         */
-        void onPing(Collection<PingResponse> pings);
-    }
+    void ping(Consumer<PingCollection> resultsConsumer, TimeValue timeout);
 
     class PingResponse implements Streamable {
 
@@ -191,13 +181,6 @@ public interface ZenPing extends Releasable {
             return false;
         }
 
-        /** adds multiple pings if newer than previous pings from the same node */
-        public synchronized void addPings(Iterable<PingResponse> pings) {
-            for (PingResponse ping : pings) {
-                addPing(ping);
-            }
-        }
-
         /** serialize current pings to a list. It is guaranteed that the list contains one ping response per node */
         public synchronized List<PingResponse> toList() {
             return new ArrayList<>(pings.values());
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 401f3f12f4b..e531408b57a 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -38,13 +38,13 @@ import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -372,7 +372,7 @@ public final class NodeEnvironment  implements Closeable {
     private static NodeMetaData loadOrCreateNodeMetaData(Settings settings, Logger logger,
                                                          NodePath... nodePaths) throws IOException {
         final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
-        NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, paths);
+        NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, paths);
         if (metaData == null) {
             metaData = new NodeMetaData(generateNodeId(settings));
         }
diff --git a/core/src/main/java/org/elasticsearch/env/NodeMetaData.java b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
index 60625b1852d..38a4fce9cdc 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
@@ -20,8 +20,6 @@
 package org.elasticsearch.env;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -70,8 +68,7 @@ public final class NodeMetaData {
         return "node_id [" + nodeId + "]";
     }
 
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("node_meta_data",
-        Builder::new);
+    private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("node_meta_data", Builder::new);
 
     static {
         PARSER.declareString(Builder::setNodeId, new ParseField(NODE_ID_KEY));
@@ -110,7 +107,7 @@ public final class NodeMetaData {
 
         @Override
         public NodeMetaData fromXContent(XContentParser parser) throws IOException {
-            return PARSER.apply(parser, () -> ParseFieldMatcher.STRICT).build();
+            return PARSER.apply(parser, null).build();
         }
     };
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index 37277586bf7..5a38a4b2b9e 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -343,7 +343,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
         private boolean valueSet;
         private Throwable failure;
 
-        public NodeEntry(String nodeId) {
+        NodeEntry(String nodeId) {
             this.nodeId = nodeId;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 605fe20a33a..69f7af6eef5 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -57,24 +58,6 @@ public class GatewayAllocator extends AbstractComponent {
         this.replicaShardAllocator = new InternalReplicaShardAllocator(settings, storeAction);
     }
 
-    /**
-     * Returns true if the given shard has an async fetch pending
-     */
-    public boolean hasFetchPending(ShardId shardId, boolean primary) {
-        if (primary) {
-            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch = asyncFetchStarted.get(shardId);
-            if (fetch != null) {
-                return fetch.getNumberOfInFlightFetches() > 0;
-            }
-        } else {
-            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch = asyncFetchStore.get(shardId);
-            if (fetch != null) {
-                return fetch.getNumberOfInFlightFetches() > 0;
-            }
-        }
-        return false;
-    }
-
     public void setReallocation(final ClusterService clusterService, final RoutingService routingService) {
         this.routingService = routingService;
         clusterService.addStateApplier(event -> {
@@ -137,9 +120,21 @@ public class GatewayAllocator extends AbstractComponent {
         replicaShardAllocator.allocateUnassigned(allocation);
     }
 
+    /**
+     * Computes and returns the design for allocating a single unassigned shard.  If called on an assigned shard,
+     * {@link AllocateUnassignedDecision#NOT_TAKEN} is returned.
+     */
+    public AllocateUnassignedDecision decideUnassignedShardAllocation(ShardRouting unassignedShard, RoutingAllocation routingAllocation) {
+        if (unassignedShard.primary()) {
+            return primaryShardAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        } else {
+            return replicaShardAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        }
+    }
+
     class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
 
-        public InternalAsyncFetch(Logger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
+        InternalAsyncFetch(Logger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
             super(logger, type, shardId, action);
         }
 
@@ -154,18 +149,15 @@ public class GatewayAllocator extends AbstractComponent {
 
         private final TransportNodesListGatewayStartedShards startedAction;
 
-        public InternalPrimaryShardAllocator(Settings settings, TransportNodesListGatewayStartedShards startedAction) {
+        InternalPrimaryShardAllocator(Settings settings, TransportNodesListGatewayStartedShards startedAction) {
             super(settings);
             this.startedAction = startedAction;
         }
 
         @Override
         protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch = asyncFetchStarted.get(shard.shardId());
-            if (fetch == null) {
-                fetch = new InternalAsyncFetch<>(logger, "shard_started", shard.shardId(), startedAction);
-                asyncFetchStarted.put(shard.shardId(), fetch);
-            }
+            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch =
+                asyncFetchStarted.computeIfAbsent(shard.shardId(), shardId -> new InternalAsyncFetch<>(logger, "shard_started", shardId, startedAction));
             AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> shardState =
                     fetch.fetchData(allocation.nodes(), allocation.getIgnoreNodes(shard.shardId()));
 
@@ -180,18 +172,15 @@ public class GatewayAllocator extends AbstractComponent {
 
         private final TransportNodesListShardStoreMetaData storeAction;
 
-        public InternalReplicaShardAllocator(Settings settings, TransportNodesListShardStoreMetaData storeAction) {
+        InternalReplicaShardAllocator(Settings settings, TransportNodesListShardStoreMetaData storeAction) {
             super(settings);
             this.storeAction = storeAction;
         }
 
         @Override
         protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetchData(ShardRouting shard, RoutingAllocation allocation) {
-            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch = asyncFetchStore.get(shard.shardId());
-            if (fetch == null) {
-                fetch = new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction);
-                asyncFetchStore.put(shard.shardId(), fetch);
-            }
+            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch =
+                asyncFetchStore.computeIfAbsent(shard.shardId(), shardId -> new InternalAsyncFetch<>(logger, "shard_store", shard.shardId(), storeAction));
             AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> shardStores =
                     fetch.fetchData(allocation.nodes(), allocation.getIgnoreNodes(shard.shardId()));
             if (shardStores.hasData()) {
diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index b1891191500..5f75771e9e6 100644
--- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -223,7 +223,7 @@ public class LocalAllocateDangledIndices extends AbstractComponent {
             fromNode = new DiscoveryNode(in);
             indices = new IndexMetaData[in.readVInt()];
             for (int i = 0; i < indices.length; i++) {
-                indices[i] = IndexMetaData.Builder.readFrom(in);
+                indices[i] = IndexMetaData.readFrom(in);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
index 71c3190e2ee..fb48405b725 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
@@ -35,6 +35,7 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -181,9 +182,9 @@ public abstract class MetaDataStateFormat<T> {
      * Reads the state from a given file and compares the expected version against the actual version of
      * the state.
      */
-    public final T read(Path file) throws IOException {
+    public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException {
         try (Directory dir = newDirectory(file.getParent())) {
-            try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
+            try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
                  // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
                 CodecUtil.checksumEntireFile(indexInput);
                 final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
@@ -196,8 +197,8 @@ public abstract class MetaDataStateFormat<T> {
                 long filePointer = indexInput.getFilePointer();
                 long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
                 try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
-                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice,
-                        contentSize))) {
+                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry,
+                            new InputStreamIndexInput(slice, contentSize))) {
                         return fromXContent(parser);
                     }
                 }
@@ -260,7 +261,7 @@ public abstract class MetaDataStateFormat<T> {
      * @param dataLocations the data-locations to try.
      * @return the latest state or <code>null</code> if no state was found.
      */
-    public  T loadLatestState(Logger logger, Path... dataLocations) throws IOException {
+    public  T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws IOException {
         List<PathAndStateId> files = new ArrayList<>();
         long maxStateId = -1;
         boolean maxStateIdIsLegacy = true;
@@ -311,14 +312,14 @@ public abstract class MetaDataStateFormat<T> {
                         logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
                         continue;
                     }
-                    try (final XContentParser parser = XContentHelper.createParser(new BytesArray(data))) {
+                    try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, new BytesArray(data))) {
                         state = fromXContent(parser);
                     }
                     if (state == null) {
                         logger.debug("{}: no data for [{}], ignoring...", prefix, stateFile.toAbsolutePath());
                     }
                 } else {
-                    state = read(stateFile);
+                    state = read(namedXContentRegistry, stateFile);
                     logger.trace("state id [{}] read from [{}]", id, stateFile.getFileName());
                 }
                 return state;
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index 1673a653a6b..b900305ab55 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 
@@ -40,10 +41,12 @@ import java.util.function.Predicate;
 public class MetaStateService extends AbstractComponent {
 
     private final NodeEnvironment nodeEnv;
+    private final NamedXContentRegistry namedXContentRegistry;
 
-    public MetaStateService(Settings settings, NodeEnvironment nodeEnv) {
+    public MetaStateService(Settings settings, NodeEnvironment nodeEnv, NamedXContentRegistry namedXContentRegistry) {
         super(settings);
         this.nodeEnv = nodeEnv;
+        this.namedXContentRegistry = namedXContentRegistry;
     }
 
     /**
@@ -59,7 +62,8 @@ public class MetaStateService extends AbstractComponent {
             metaDataBuilder = MetaData.builder();
         }
         for (String indexFolderName : nodeEnv.availableIndexFolders()) {
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.resolveIndexFolder(indexFolderName));
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
+                nodeEnv.resolveIndexFolder(indexFolderName));
             if (indexMetaData != null) {
                 metaDataBuilder.put(indexMetaData, false);
             } else {
@@ -74,7 +78,7 @@ public class MetaStateService extends AbstractComponent {
      */
     @Nullable
     public IndexMetaData loadIndexState(Index index) throws IOException {
-        return IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+        return IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index));
     }
 
     /**
@@ -86,7 +90,7 @@ public class MetaStateService extends AbstractComponent {
             if (excludeIndexPathIdsPredicate.test(indexFolderName)) {
                 continue;
             }
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry,
                 nodeEnv.resolveIndexFolder(indexFolderName));
             if (indexMetaData != null) {
                 final String indexPathId = indexMetaData.getIndex().getUUID();
@@ -106,7 +110,7 @@ public class MetaStateService extends AbstractComponent {
      * Loads the global state, *without* index state, see {@link #loadFullState()} for that.
      */
     MetaData loadGlobalState() throws IOException {
-        return MetaData.FORMAT.loadLatestState(logger, nodeEnv.nodeDataPaths());
+        return MetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.nodeDataPaths());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
index 2200ed1b4f2..8e85081a3c1 100644
--- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
@@ -49,6 +49,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Function;
@@ -183,7 +184,8 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
                 // this shard will be picked up when the node joins and we do another allocation reroute
                 logger.debug("[{}][{}]: not allocating, number_of_allocated_shards_found [{}]",
                              unassignedShard.index(), unassignedShard.id(), nodeShardsResult.allocationsFound);
-                return AllocateUnassignedDecision.no(AllocationStatus.NO_VALID_SHARD_COPY, null);
+                return AllocateUnassignedDecision.no(AllocationStatus.NO_VALID_SHARD_COPY,
+                    explain ? buildNodeDecisions(null, shardState, inSyncAllocationIds) : null);
             }
         }
 
@@ -228,7 +230,7 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
 
         List<NodeAllocationResult> nodeResults = null;
         if (explain) {
-            nodeResults = buildNodeDecisions(nodesToAllocate, inSyncAllocationIds);
+            nodeResults = buildNodeDecisions(nodesToAllocate, shardState, inSyncAllocationIds);
         }
         if (allocation.hasPendingAsyncFetch()) {
             return AllocateUnassignedDecision.no(AllocationStatus.FETCHING_SHARD_DATA, nodeResults);
@@ -244,13 +246,35 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
     /**
      * Builds a map of nodes to the corresponding allocation decisions for those nodes.
      */
-    private static List<NodeAllocationResult> buildNodeDecisions(NodesToAllocate nodesToAllocate, Set<String> inSyncAllocationIds) {
-        return Stream.of(nodesToAllocate.yesNodeShards, nodesToAllocate.throttleNodeShards, nodesToAllocate.noNodeShards)
-                   .flatMap(Collection::stream)
-                   .map(dnode -> new NodeAllocationResult(dnode.nodeShardState.getNode(),
-                                                             shardStoreInfo(dnode.nodeShardState, inSyncAllocationIds),
-                                                             dnode.decision))
-                   .collect(Collectors.toList());
+    private static List<NodeAllocationResult> buildNodeDecisions(NodesToAllocate nodesToAllocate,
+                                                                 FetchResult<NodeGatewayStartedShards> fetchedShardData,
+                                                                 Set<String> inSyncAllocationIds) {
+        List<NodeAllocationResult> nodeResults = new ArrayList<>();
+        Collection<NodeGatewayStartedShards> ineligibleShards;
+        if (nodesToAllocate != null) {
+            final Set<DiscoveryNode> discoNodes = new HashSet<>();
+            nodeResults.addAll(Stream.of(nodesToAllocate.yesNodeShards, nodesToAllocate.throttleNodeShards, nodesToAllocate.noNodeShards)
+                                .flatMap(Collection::stream)
+                                .map(dnode -> {
+                                    discoNodes.add(dnode.nodeShardState.getNode());
+                                    return new NodeAllocationResult(dnode.nodeShardState.getNode(),
+                                                                       shardStoreInfo(dnode.nodeShardState, inSyncAllocationIds),
+                                                                       dnode.decision);
+                                }).collect(Collectors.toList()));
+            ineligibleShards = fetchedShardData.getData().values().stream().filter(shardData ->
+                discoNodes.contains(shardData.getNode()) == false
+            ).collect(Collectors.toList());
+        } else {
+            // there were no shard copies that were eligible for being assigned the allocation,
+            // so all fetched shard data are ineligible shards
+            ineligibleShards = fetchedShardData.getData().values();
+        }
+
+        nodeResults.addAll(ineligibleShards.stream().map(shardData ->
+            new NodeAllocationResult(shardData.getNode(), shardStoreInfo(shardData, inSyncAllocationIds), null)
+        ).collect(Collectors.toList()));
+
+        return nodeResults;
     }
 
     private static ShardStoreInfo shardStoreInfo(NodeGatewayStartedShards nodeShardState, Set<String> inSyncAllocationIds) {
@@ -389,7 +413,8 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
                 yesNodeShards.add(decidedNode);
             }
         }
-        return new NodesToAllocate(Collections.unmodifiableList(yesNodeShards), Collections.unmodifiableList(throttledNodeShards), Collections.unmodifiableList(noNodeShards));
+        return new NodesToAllocate(Collections.unmodifiableList(yesNodeShards), Collections.unmodifiableList(throttledNodeShards),
+                                      Collections.unmodifiableList(noNodeShards));
     }
 
     /**
@@ -476,17 +501,21 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
      * recovered on any node
      */
     private boolean recoverOnAnyNode(IndexMetaData metaData) {
-        return (IndexMetaData.isOnSharedFilesystem(metaData.getSettings()) || IndexMetaData.isOnSharedFilesystem(this.settings))
-            && IndexMetaData.INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING.get(metaData.getSettings(), this.settings);
+        // don't use the setting directly, not to trigger verbose deprecation logging
+        return (metaData.isOnSharedFilesystem(metaData.getSettings()) || metaData.isOnSharedFilesystem(this.settings))
+            && (metaData.getSettings().getAsBooleanLenientForPreEs6Indices(
+                metaData.getCreationVersion(), IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, deprecationLogger) ||
+                this.settings.getAsBooleanLenientForPreEs6Indices
+                    (metaData.getCreationVersion(), IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, deprecationLogger));
     }
 
     protected abstract FetchResult<NodeGatewayStartedShards> fetchData(ShardRouting shard, RoutingAllocation allocation);
 
-    static class NodeShardsResult {
-        public final List<NodeGatewayStartedShards> orderedAllocationCandidates;
-        public final int allocationsFound;
+    private static class NodeShardsResult {
+        final List<NodeGatewayStartedShards> orderedAllocationCandidates;
+        final int allocationsFound;
 
-        public NodeShardsResult(List<NodeGatewayStartedShards> orderedAllocationCandidates, int allocationsFound) {
+        NodeShardsResult(List<NodeGatewayStartedShards> orderedAllocationCandidates, int allocationsFound) {
             this.orderedAllocationCandidates = orderedAllocationCandidates;
             this.allocationsFound = allocationsFound;
         }
@@ -497,7 +526,7 @@ public abstract class PrimaryShardAllocator extends BaseGatewayShardAllocator {
         final List<DecidedNode> throttleNodeShards;
         final List<DecidedNode> noNodeShards;
 
-        public NodesToAllocate(List<DecidedNode> yesNodeShards, List<DecidedNode> throttleNodeShards, List<DecidedNode> noNodeShards) {
+        NodesToAllocate(List<DecidedNode> yesNodeShards, List<DecidedNode> throttleNodeShards, List<DecidedNode> noNodeShards) {
             this.yesNodeShards = yesNodeShards;
             this.throttleNodeShards = throttleNodeShards;
             this.noNodeShards = noNodeShards;
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 3a838ef3785..b91637e072f 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -395,7 +395,7 @@ public abstract class ReplicaShardAllocator extends BaseGatewayShardAllocator {
         @Nullable
         private final Map<String, NodeAllocationResult> nodeDecisions;
 
-        public MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize, @Nullable Map<String, NodeAllocationResult> nodeDecisions) {
+        MatchingNodes(ObjectLongMap<DiscoveryNode> nodesToSize, @Nullable Map<String, NodeAllocationResult> nodeDecisions) {
             this.nodesToSize = nodesToSize;
             this.nodeDecisions = nodeDecisions;
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index 3e6769200e9..13c317c53e9 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -185,7 +185,7 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             if (in.readBoolean()) {
-                metaData = MetaData.Builder.readFrom(in);
+                metaData = MetaData.readFrom(in);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 31fc290c10c..5ef29b1cc40 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
@@ -115,7 +116,7 @@ public class TransportNodesListGatewayStartedShards extends
         try {
             final ShardId shardId = request.getShardId();
             logger.trace("{} loading local shard state info", shardId);
-            ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger,
+            ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
                 nodeEnv.availableShardPaths(request.shardId));
             if (shardStateMetaData != null) {
                 IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex());
@@ -123,7 +124,8 @@ public class TransportNodesListGatewayStartedShards extends
                     // we may send this requests while processing the cluster state that recovered the index
                     // sometimes the request comes in before the local node processed that cluster state
                     // in such cases we can load it from disk
-                    metaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(shardId.getIndex()));
+                    metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                        nodeEnv.indexPaths(shardId.getIndex()));
                 }
                 if (metaData == null) {
                     ElasticsearchException e = new ElasticsearchException("failed to find local IndexMetaData");
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
deleted file mode 100644
index 06bc392587a..00000000000
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http;
-
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
-import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
-
-/**
- * A component to serve http requests, backed by rest handlers.
- */
-public class HttpServer extends AbstractLifecycleComponent implements HttpServerAdapter {
-    private final HttpServerTransport transport;
-
-    private final RestController restController;
-
-    private final NodeClient client;
-
-    private final CircuitBreakerService circuitBreakerService;
-
-    public HttpServer(Settings settings, HttpServerTransport transport, RestController restController,
-                      NodeClient client, CircuitBreakerService circuitBreakerService) {
-        super(settings);
-        this.transport = transport;
-        this.restController = restController;
-        this.client = client;
-        this.circuitBreakerService = circuitBreakerService;
-        transport.httpServerAdapter(this);
-    }
-
-
-    @Override
-    protected void doStart() {
-        transport.start();
-        if (logger.isInfoEnabled()) {
-            logger.info("{}", transport.boundAddress());
-        }
-    }
-
-    @Override
-    protected void doStop() {
-        transport.stop();
-    }
-
-    @Override
-    protected void doClose() {
-        transport.close();
-    }
-
-    public HttpInfo info() {
-        return transport.info();
-    }
-
-    public HttpStats stats() {
-        return transport.stats();
-    }
-
-    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
-        if (request.rawPath().equals("/favicon.ico")) {
-            handleFavicon(request, channel);
-            return;
-        }
-        RestChannel responseChannel = channel;
-        try {
-            int contentLength = request.content().length();
-            if (restController.canTripCircuitBreaker(request)) {
-                inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
-            } else {
-                inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength);
-            }
-            // iff we could reserve bytes for the request we need to send the response also over this channel
-            responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
-            restController.dispatchRequest(request, responseChannel, client, threadContext);
-        } catch (Exception e) {
-            try {
-                responseChannel.sendResponse(new BytesRestResponse(channel, e));
-            } catch (Exception inner) {
-                inner.addSuppressed(e);
-                logger.error((Supplier<?>) () ->
-                    new ParameterizedMessage("failed to send failure response for uri [{}]", request.uri()), inner);
-            }
-        }
-    }
-
-    void handleFavicon(RestRequest request, RestChannel channel) {
-        if (request.method() == RestRequest.Method.GET) {
-            try {
-                try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
-                    ByteArrayOutputStream out = new ByteArrayOutputStream();
-                    Streams.copy(stream, out);
-                    BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
-                    channel.sendResponse(restResponse);
-                }
-            } catch (IOException e) {
-                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-            }
-        } else {
-            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-        }
-    }
-
-    private static final class ResourceHandlingHttpChannel implements RestChannel {
-        private final RestChannel delegate;
-        private final CircuitBreakerService circuitBreakerService;
-        private final int contentLength;
-        private final AtomicBoolean closed = new AtomicBoolean();
-
-        public ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength) {
-            this.delegate = delegate;
-            this.circuitBreakerService = circuitBreakerService;
-            this.contentLength = contentLength;
-        }
-
-        @Override
-        public XContentBuilder newBuilder() throws IOException {
-            return delegate.newBuilder();
-        }
-
-        @Override
-        public XContentBuilder newErrorBuilder() throws IOException {
-            return delegate.newErrorBuilder();
-        }
-
-        @Override
-        public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-            return delegate.newBuilder(autoDetectSource, useFiltering);
-        }
-
-        @Override
-        public BytesStreamOutput bytesOutput() {
-            return delegate.bytesOutput();
-        }
-
-        @Override
-        public RestRequest request() {
-            return delegate.request();
-        }
-
-        @Override
-        public boolean detailedErrorsEnabled() {
-            return delegate.detailedErrorsEnabled();
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            close();
-            delegate.sendResponse(response);
-        }
-
-        private void close() {
-            // attempt to close once atomically
-            if (closed.compareAndSet(false, true) == false) {
-                throw new IllegalStateException("Channel is already closed");
-            }
-            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength);
-        }
-
-    }
-
-    private static CircuitBreaker inFlightRequestsBreaker(CircuitBreakerService circuitBreakerService) {
-        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
-        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
index 4dc4a888d8a..134557a28ad 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
@@ -21,11 +21,13 @@ package org.elasticsearch.http;
 
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
 
 public interface HttpServerTransport extends LifecycleComponent {
 
     String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
-    String HTTP_SERVER_BOSS_THREAD_NAME_PREFIX = "http_server_boss";
 
     BoundTransportAddress boundAddress();
 
@@ -33,6 +35,15 @@ public interface HttpServerTransport extends LifecycleComponent {
 
     HttpStats stats();
 
-    void httpServerAdapter(HttpServerAdapter httpServerAdapter);
-
+    @FunctionalInterface
+    interface Dispatcher {
+        /**
+         * Dispatches the {@link RestRequest} to the relevant request handler or responds to the given rest channel directly if
+         * the request can't be handled by any request handler.
+         * @param request the request to dispatch
+         * @param channel the response channel of this request
+         * @param threadContext the nodes thread context
+         */
+        void dispatch(RestRequest request, RestChannel channel, ThreadContext threadContext);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 60bc3449d0b..b5e254aa4c2 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -68,6 +68,8 @@ public final class HttpTransportSettings {
         Setting.intSetting("http.publish_port", -1, -1, Property.NodeScope);
     public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED =
         Setting.boolSetting("http.detailed_errors.enabled", true, Property.NodeScope);
+    public static final Setting<Boolean> SETTING_HTTP_CONTENT_TYPE_REQUIRED =
+        Setting.boolSetting("http.content_type.required", false, Property.NodeScope);
     public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH =
         Setting.byteSizeSetting("http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope);
     public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE =
diff --git a/core/src/main/java/org/elasticsearch/index/Index.java b/core/src/main/java/org/elasticsearch/index/Index.java
index 25b293ad387..da94ad2ec72 100644
--- a/core/src/main/java/org/elasticsearch/index/Index.java
+++ b/core/src/main/java/org/elasticsearch/index/Index.java
@@ -21,8 +21,6 @@ package org.elasticsearch.index;
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -42,7 +40,7 @@ public class Index implements Writeable, ToXContent {
     public static final Index[] EMPTY_ARRAY = new Index[0];
     private static final String INDEX_UUID_KEY = "index_uuid";
     private static final String INDEX_NAME_KEY = "index_name";
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
+    private static final ObjectParser<Builder, Void> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
     static {
         INDEX_PARSER.declareString(Builder::name, new ParseField(INDEX_NAME_KEY));
         INDEX_PARSER.declareString(Builder::uuid, new ParseField(INDEX_UUID_KEY));
@@ -118,11 +116,7 @@ public class Index implements Writeable, ToXContent {
     }
 
     public static Index fromXContent(final XContentParser parser) throws IOException {
-        return INDEX_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build();
-    }
-
-    public static final Index parseIndex(final XContentParser parser, final ParseFieldMatcherSupplier supplier) {
-        return INDEX_PARSER.apply(parser, supplier).build();
+        return INDEX_PARSER.parse(parser, null).build();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index 8389335d889..dc7021e81fc 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -23,10 +23,12 @@ import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
@@ -47,7 +49,6 @@ import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -69,7 +70,7 @@ import java.util.function.Function;
  * IndexModule represents the central extension point for index level custom implementations like:
  * <ul>
  *     <li>{@link SimilarityProvider} - New {@link SimilarityProvider} implementations can be registered through
- *     {@link #addSimilarity(String, BiFunction)}while existing Providers can be referenced through Settings under the
+ *     {@link #addSimilarity(String, TriFunction)}while existing Providers can be referenced through Settings under the
  *     {@link IndexModule#SIMILARITY_SETTINGS_PREFIX} prefix along with the "type" value.  For example, to reference the
  *     {@link BM25SimilarityProvider}, the configuration <tt>"index.similarity.my_similarity.type : "BM25"</tt> can be used.</li>
  *      <li>{@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link #addIndexStore(String, Function)}</li>
@@ -112,7 +113,7 @@ public final class IndexModule {
     final SetOnce<EngineFactory> engineFactory = new SetOnce<>();
     private SetOnce<IndexSearcherWrapperFactory> indexSearcherWrapper = new SetOnce<>();
     private final Set<IndexEventListener> indexEventListeners = new HashSet<>();
-    private final Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities = new HashMap<>();
+    private final Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> similarities = new HashMap<>();
     private final Map<String, Function<IndexSettings, IndexStore>> storeTypes = new HashMap<>();
     private final SetOnce<BiFunction<IndexSettings, IndicesQueryCache, QueryCache>> forceQueryCacheProvider = new SetOnce<>();
     private final List<SearchOperationListener> searchOperationListeners = new ArrayList<>();
@@ -256,7 +257,7 @@ public final class IndexModule {
      * @param name Name of the SimilarityProvider
      * @param similarity SimilarityProvider to register
      */
-    public void addSimilarity(String name, BiFunction<String, Settings, SimilarityProvider> similarity) {
+    public void addSimilarity(String name, TriFunction<String, Settings, Settings, SimilarityProvider> similarity) {
         ensureNotFrozen();
         if (similarities.containsKey(name) || SimilarityService.BUILT_IN.containsKey(name)) {
             throw new IllegalArgumentException("similarity for name: [" + name + " is already registered");
@@ -315,17 +316,17 @@ public final class IndexModule {
         /**
          * Returns a new IndexSearcherWrapper. This method is called once per index per node
          */
-        IndexSearcherWrapper newWrapper(final IndexService indexService);
+        IndexSearcherWrapper newWrapper(IndexService indexService);
     }
 
     public IndexService newIndexService(
         NodeEnvironment environment,
+        NamedXContentRegistry xContentRegistry,
         IndexService.ShardStoreDeleter shardStoreDeleter,
         CircuitBreakerService circuitBreakerService,
         BigArrays bigArrays,
         ThreadPool threadPool,
         ScriptService scriptService,
-        IndicesQueriesRegistry indicesQueriesRegistry,
         ClusterService clusterService,
         Client client,
         IndicesQueryCache indicesQueryCache,
@@ -362,18 +363,18 @@ public final class IndexModule {
         } else {
             queryCache = new DisabledQueryCache(indexSettings);
         }
-        return new IndexService(indexSettings, environment, new SimilarityService(indexSettings, similarities), shardStoreDeleter,
-                analysisRegistry, engineFactory.get(), circuitBreakerService, bigArrays, threadPool, scriptService, indicesQueriesRegistry,
-                clusterService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry, indicesFieldDataCache,
-                globalCheckpointSyncer, searchOperationListeners, indexOperationListeners);
+        return new IndexService(indexSettings, environment, xContentRegistry, new SimilarityService(indexSettings, similarities),
+                shardStoreDeleter, analysisRegistry, engineFactory.get(), circuitBreakerService, bigArrays, threadPool, scriptService,
+                clusterService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry,
+                indicesFieldDataCache, globalCheckpointSyncer, searchOperationListeners, indexOperationListeners);
     }
 
     /**
      * creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing.
      * doing so will result in an exception.
      */
-    public MapperService newIndexMapperService(MapperRegistry mapperRegistry) throws IOException {
-        return new MapperService(indexSettings, analysisRegistry.build(indexSettings),
+    public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry) throws IOException {
+        return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry,
             new SimilarityService(indexSettings, similarities), mapperRegistry,
             () -> { throw new UnsupportedOperationException("no index query shard context available"); });
     }
diff --git a/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java b/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
index 035b90dd25e..4442ee276c9 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexNotFoundException.java
@@ -24,9 +24,16 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import java.io.IOException;
 
 public final class IndexNotFoundException extends ResourceNotFoundException {
+    /**
+     * Construct with a custom message.
+     */
+    public IndexNotFoundException(String message, String index) {
+        super(message);
+        setIndex(index);
+    }
 
     public IndexNotFoundException(String index) {
-        this(index, null);
+        this(index, (Throwable) null);
     }
 
     public IndexNotFoundException(String index, Throwable cause) {
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 5ba2889f4bb..ee35993c01e 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.env.ShardLockObtainFailedException;
@@ -43,7 +44,6 @@ import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
@@ -67,7 +67,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -102,6 +101,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final IndexSearcherWrapper searcherWrapper;
     private final IndexCache indexCache;
     private final MapperService mapperService;
+    private final NamedXContentRegistry xContentRegistry;
     private final SimilarityService similarityService;
     private final EngineFactory engineFactory;
     private final IndexWarmer warmer;
@@ -118,11 +118,11 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final BigArrays bigArrays;
     private final AsyncGlobalCheckpointTask globalCheckpointTask;
     private final ScriptService scriptService;
-    private final IndicesQueriesRegistry queryRegistry;
     private final ClusterService clusterService;
     private final Client client;
 
     public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv,
+                        NamedXContentRegistry xContentRegistry,
                         SimilarityService similarityService,
                         ShardStoreDeleter shardStoreDeleter,
                         AnalysisRegistry registry,
@@ -131,7 +131,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                         BigArrays bigArrays,
                         ThreadPool threadPool,
                         ScriptService scriptService,
-                        IndicesQueriesRegistry queryRegistry,
                         ClusterService clusterService,
                         Client client,
                         QueryCache queryCache,
@@ -146,8 +145,10 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         super(indexSettings);
         this.indexSettings = indexSettings;
         this.globalCheckpointSyncer = globalCheckpointSyncer;
+        this.xContentRegistry = xContentRegistry;
         this.similarityService = similarityService;
-        this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), similarityService, mapperRegistry,
+        this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
+            mapperRegistry,
             // we parse all percolator queries as they would be parsed on shard 0
             () -> newQueryShardContext(0, null, () -> {
                 throw new IllegalArgumentException("Percolator queries are not allowed to use the current timestamp");
@@ -157,7 +158,6 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         this.bigArrays = bigArrays;
         this.threadPool = threadPool;
         this.scriptService = scriptService;
-        this.queryRegistry = queryRegistry;
         this.clusterService = clusterService;
         this.client = client;
         this.eventListener = eventListener;
@@ -236,6 +236,10 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         return mapperService;
     }
 
+    public NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     public SimilarityService similarityService() {
         return similarityService;
     }
@@ -339,8 +343,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
 
             logger.debug("creating shard_id {}", shardId);
             // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
-            final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false ||
-                (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
+            final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false ||
+                (primary && this.indexSettings.isOnSharedFilesystem());
             final Engine.Warmer engineWarmer = (searcher) -> {
                 IndexShard shard =  getShardOrNull(shardId.getId());
                 if (shard != null) {
@@ -349,7 +353,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
             };
             store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock,
                 new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
-            if (useShadowEngine(primary, indexSettings)) {
+            if (useShadowEngine(primary, this.indexSettings)) {
                 indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService,
                     indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer,
                     searchOperationListeners);
@@ -377,8 +381,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         }
     }
 
-    static boolean useShadowEngine(boolean primary, Settings indexSettings) {
-        return primary == false && IndexMetaData.isIndexUsingShadowReplicas(indexSettings);
+    static boolean useShadowEngine(boolean primary, IndexSettings indexSettings) {
+        return primary == false && indexSettings.isShadowReplicaIndex();
     }
 
     @Override
@@ -469,7 +473,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     public QueryShardContext newQueryShardContext(int shardId, IndexReader indexReader, LongSupplier nowInMillis) {
         return new QueryShardContext(
             shardId, indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(),
-                similarityService(), scriptService, queryRegistry,
+                similarityService(), scriptService, xContentRegistry,
                 client, indexReader,
             nowInMillis);
     }
@@ -513,7 +517,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         private final boolean ownsShard;
         private final Closeable[] toClose;
 
-        public StoreCloseListener(ShardId shardId, boolean ownsShard, Closeable... toClose) {
+        StoreCloseListener(ShardId shardId, boolean ownsShard, Closeable... toClose) {
             this.shardId = shardId;
             this.ownsShard = ownsShard;
             this.toClose = toClose;
@@ -568,7 +572,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final class FieldDataCacheListener implements IndexFieldDataCache.Listener {
         final IndexService indexService;
 
-        public FieldDataCacheListener(IndexService indexService) {
+        FieldDataCacheListener(IndexService indexService) {
             this.indexService = indexService;
         }
 
@@ -670,7 +674,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                     if (translog.syncNeeded()) {
                         translog.sync();
                     }
-                } catch (EngineClosedException | AlreadyClosedException ex) {
+                } catch (AlreadyClosedException ex) {
                     // fine - continue;
                 } catch (IOException e) {
                     logger.warn("failed to sync translog", e);
@@ -718,7 +722,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                     case STARTED:
                         try {
                             shard.updateGlobalCheckpointOnPrimary();
-                        } catch (EngineClosedException | AlreadyClosedException ex) {
+                        } catch (AlreadyClosedException ex) {
                             // fine - continue, the shard was concurrently closed on us.
                         }
                         continue;
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index dbe6d3505a4..47c7ffb71bc 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -22,7 +22,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.MergePolicy;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -97,6 +97,13 @@ public final class IndexSettings {
      */
     public static final Setting<Integer> MAX_RESCORE_WINDOW_SETTING =
             Setting.intSetting("index.max_rescore_window", MAX_RESULT_WINDOW_SETTING, 1, Property.Dynamic, Property.IndexScope);
+    /**
+     * Index setting describing the maximum number of filters clauses that can be used
+     * in an adjacency_matrix aggregation. The max number of buckets produced by  
+     * N filters is (N*N)/2 so a limit of 100 filters is imposed by default.
+     */
+    public static final Setting<Integer> MAX_ADJACENCY_MATRIX_FILTERS_SETTING =
+        Setting.intSetting("index.max_adjacency_matrix_filters", 100, 2, Property.Dynamic, Property.IndexScope);    
     public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
     public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.MILLISECONDS),
@@ -136,7 +143,6 @@ public final class IndexSettings {
     private final Settings nodeSettings;
     private final int numberOfShards;
     private final boolean isShadowReplicaIndex;
-    private final ParseFieldMatcher parseFieldMatcher;
     // volatile fields are updated via #updateIndexMetaData(IndexMetaData) under lock
     private volatile Settings settings;
     private volatile IndexMetaData indexMetaData;
@@ -156,6 +162,7 @@ public final class IndexSettings {
     private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
     private volatile boolean warmerEnabled;
     private volatile int maxResultWindow;
+    private volatile int maxAdjacencyMatrixFilters;
     private volatile int maxRescoreWindow;
     private volatile boolean TTLPurgeDisabled;
     /**
@@ -231,13 +238,12 @@ public final class IndexSettings {
         nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.indexMetaData = indexMetaData;
         numberOfShards = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
-        isShadowReplicaIndex = IndexMetaData.isIndexUsingShadowReplicas(settings);
+        isShadowReplicaIndex = indexMetaData.isIndexUsingShadowReplicas(settings);
 
         this.defaultField = DEFAULT_FIELD_SETTING.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
         this.queryStringAnalyzeWildcard = QUERY_STRING_ANALYZE_WILDCARD.get(nodeSettings);
         this.queryStringAllowLeadingWildcard = QUERY_STRING_ALLOW_LEADING_WILDCARD.get(nodeSettings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.defaultAllowUnmappedFields = scopedSettings.get(ALLOW_UNMAPPED);
         this.durability = scopedSettings.get(INDEX_TRANSLOG_DURABILITY_SETTING);
         syncInterval = INDEX_TRANSLOG_SYNC_INTERVAL_SETTING.get(settings);
@@ -248,6 +254,7 @@ public final class IndexSettings {
         gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
         warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
         maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
+        maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
         maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
@@ -269,6 +276,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
         scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
+        scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
         scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
         scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
         scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
@@ -330,16 +338,7 @@ public final class IndexSettings {
      * filesystem.
      */
     public boolean isOnSharedFilesystem() {
-        return IndexMetaData.isOnSharedFilesystem(getSettings());
-    }
-
-    /**
-     * Returns <code>true</code> iff the given settings indicate that the index associated
-     * with these settings uses shadow replicas. Otherwise <code>false</code>. The default
-     * setting for this is <code>false</code>.
-     */
-    public boolean isIndexUsingShadowReplicas() {
-        return IndexMetaData.isOnSharedFilesystem(getSettings());
+        return indexMetaData.isOnSharedFilesystem(getSettings());
     }
 
     /**
@@ -388,11 +387,6 @@ public final class IndexSettings {
         return nodeSettings;
     }
 
-    /**
-     * Returns a {@link ParseFieldMatcher} for this index.
-     */
-    public ParseFieldMatcher getParseFieldMatcher() { return parseFieldMatcher; }
-
     /**
      * Updates the settings and index metadata and notifies all registered settings consumers with the new settings iff at least one setting has changed.
      *
@@ -482,6 +476,17 @@ public final class IndexSettings {
     private void setMaxResultWindow(int maxResultWindow) {
         this.maxResultWindow = maxResultWindow;
     }
+    
+    /**
+     * Returns the max number of filters in adjacency_matrix aggregation search requests
+     */
+    public int getMaxAdjacencyMatrixFilters() {
+        return this.maxAdjacencyMatrixFilters;
+    }
+
+    private void setMaxAdjacencyMatrixFilters(int maxAdjacencyFilters) {
+        this.maxAdjacencyMatrixFilters = maxAdjacencyFilters;
+    }    
 
     /**
      * Returns the maximum rescore window for search requests.
diff --git a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
index 72068f3d1d2..fdaad19e528 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -111,7 +111,7 @@ public final class IndexWarmer extends AbstractComponent {
     private static class FieldDataWarmer implements IndexWarmer.Listener {
 
         private final Executor executor;
-        public FieldDataWarmer(Executor executor) {
+        FieldDataWarmer(Executor executor) {
             this.executor = executor;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
index 215013bf246..b1d8ac18862 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
@@ -76,13 +77,14 @@ public final class IndexingSlowLog implements IndexingOperationListener {
      * and everything else is interpreted as Elasticsearch interprets booleans
      * which is then converted to 0 for false and Integer.MAX_VALUE for true.
      */
-    public static final Setting<Integer> INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING = new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX + ".source", "1000", (value) -> {
-        try {
-            return Integer.parseInt(value, 10);
-        } catch (NumberFormatException e) {
-            return Booleans.parseBoolean(value, true) ? Integer.MAX_VALUE : 0;
-        }
-    }, Property.Dynamic, Property.IndexScope);
+    public static final Setting<Integer> INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING =
+            new Setting<>(INDEX_INDEXING_SLOWLOG_PREFIX + ".source", "1000", (value) -> {
+                try {
+                    return Integer.parseInt(value, 10);
+                } catch (NumberFormatException e) {
+                    return Booleans.parseBoolean(value, true) ? Integer.MAX_VALUE : 0;
+                }
+            }, Property.Dynamic, Property.IndexScope);
 
     IndexingSlowLog(IndexSettings indexSettings) {
         this.indexLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index", indexSettings.getSettings());
@@ -90,17 +92,22 @@ public final class IndexingSlowLog implements IndexingOperationListener {
 
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING, this::setReformat);
         this.reformat = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, this::setWarnThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING, this::setWarnThreshold);
         this.indexWarnThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, this::setInfoThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING, this::setInfoThreshold);
         this.indexInfoThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, this::setDebugThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING, this::setDebugThreshold);
         this.indexDebugThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING).nanos();
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, this::setTraceThreshold);
+        indexSettings.getScopedSettings()
+                .addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING, this::setTraceThreshold);
         this.indexTraceThreshold = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING).nanos();
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_LEVEL_SETTING, this::setLevel);
         setLevel(indexSettings.getValue(INDEX_INDEXING_SLOWLOG_LEVEL_SETTING));
-        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING, this::setMaxSourceCharsToLog);
+        indexSettings.getScopedSettings().addSettingsUpdateConsumer(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING,
+                this::setMaxSourceCharsToLog);
         this.maxSourceCharsToLog = indexSettings.getValue(INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);
     }
 
@@ -134,7 +141,7 @@ public final class IndexingSlowLog implements IndexingOperationListener {
     }
 
     @Override
-    public void postIndex(Engine.Index indexOperation, Engine.IndexResult result) {
+    public void postIndex(ShardId shardId, Engine.Index indexOperation, Engine.IndexResult result) {
         if (result.hasFailure() == false) {
             final ParsedDocument doc = indexOperation.parsedDoc();
             final long tookInNanos = result.getTook();
@@ -169,7 +176,8 @@ public final class IndexingSlowLog implements IndexingOperationListener {
         public String toString() {
             StringBuilder sb = new StringBuilder();
             sb.append(index).append(" ");
-            sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ");
+            sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], ");
+            sb.append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ");
             sb.append("type[").append(doc.type()).append("], ");
             sb.append("id[").append(doc.id()).append("], ");
             if (doc.routing() == null) {
@@ -182,7 +190,7 @@ public final class IndexingSlowLog implements IndexingOperationListener {
                 return sb.toString();
             }
             try {
-                String source = XContentHelper.convertToJson(doc.source(), reformat);
+                String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
                 sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]");
             } catch (IOException e) {
                 sb.append(", source[_failed_to_convert_]");
diff --git a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index 0f7305789ec..b3a0224f983 100644
--- a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.TieredMergePolicy;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -164,7 +165,8 @@ public final class MergePolicyConfig {
         ByteSizeValue maxMergedSegment = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
         double segmentsPerTier = indexSettings.getValue(INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
         double reclaimDeletesWeight = indexSettings.getValue(INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING);
-        this.mergesEnabled = indexSettings.getSettings().getAsBoolean(INDEX_MERGE_ENABLED, true);
+        this.mergesEnabled = indexSettings.getSettings()
+            .getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), INDEX_MERGE_ENABLED, true, new DeprecationLogger(logger));
         if (mergesEnabled == false) {
             logger.warn("[{}] is set to false, this should only be used in tests and can cause serious problems in production environments", INDEX_MERGE_ENABLED);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
index b52164699ec..a48e3d7bd72 100644
--- a/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
+++ b/core/src/main/java/org/elasticsearch/index/SearchSlowLog.java
@@ -141,7 +141,7 @@ public final class SearchSlowLog implements SearchOperationListener {
         private final SearchContext context;
         private final long tookInNanos;
 
-        public SlowLogSearchContextPrinter(SearchContext context, long tookInNanos) {
+        SlowLogSearchContextPrinter(SearchContext context, long tookInNanos) {
             this.context = context;
             this.tookInNanos = tookInNanos;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
index 4318ef273dc..5e53a86129a 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
@@ -37,7 +37,8 @@ public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory i
 
     public ASCIIFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        preserveOriginal = settings.getAsBoolean(PRESERVE_ORIGINAL.getPreferredName(), DEFAULT_PRESERVE_ORIGINAL);
+        preserveOriginal = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), PRESERVE_ORIGINAL.getPreferredName(), DEFAULT_PRESERVE_ORIGINAL, deprecationLogger);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
index aded2bb4ee9..80129e621d3 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/Analysis.java
@@ -56,6 +56,8 @@ import org.apache.lucene.analysis.tr.TurkishAnalyzer;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -79,6 +81,8 @@ import static java.util.Collections.unmodifiableMap;
 
 public class Analysis {
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Analysis.class));
+
     public static Version parseAnalysisVersion(Settings indexSettings, Settings settings, Logger logger) {
         // check for explicit version on the specific analyzer component
         String sVersion = settings.get("version");
@@ -178,12 +182,16 @@ public class Analysis {
         return parseWords(env, settings, "common_words", defaultCommonWords, NAMED_STOP_WORDS, ignoreCase);
     }
 
-    public static CharArraySet parseArticles(Environment env, Settings settings) {
-        return parseWords(env, settings, "articles", null, null, settings.getAsBoolean("articles_case", false));
+    public static CharArraySet parseArticles(Environment env, org.elasticsearch.Version indexCreatedVersion, Settings settings) {
+        boolean articlesCase = settings.getAsBooleanLenientForPreEs6Indices(indexCreatedVersion, "articles_case", false, deprecationLogger);
+        return parseWords(env, settings, "articles", null, null, articlesCase);
     }
 
-    public static CharArraySet parseStopWords(Environment env, Settings settings, CharArraySet defaultStopWords) {
-        return parseStopWords(env, settings, defaultStopWords, settings.getAsBoolean("stopwords_case", false));
+    public static CharArraySet parseStopWords(Environment env, org.elasticsearch.Version indexCreatedVersion, Settings settings,
+                                              CharArraySet defaultStopWords) {
+        boolean stopwordsCase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexCreatedVersion, "stopwords_case", false, deprecationLogger);
+        return parseStopWords(env, settings, defaultStopWords, stopwordsCase);
     }
 
     public static CharArraySet parseStopWords(Environment env, Settings settings, CharArraySet defaultStopWords, boolean ignoreCase) {
@@ -205,12 +213,15 @@ public class Analysis {
         return setWords;
     }
 
-    public static CharArraySet getWordSet(Environment env, Settings settings, String settingsPrefix) {
+    public static CharArraySet getWordSet(Environment env, org.elasticsearch.Version indexCreatedVersion, Settings settings,
+                                          String settingsPrefix) {
         List<String> wordList = getWordList(env, settings, settingsPrefix);
         if (wordList == null) {
             return null;
         }
-        return new CharArraySet(wordList, settings.getAsBoolean(settingsPrefix + "_case", false));
+        boolean ignoreCase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexCreatedVersion, settingsPrefix + "_case", false, deprecationLogger);
+        return new CharArraySet(wordList, ignoreCase);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index 87f9692f625..36357afe678 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -67,17 +67,20 @@ public final class AnalysisRegistry implements Closeable {
     private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters;
     private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers;
     private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers;
+    private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> normalizers;
 
     public AnalysisRegistry(Environment environment,
                             Map<String, AnalysisProvider<CharFilterFactory>> charFilters,
                             Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters,
                             Map<String, AnalysisProvider<TokenizerFactory>> tokenizers,
-                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers) {
+                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers,
+                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> normalizers) {
         this.environment = environment;
         this.charFilters = unmodifiableMap(charFilters);
         this.tokenFilters = unmodifiableMap(tokenFilters);
         this.tokenizers = unmodifiableMap(tokenizers);
         this.analyzers = unmodifiableMap(analyzers);
+        this.normalizers = unmodifiableMap(normalizers);
     }
 
     /**
@@ -151,7 +154,8 @@ public final class AnalysisRegistry implements Closeable {
         final Map<String, TokenizerFactory> tokenizerFactories = buildTokenizerFactories(indexSettings);
         final Map<String, TokenFilterFactory> tokenFilterFactories = buildTokenFilterFactories(indexSettings);
         final Map<String, AnalyzerProvider<?>> analyzierFactories = buildAnalyzerFactories(indexSettings);
-        return build(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
+        final Map<String, AnalyzerProvider<?>> normalizerFactories = buildNormalizerFactories(indexSettings);
+        return build(indexSettings, analyzierFactories, normalizerFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
     }
 
     public Map<String, TokenFilterFactory> buildTokenFilterFactories(IndexSettings indexSettings) throws IOException {
@@ -162,24 +166,30 @@ public final class AnalysisRegistry implements Closeable {
          * instead of building the infrastructure for plugins we rather make it a real exception to not pollute the general interface and
          * hide internal data-structures as much as possible.
          */
-        tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
-        tokenFilters.put("synonym_graph", requriesAnalysisSettings((is, env, name, settings) -> new SynonymGraphFilterFactory(is, env, this, name, settings)));
-        return buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
+        tokenFilters.put("synonym", requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings)));
+        tokenFilters.put("synonym_graph", requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings)));
+        return buildMapping(Component.FILTER, indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
     }
 
     public Map<String, TokenizerFactory> buildTokenizerFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> tokenizersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_TOKENIZER);
-        return buildMapping(false, "tokenizer", indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories);
+        return buildMapping(Component.TOKENIZER, indexSettings, tokenizersSettings, tokenizers, prebuiltAnalysis.tokenizerFactories);
     }
 
     public Map<String, CharFilterFactory> buildCharFilterFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> charFiltersSettings = indexSettings.getSettings().getGroups(INDEX_ANALYSIS_CHAR_FILTER);
-        return buildMapping(false, "charfilter", indexSettings, charFiltersSettings, charFilters, prebuiltAnalysis.charFilterFactories);
+        return buildMapping(Component.CHAR_FILTER, indexSettings, charFiltersSettings, charFilters, prebuiltAnalysis.charFilterFactories);
     }
 
     public Map<String, AnalyzerProvider<?>> buildAnalyzerFactories(IndexSettings indexSettings) throws IOException {
         final Map<String, Settings> analyzersSettings = indexSettings.getSettings().getGroups("index.analysis.analyzer");
-        return  buildMapping(true, "analyzer", indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
+        return buildMapping(Component.ANALYZER, indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
+    }
+
+    public Map<String, AnalyzerProvider<?>> buildNormalizerFactories(IndexSettings indexSettings) throws IOException {
+        final Map<String, Settings> noralizersSettings = indexSettings.getSettings().getGroups("index.analysis.normalizer");
+        // TODO: Have pre-built normalizers
+        return buildMapping(Component.NORMALIZER, indexSettings, noralizersSettings, normalizers, Collections.emptyMap());
     }
 
     /**
@@ -194,7 +204,7 @@ public final class AnalysisRegistry implements Closeable {
         final Map<String, Settings> tokenizerSettings = indexSettings.getSettings().getGroups("index.analysis.tokenizer");
         if (tokenizerSettings.containsKey(tokenizer)) {
             Settings currentSettings = tokenizerSettings.get(tokenizer);
-            return getAnalysisProvider("tokenizer", tokenizers, tokenizer, currentSettings.get("type"));
+            return getAnalysisProvider(Component.TOKENIZER, tokenizers, tokenizer, currentSettings.get("type"));
         } else {
             return getTokenizerProvider(tokenizer);
         }
@@ -219,11 +229,11 @@ public final class AnalysisRegistry implements Closeable {
              * hide internal data-structures as much as possible.
              */
             if ("synonym".equals(typeName)) {
-                return requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings));
+                return requiresAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, this, name, settings));
             } else if ("synonym_graph".equals(typeName)) {
-                return requriesAnalysisSettings((is, env, name, settings) -> new SynonymGraphFilterFactory(is, env, this, name, settings));
+                return requiresAnalysisSettings((is, env, name, settings) -> new SynonymGraphTokenFilterFactory(is, env, this, name, settings));
             } else {
-                return getAnalysisProvider("tokenfilter", tokenFilters, tokenFilter, typeName);
+                return getAnalysisProvider(Component.FILTER, tokenFilters, tokenFilter, typeName);
             }
         } else {
             return getTokenFilterProvider(tokenFilter);
@@ -242,13 +252,13 @@ public final class AnalysisRegistry implements Closeable {
         final Map<String, Settings> tokenFilterSettings = indexSettings.getSettings().getGroups("index.analysis.char_filter");
         if (tokenFilterSettings.containsKey(charFilter)) {
             Settings currentSettings = tokenFilterSettings.get(charFilter);
-            return getAnalysisProvider("charfilter", charFilters, charFilter, currentSettings.get("type"));
+            return getAnalysisProvider(Component.CHAR_FILTER, charFilters, charFilter, currentSettings.get("type"));
         } else {
             return getCharFilterProvider(charFilter);
         }
     }
 
-    private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
+    private static <T> AnalysisModule.AnalysisProvider<T> requiresAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
         return new AnalysisModule.AnalysisProvider<T>() {
             @Override
             public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
@@ -261,7 +271,40 @@ public final class AnalysisRegistry implements Closeable {
         };
     }
 
-    private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap,
+    enum Component {
+        ANALYZER {
+            @Override
+            public String toString() {
+                return "analyzer";
+            }
+        },
+        NORMALIZER {
+            @Override
+            public String toString() {
+                return "normalizer";
+            }
+        },
+        CHAR_FILTER {
+            @Override
+            public String toString() {
+                return "char_filter";
+            }
+        },
+        TOKENIZER {
+            @Override
+            public String toString() {
+                return "tokenizer";
+            }
+        },
+        FILTER {
+            @Override
+            public String toString() {
+                return "filter";
+            }
+        };
+    }
+
+    private <T> Map<String, T> buildMapping(Component component, IndexSettings settings, Map<String, Settings> settingsMap,
             Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance)
             throws IOException {
         Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build();
@@ -270,29 +313,34 @@ public final class AnalysisRegistry implements Closeable {
             String name = entry.getKey();
             Settings currentSettings = entry.getValue();
             String typeName = currentSettings.get("type");
-            if (analyzer) {
-                T factory;
+            if (component == Component.ANALYZER) {
+                T factory = null;
                 if (typeName == null) {
                     if (currentSettings.get("tokenizer") != null) {
                         factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
                     } else {
-                        throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
+                        throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer");
                     }
                 } else if (typeName.equals("custom")) {
                     factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
-                } else {
-                    AnalysisModule.AnalysisProvider<T> type = providerMap.get(typeName);
-                    if (type == null) {
-                        throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]");
-                    }
-                    factory = type.get(settings, environment, name, currentSettings);
                 }
-                factories.put(name, factory);
-            }  else {
-                AnalysisProvider<T> type = getAnalysisProvider(toBuild, providerMap, name, typeName);
-                final T factory = type.get(settings, environment, name, currentSettings);
-                factories.put(name, factory);
+                if (factory != null) {
+                    factories.put(name, factory);
+                    continue;
+                }
+            } else if (component == Component.NORMALIZER) {
+                if (typeName == null || typeName.equals("custom")) {
+                    T factory = (T) new CustomNormalizerProvider(settings, name, currentSettings);
+                    factories.put(name, factory);
+                    continue;
+                }
             }
+            AnalysisProvider<T> type = getAnalysisProvider(component, providerMap, name, typeName);
+            if (type == null) {
+                throw new IllegalArgumentException("Unknown " + component + " type [" + typeName + "] for [" + name + "]");
+            }
+            final T factory = type.get(settings, environment, name, currentSettings);
+            factories.put(name, factory);
 
         }
         // go over the char filters in the bindings and register the ones that are not configured
@@ -330,13 +378,13 @@ public final class AnalysisRegistry implements Closeable {
         return factories;
     }
 
-    private <T> AnalysisProvider<T> getAnalysisProvider(String toBuild, Map<String, AnalysisProvider<T>> providerMap, String name, String typeName) {
+    private <T> AnalysisProvider<T> getAnalysisProvider(Component component, Map<String, AnalysisProvider<T>> providerMap, String name, String typeName) {
         if (typeName == null) {
-            throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
+            throw new IllegalArgumentException(component + " [" + name + "] must specify either an analyzer type, or a tokenizer");
         }
         AnalysisProvider<T> type = providerMap.get(typeName);
         if (type == null) {
-            throw new IllegalArgumentException("Unknown " + toBuild + " type [" + typeName + "] for [" + name + "]");
+            throw new IllegalArgumentException("Unknown " + component + " type [" + typeName + "] for [" + name + "]");
         }
         return type;
     }
@@ -426,6 +474,7 @@ public final class AnalysisRegistry implements Closeable {
 
     public IndexAnalyzers build(IndexSettings indexSettings,
                                 Map<String, AnalyzerProvider<?>> analyzerProviders,
+                                Map<String, AnalyzerProvider<?>> normalizerProviders,
                                 Map<String, TokenizerFactory> tokenizerFactoryFactories,
                                 Map<String, CharFilterFactory> charFilterFactoryFactories,
                                 Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
@@ -436,10 +485,15 @@ public final class AnalysisRegistry implements Closeable {
         DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
         Map<String, NamedAnalyzer> analyzerAliases = new HashMap<>();
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
+        Map<String, NamedAnalyzer> normalizers = new HashMap<>();
         for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
             processAnalyzerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), analyzerAliases, analyzers,
                 tokenFilterFactoryFactories, charFilterFactoryFactories, tokenizerFactoryFactories);
         }
+        for (Map.Entry<String, AnalyzerProvider<?>> entry : normalizerProviders.entrySet()) {
+            processNormalizerFactory(deprecationLogger, indexSettings, entry.getKey(), entry.getValue(), normalizers,
+                    tokenFilterFactoryFactories, charFilterFactoryFactories);
+        }
         for (Map.Entry<String, NamedAnalyzer> entry : analyzerAliases.entrySet()) {
             String key = entry.getKey();
             if (analyzers.containsKey(key) &&
@@ -485,7 +539,7 @@ public final class AnalysisRegistry implements Closeable {
             }
         }
         return new IndexAnalyzers(indexSettings, defaultIndexAnalyzer, defaultSearchAnalyzer, defaultSearchQuoteAnalyzer,
-            unmodifiableMap(analyzers));
+            unmodifiableMap(analyzers), unmodifiableMap(normalizers));
     }
 
     private void processAnalyzerFactory(DeprecationLogger deprecationLogger,
@@ -551,4 +605,25 @@ public final class AnalysisRegistry implements Closeable {
             }
         }
     }
+
+    private void processNormalizerFactory(DeprecationLogger deprecationLogger,
+            IndexSettings indexSettings,
+            String name,
+            AnalyzerProvider<?> normalizerFactory,
+            Map<String, NamedAnalyzer> normalizers,
+            Map<String, TokenFilterFactory> tokenFilters,
+            Map<String, CharFilterFactory> charFilters) {
+        if (normalizerFactory instanceof CustomNormalizerProvider) {
+            ((CustomNormalizerProvider) normalizerFactory).build(charFilters, tokenFilters);
+        }
+        Analyzer normalizerF = normalizerFactory.get();
+        if (normalizerF == null) {
+            throw new IllegalArgumentException("normalizer [" + normalizerFactory.name() + "] created null normalizer");
+        }
+        NamedAnalyzer normalizer = new NamedAnalyzer(name, normalizerFactory.scope(), normalizerF);
+        if (normalizers.containsKey(name)) {
+            throw new IllegalStateException("already registered analyzer with name: " + name);
+        }
+        normalizers.put(name, normalizer);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java
index fffa594b9e1..10d8f22bde7 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ArabicAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class ArabicAnalyzerProvider extends AbstractIndexAnalyzerProvider<Arabic
 
     public ArabicAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        arabicAnalyzer = new ArabicAnalyzer(Analysis.parseStopWords(env, settings, ArabicAnalyzer.getDefaultStopSet()),
-                                            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        arabicAnalyzer = new ArabicAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, ArabicAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         arabicAnalyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ArmenianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ArmenianAnalyzerProvider.java
index c53d9da8676..6c5193bbb77 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ArmenianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ArmenianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class ArmenianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Arme
 
     public ArmenianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new ArmenianAnalyzer(Analysis.parseStopWords(env, settings, ArmenianAnalyzer.getDefaultStopSet()),
-                                        Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new ArmenianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, ArmenianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/BasqueAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/BasqueAnalyzerProvider.java
index cd91cb7522c..d55e3fdcba4 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/BasqueAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/BasqueAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class BasqueAnalyzerProvider extends AbstractIndexAnalyzerProvider<Basque
 
     public BasqueAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new BasqueAnalyzer(Analysis.parseStopWords(env, settings, BasqueAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new BasqueAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, BasqueAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/BrazilianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/BrazilianAnalyzerProvider.java
index c2640390f2d..7ca11542ac6 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/BrazilianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/BrazilianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class BrazilianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Bra
 
     public BrazilianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new BrazilianAnalyzer(Analysis.parseStopWords(env, settings, BrazilianAnalyzer.getDefaultStopSet()),
-                                         Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new BrazilianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, BrazilianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/BulgarianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/BulgarianAnalyzerProvider.java
index b1017a1ee7d..f64987d95e8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/BulgarianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/BulgarianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class BulgarianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Bul
 
     public BulgarianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new BulgarianAnalyzer(Analysis.parseStopWords(env, settings, BulgarianAnalyzer.getDefaultStopSet()),
-                                         Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new BulgarianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, BulgarianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
index d7d5139bed5..65fc98d304b 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CJKBigramFilterFactory.java
@@ -50,7 +50,8 @@ public final class CJKBigramFilterFactory extends AbstractTokenFilterFactory {
 
     public CJKBigramFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        outputUnigrams = settings.getAsBoolean("output_unigrams", false);
+        outputUnigrams = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), "output_unigrams", false, deprecationLogger);
         final String[] asArray = settings.getAsArray("ignored_scripts");
         Set<String> scripts = new HashSet<>(Arrays.asList("han", "hiragana", "katakana", "hangul"));
         if (asArray != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java
index 25328421f99..ff0f9e32309 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CatalanAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class CatalanAnalyzerProvider extends AbstractIndexAnalyzerProvider<Catal
 
     public CatalanAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new CatalanAnalyzer(Analysis.parseStopWords(env, settings, CatalanAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new CatalanAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, CatalanAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
index 238fde5e6b3..b9e70d05bb7 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
@@ -45,7 +45,7 @@ public interface CharMatcher {
         }
     }
 
-    public enum Basic implements CharMatcher {
+    enum Basic implements CharMatcher {
         LETTER {
             @Override
             public boolean isTokenChar(int c) {
@@ -97,7 +97,7 @@ public interface CharMatcher {
         }
     }
 
-    public final class Builder {
+    final class Builder {
         private final Set<CharMatcher> matchers;
         Builder() {
             matchers = new HashSet<>();
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CjkAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CjkAnalyzerProvider.java
index dd7f61f706c..a10c539faab 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CjkAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CjkAnalyzerProvider.java
@@ -31,7 +31,8 @@ public class CjkAnalyzerProvider extends AbstractIndexAnalyzerProvider<CJKAnalyz
 
     public CjkAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, CJKAnalyzer.getDefaultStopSet());
+        CharArraySet stopWords = Analysis.parseStopWords(
+            env, indexSettings.getIndexVersionCreated(), settings, CJKAnalyzer.getDefaultStopSet());
 
         analyzer = new CJKAnalyzer(stopWords);
         analyzer.setVersion(version);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CommonGramsTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/CommonGramsTokenFilterFactory.java
index eb6dae82061..b64eb917df8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CommonGramsTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CommonGramsTokenFilterFactory.java
@@ -37,8 +37,8 @@ public class CommonGramsTokenFilterFactory extends AbstractTokenFilterFactory {
 
     public CommonGramsTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
-        this.queryMode = settings.getAsBoolean("query_mode", false);
+        this.ignoreCase = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "ignore_case", false, deprecationLogger);
+        this.queryMode = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "query_mode", false, deprecationLogger);
         this.words = Analysis.parseCommonWords(env, settings, null, ignoreCase);
 
         if (this.words == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
index 6185f358568..68799413907 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CustomAnalyzer.java
@@ -94,4 +94,27 @@ public final class CustomAnalyzer extends Analyzer {
         }
         return reader;
     }
+
+    @Override
+    protected Reader initReaderForNormalization(String fieldName, Reader reader) {
+      for (CharFilterFactory charFilter : charFilters) {
+        if (charFilter instanceof MultiTermAwareComponent) {
+          charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
+          reader = charFilter.create(reader);
+        }
+      }
+      return reader;
+    }
+
+    @Override
+    protected TokenStream normalize(String fieldName, TokenStream in) {
+      TokenStream result = in;
+      for (TokenFilterFactory filter : tokenFilters) {
+        if (filter instanceof MultiTermAwareComponent) {
+          filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
+          result = filter.create(result);
+        }
+      }
+      return result;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java
new file mode 100644
index 00000000000..4f50a34dd9e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CustomNormalizerProvider.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.analysis.PreBuiltTokenizers;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A custom normalizer that is built out of a char and token filters. On the
+ * contrary to analyzers, it does not support tokenizers and only supports a
+ * subset of char and token filters.
+ */
+public final class CustomNormalizerProvider extends AbstractIndexAnalyzerProvider<CustomAnalyzer> {
+
+    private final Settings analyzerSettings;
+
+    private CustomAnalyzer customAnalyzer;
+
+    public CustomNormalizerProvider(IndexSettings indexSettings,
+                                  String name, Settings settings) {
+        super(indexSettings, name, settings);
+        this.analyzerSettings = settings;
+    }
+
+    public void build(final Map<String, CharFilterFactory> charFilters, final Map<String, TokenFilterFactory> tokenFilters) {
+        String tokenizerName = analyzerSettings.get("tokenizer");
+        if (tokenizerName != null) {
+            throw new IllegalArgumentException("Custom normalizer [" + name() + "] cannot configure a tokenizer");
+        }
+
+        List<CharFilterFactory> charFiltersList = new ArrayList<>();
+        String[] charFilterNames = analyzerSettings.getAsArray("char_filter");
+        for (String charFilterName : charFilterNames) {
+            CharFilterFactory charFilter = charFilters.get(charFilterName);
+            if (charFilter == null) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] failed to find char_filter under name ["
+                        + charFilterName + "]");
+            }
+            if (charFilter instanceof MultiTermAwareComponent == false) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use char filter ["
+                        + charFilterName + "]");
+            }
+            charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
+            charFiltersList.add(charFilter);
+        }
+
+        List<TokenFilterFactory> tokenFilterList = new ArrayList<>();
+        String[] tokenFilterNames = analyzerSettings.getAsArray("filter");
+        for (String tokenFilterName : tokenFilterNames) {
+            TokenFilterFactory tokenFilter = tokenFilters.get(tokenFilterName);
+            if (tokenFilter == null) {
+                throw new IllegalArgumentException("Custom Analyzer [" + name() + "] failed to find filter under name ["
+                        + tokenFilterName + "]");
+            }
+            if (tokenFilter instanceof MultiTermAwareComponent == false) {
+                throw new IllegalArgumentException("Custom normalizer [" + name() + "] may not use filter [" + tokenFilterName + "]");
+            }
+            tokenFilter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenFilter).getMultiTermComponent();
+            tokenFilterList.add(tokenFilter);
+        }
+
+        this.customAnalyzer = new CustomAnalyzer(
+                PreBuiltTokenizers.KEYWORD.getTokenizerFactory(indexSettings.getIndexVersionCreated()),
+                charFiltersList.toArray(new CharFilterFactory[charFiltersList.size()]),
+                tokenFilterList.toArray(new TokenFilterFactory[tokenFilterList.size()])
+        );
+    }
+
+    @Override
+    public CustomAnalyzer get() {
+        return this.customAnalyzer;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CzechAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/CzechAnalyzerProvider.java
index f93e9e16e5e..27d20beef43 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CzechAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CzechAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class CzechAnalyzerProvider extends AbstractIndexAnalyzerProvider<CzechAn
 
     public CzechAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new CzechAnalyzer(Analysis.parseStopWords(env, settings, CzechAnalyzer.getDefaultStopSet()),
-                                     Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new CzechAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, CzechAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/DanishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/DanishAnalyzerProvider.java
index 22e4dc07d0d..897997992b2 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/DanishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/DanishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class DanishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Danish
 
     public DanishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new DanishAnalyzer(Analysis.parseStopWords(env, settings, DanishAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new DanishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, DanishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/DutchAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/DutchAnalyzerProvider.java
index c747a9a5a3d..eaa69e939cb 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/DutchAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/DutchAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class DutchAnalyzerProvider extends AbstractIndexAnalyzerProvider<DutchAn
 
     public DutchAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new DutchAnalyzer(Analysis.parseStopWords(env, settings, DutchAnalyzer.getDefaultStopSet()),
-                                     Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new DutchAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, DutchAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
index 0859824370b..401f2caf03f 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
@@ -32,7 +32,7 @@ public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory implem
 
     public ElisionTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.articles = Analysis.parseArticles(env, settings);
+        this.articles = Analysis.parseArticles(env, indexSettings.getIndexVersionCreated(), settings);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/EnglishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/EnglishAnalyzerProvider.java
index d124f27db57..952f43296ff 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/EnglishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/EnglishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class EnglishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Engli
 
     public EnglishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new EnglishAnalyzer(Analysis.parseStopWords(env, settings, EnglishAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new EnglishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, EnglishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
index 490b90efb71..fa0ce5223f8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
@@ -45,7 +45,7 @@ public class FingerprintAnalyzerProvider extends AbstractIndexAnalyzerProvider<A
 
         char separator = FingerprintTokenFilterFactory.parseSeparator(settings);
         int maxOutputSize = settings.getAsInt(MAX_OUTPUT_SIZE.getPreferredName(),DEFAULT_MAX_OUTPUT_SIZE);
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, DEFAULT_STOP_WORDS);
+        CharArraySet stopWords = Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, DEFAULT_STOP_WORDS);
 
         this.analyzer = new FingerprintAnalyzer(stopWords, separator, maxOutputSize);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FinnishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/FinnishAnalyzerProvider.java
index 5f728619f9e..b914fab66fd 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FinnishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FinnishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class FinnishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Finni
 
     public FinnishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new FinnishAnalyzer(Analysis.parseStopWords(env, settings, FinnishAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new FinnishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, FinnishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java b/core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java
similarity index 55%
rename from core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
rename to core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java
index 583ac365fad..3af472f54bc 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactory.java
@@ -16,18 +16,23 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.SearchPlugin;
+package org.elasticsearch.index.analysis;
 
-import java.util.Map;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.synonym.FlattenGraphFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
 
-import static java.util.Collections.singletonMap;
+public class FlattenGraphTokenFilterFactory extends AbstractTokenFilterFactory {
+
+    public FlattenGraphTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
+        super(indexSettings, name, settings);
+    }
 
-public class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
     @Override
-    public Map<String, Suggester<?>> getSuggesters() {
-        return singletonMap("custom", CustomSuggester.INSTANCE);
+    public TokenStream create(TokenStream tokenStream) {
+        return new FlattenGraphFilter(tokenStream);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FrenchAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/FrenchAnalyzerProvider.java
index f312161819d..96cdb8ed03e 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FrenchAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FrenchAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class FrenchAnalyzerProvider extends AbstractIndexAnalyzerProvider<French
 
     public FrenchAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new FrenchAnalyzer(Analysis.parseStopWords(env, settings, FrenchAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new FrenchAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, FrenchAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/GalicianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/GalicianAnalyzerProvider.java
index 646fa9fa681..1dc6de99a4d 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/GalicianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/GalicianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class GalicianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Gali
 
     public GalicianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new GalicianAnalyzer(Analysis.parseStopWords(env, settings, GalicianAnalyzer.getDefaultStopSet()),
-                                        Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new GalicianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, GalicianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/GermanAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/GermanAnalyzerProvider.java
index c11ac7eb9ba..52a116acca5 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/GermanAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/GermanAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class GermanAnalyzerProvider extends AbstractIndexAnalyzerProvider<German
 
     public GermanAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new GermanAnalyzer(Analysis.parseStopWords(env, settings, GermanAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new GermanAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, GermanAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/GreekAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/GreekAnalyzerProvider.java
index 625acddfadc..012fc64c971 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/GreekAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/GreekAnalyzerProvider.java
@@ -30,7 +30,8 @@ public class GreekAnalyzerProvider extends AbstractIndexAnalyzerProvider<GreekAn
 
     public GreekAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new GreekAnalyzer(Analysis.parseStopWords(env, settings, GreekAnalyzer.getDefaultStopSet()));
+        analyzer = new GreekAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, GreekAnalyzer.getDefaultStopSet()));
         analyzer.setVersion(version);
     }
 
@@ -38,4 +39,4 @@ public class GreekAnalyzerProvider extends AbstractIndexAnalyzerProvider<GreekAn
     public GreekAnalyzer get() {
         return this.analyzer;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/HindiAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/HindiAnalyzerProvider.java
index 123117abced..43ebe2677ae 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/HindiAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/HindiAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class HindiAnalyzerProvider extends AbstractIndexAnalyzerProvider<HindiAn
 
     public HindiAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new HindiAnalyzer(Analysis.parseStopWords(env, settings, HindiAnalyzer.getDefaultStopSet()),
-                                     Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new HindiAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, HindiAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java
index 99268ee7ee9..81745ba0f27 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/HungarianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class HungarianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Hun
 
     public HungarianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new HungarianAnalyzer(Analysis.parseStopWords(env, settings, HungarianAnalyzer.getDefaultStopSet()),
-                                         Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new HungarianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, HungarianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactory.java
index 357d4dc2405..d7ceee5cb95 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/HunspellTokenFilterFactory.java
@@ -33,7 +33,7 @@ public class HunspellTokenFilterFactory extends AbstractTokenFilterFactory {
     private final boolean dedup;
     private final boolean longestOnly;
 
-    public HunspellTokenFilterFactory(IndexSettings indexSettings, String name,  Settings settings, HunspellService hunspellService)  {
+    public HunspellTokenFilterFactory(IndexSettings indexSettings, String name, Settings settings, HunspellService hunspellService) {
         super(indexSettings, name, settings);
 
         String locale = settings.get("locale", settings.get("language", settings.get("lang", null)));
@@ -46,8 +46,9 @@ public class HunspellTokenFilterFactory extends AbstractTokenFilterFactory {
             throw new IllegalArgumentException(String.format(Locale.ROOT, "Unknown hunspell dictionary for locale [%s]", locale));
         }
 
-        dedup = settings.getAsBoolean("dedup", true);
-        longestOnly = settings.getAsBoolean("longest_only", false);
+        dedup = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "dedup", true, deprecationLogger);
+        longestOnly =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "longest_only", false, deprecationLogger);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java b/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
index 127714178b5..f3200d606fb 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/IndexAnalyzers.java
@@ -25,6 +25,7 @@ import org.elasticsearch.index.IndexSettings;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.stream.Stream;
 
 /**
  * IndexAnalyzers contains a name to analyzer mapping for a specific index.
@@ -38,15 +39,18 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos
     private final NamedAnalyzer defaultSearchAnalyzer;
     private final NamedAnalyzer defaultSearchQuoteAnalyzer;
     private final Map<String, NamedAnalyzer> analyzers;
+    private final Map<String, NamedAnalyzer> normalizers;
     private final IndexSettings indexSettings;
 
     public IndexAnalyzers(IndexSettings indexSettings, NamedAnalyzer defaultIndexAnalyzer, NamedAnalyzer defaultSearchAnalyzer,
-                          NamedAnalyzer defaultSearchQuoteAnalyzer, Map<String, NamedAnalyzer> analyzers) {
+                          NamedAnalyzer defaultSearchQuoteAnalyzer, Map<String, NamedAnalyzer> analyzers,
+                          Map<String, NamedAnalyzer> normalizers) {
         super(indexSettings);
         this.defaultIndexAnalyzer = defaultIndexAnalyzer;
         this.defaultSearchAnalyzer = defaultSearchAnalyzer;
         this.defaultSearchQuoteAnalyzer = defaultSearchQuoteAnalyzer;
         this.analyzers = analyzers;
+        this.normalizers = normalizers;
         this.indexSettings = indexSettings;
     }
 
@@ -57,6 +61,12 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos
         return analyzers.get(name);
     }
 
+    /**
+     * Returns a normalizer mapped to the given name or <code>null</code> if not present
+     */
+    public NamedAnalyzer getNormalizer(String name) {
+        return normalizers.get(name);
+    }
 
     /**
      * Returns the default index analyzer for this index
@@ -81,7 +91,7 @@ public final class IndexAnalyzers extends AbstractIndexComponent implements Clos
 
     @Override
     public void close() throws IOException {
-       IOUtils.close(() -> analyzers.values().stream()
+       IOUtils.close(() -> Stream.concat(analyzers.values().stream(), normalizers.values().stream())
            .filter(a -> a.scope() == AnalyzerScope.INDEX)
            .iterator());
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/IndonesianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/IndonesianAnalyzerProvider.java
index 60f40612cd0..d5633025804 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/IndonesianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/IndonesianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class IndonesianAnalyzerProvider extends AbstractIndexAnalyzerProvider<In
 
     public IndonesianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new IndonesianAnalyzer(Analysis.parseStopWords(env, settings, IndonesianAnalyzer.getDefaultStopSet()),
-                                          Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new IndonesianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, IndonesianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/IrishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/IrishAnalyzerProvider.java
index 0b27cb73a3c..69250df7360 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/IrishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/IrishAnalyzerProvider.java
@@ -34,8 +34,10 @@ public class IrishAnalyzerProvider extends AbstractIndexAnalyzerProvider<IrishAn
 
     public IrishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new IrishAnalyzer(Analysis.parseStopWords(env, settings, IrishAnalyzer.getDefaultStopSet()),
-                                     Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new IrishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, IrishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ItalianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ItalianAnalyzerProvider.java
index 38f7094bdeb..7a9ae2d7ab4 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ItalianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ItalianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class ItalianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Itali
 
     public ItalianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new ItalianAnalyzer(Analysis.parseStopWords(env, settings, ItalianAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new ItalianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, ItalianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java
index 334bbab5691..631040d2333 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java
@@ -50,7 +50,7 @@ public class KeepWordFilterFactory extends AbstractTokenFilterFactory {
     private static final String KEEP_WORDS_KEY = "keep_words";
     private static final String KEEP_WORDS_PATH_KEY = KEEP_WORDS_KEY + "_path";
     private static final String KEEP_WORDS_CASE_KEY = KEEP_WORDS_KEY + "_case"; // for javadoc
-    
+
     // unsupported ancient option
     private static final String ENABLE_POS_INC_KEY = "enable_position_increments";
 
@@ -68,7 +68,7 @@ public class KeepWordFilterFactory extends AbstractTokenFilterFactory {
         if (settings.get(ENABLE_POS_INC_KEY) != null) {
             throw new IllegalArgumentException(ENABLE_POS_INC_KEY + " is not supported anymore. Please fix your analysis chain");
         }
-        this.keepWords = Analysis.getWordSet(env, settings, KEEP_WORDS_KEY);
+        this.keepWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, KEEP_WORDS_KEY);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
index 0805a3bdf86..9cb01e75e55 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/KeywordMarkerTokenFilterFactory.java
@@ -35,8 +35,9 @@ public class KeywordMarkerTokenFilterFactory extends AbstractTokenFilterFactory
     public KeywordMarkerTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
 
-        boolean ignoreCase = settings.getAsBoolean("ignore_case", false);
-        Set<?> rules = Analysis.getWordSet(env, settings, "keywords");
+        boolean ignoreCase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "ignore_case", false, deprecationLogger);
+        Set<?> rules = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, "keywords");
         if (rules == null) {
             throw new IllegalArgumentException("keyword filter requires either `keywords` or `keywords_path` to be configured");
         }
@@ -47,4 +48,5 @@ public class KeywordMarkerTokenFilterFactory extends AbstractTokenFilterFactory
     public TokenStream create(TokenStream tokenStream) {
         return new SetKeywordMarkerFilter(tokenStream, keywordLookup);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java
index a288747ab4b..a7731f352b9 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/LatvianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class LatvianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Latvi
 
     public LatvianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new LatvianAnalyzer(Analysis.parseStopWords(env, settings, LatvianAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new LatvianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, LatvianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactory.java
index ddf60911187..459f3b94229 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/LimitTokenCountFilterFactory.java
@@ -36,11 +36,12 @@ public class LimitTokenCountFilterFactory extends AbstractTokenFilterFactory {
     public LimitTokenCountFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
         this.maxTokenCount = settings.getAsInt("max_token_count", DEFAULT_MAX_TOKEN_COUNT);
-        this.consumeAllTokens = settings.getAsBoolean("consume_all_tokens", DEFAULT_CONSUME_ALL_TOKENS);
+        this.consumeAllTokens = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), "consume_all_tokens", DEFAULT_CONSUME_ALL_TOKENS, deprecationLogger);
     }
 
     @Override
     public TokenStream create(TokenStream tokenStream) {
         return new LimitTokenCountFilter(tokenStream, maxTokenCount, consumeAllTokens);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LithuanianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/LithuanianAnalyzerProvider.java
index 7d180b3e0f2..307904b8942 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/LithuanianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/LithuanianAnalyzerProvider.java
@@ -34,8 +34,10 @@ public class LithuanianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Li
 
     public LithuanianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new LithuanianAnalyzer(Analysis.parseStopWords(env, settings, LithuanianAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new LithuanianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, LithuanianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java
index 65a7dff331f..ca7d898fb47 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/NorwegianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class NorwegianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Nor
 
     public NorwegianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new NorwegianAnalyzer(Analysis.parseStopWords(env, settings, NorwegianAnalyzer.getDefaultStopSet()),
-                                         Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new NorwegianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, NorwegianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PathHierarchyTokenizerFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PathHierarchyTokenizerFactory.java
index b9930245252..d674143d308 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PathHierarchyTokenizerFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PathHierarchyTokenizerFactory.java
@@ -56,7 +56,8 @@ public class PathHierarchyTokenizerFactory extends AbstractTokenizerFactory {
             this.replacement = replacement.charAt(0);
         }
         this.skip = settings.getAsInt("skip", PathHierarchyTokenizer.DEFAULT_SKIP);
-        this.reverse = settings.getAsBoolean("reverse", false);
+        this.reverse =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "reverse", false, deprecationLogger);
     }
 
     @Override
@@ -66,4 +67,5 @@ public class PathHierarchyTokenizerFactory extends AbstractTokenizerFactory {
         }
         return new PathHierarchyTokenizer(bufferSize, delimiter, replacement, skip);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzerProvider.java
index b3bbb872f2a..a901ad4f1ad 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternAnalyzerProvider.java
@@ -21,8 +21,6 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.core.StopAnalyzer;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -38,8 +36,9 @@ public class PatternAnalyzerProvider extends AbstractIndexAnalyzerProvider<Analy
         super(indexSettings, name, settings);
 
         final CharArraySet defaultStopwords = CharArraySet.EMPTY_SET;
-        boolean lowercase = settings.getAsBoolean("lowercase", true);
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
+        boolean lowercase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "lowercase", true, deprecationLogger);
+        CharArraySet stopWords = Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, defaultStopwords);
 
         String sPattern = settings.get("pattern", "\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/);
         if (sPattern == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternCaptureGroupTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternCaptureGroupTokenFilterFactory.java
index 006fb447368..d129f77ecb7 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternCaptureGroupTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternCaptureGroupTokenFilterFactory.java
@@ -45,7 +45,8 @@ public class PatternCaptureGroupTokenFilterFactory extends AbstractTokenFilterFa
             patterns[i] = Pattern.compile(regexes[i]);
         }
 
-        preserveOriginal = settings.getAsBoolean(PRESERVE_ORIG_KEY, true);
+        preserveOriginal = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), PRESERVE_ORIG_KEY, true, deprecationLogger);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceTokenFilterFactory.java
index a6d22b2be95..a47f0d72d81 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceTokenFilterFactory.java
@@ -43,7 +43,7 @@ public class PatternReplaceTokenFilterFactory extends AbstractTokenFilterFactory
         }
         this.pattern = Regex.compile(sPattern, settings.get("flags"));
         this.replacement = settings.get("replacement", "");
-        this.all = settings.getAsBoolean("all", true);
+        this.all = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "all", true, deprecationLogger);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PersianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/PersianAnalyzerProvider.java
index 2c5b427e51a..ed92e3e0c02 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PersianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PersianAnalyzerProvider.java
@@ -30,7 +30,8 @@ public class PersianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Persi
 
     public PersianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new PersianAnalyzer(Analysis.parseStopWords(env, settings, PersianAnalyzer.getDefaultStopSet()));
+        analyzer = new PersianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, PersianAnalyzer.getDefaultStopSet()));
         analyzer.setVersion(version);
     }
 
@@ -38,4 +39,4 @@ public class PersianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Persi
     public PersianAnalyzer get() {
         return this.analyzer;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PortugueseAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/PortugueseAnalyzerProvider.java
index 1d5f0561e24..73dc016fe69 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PortugueseAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PortugueseAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class PortugueseAnalyzerProvider extends AbstractIndexAnalyzerProvider<Po
 
     public PortugueseAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new PortugueseAnalyzer(Analysis.parseStopWords(env, settings, PortugueseAnalyzer.getDefaultStopSet()),
-                                          Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new PortugueseAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, PortugueseAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java
index 1ba780bf4cb..5eeb22f5118 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/RomanianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class RomanianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Roma
 
     public RomanianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new RomanianAnalyzer(Analysis.parseStopWords(env, settings, RomanianAnalyzer.getDefaultStopSet()),
-                                        Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new RomanianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, RomanianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/RussianAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/RussianAnalyzerProvider.java
index f7187cfb490..6e57603794c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/RussianAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/RussianAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class RussianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Russi
 
     public RussianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new RussianAnalyzer(Analysis.parseStopWords(env, settings, RussianAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new RussianAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, RussianAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
index 453d7bad896..65f1a52e39c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ShingleTokenFilterFactory.java
@@ -33,8 +33,8 @@ public class ShingleTokenFilterFactory extends AbstractTokenFilterFactory {
         super(indexSettings, name, settings);
         Integer maxShingleSize = settings.getAsInt("max_shingle_size", ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE);
         Integer minShingleSize = settings.getAsInt("min_shingle_size", ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE);
-        Boolean outputUnigrams = settings.getAsBoolean("output_unigrams", true);
-        Boolean outputUnigramsIfNoShingles = settings.getAsBoolean("output_unigrams_if_no_shingles", false);
+        Boolean outputUnigrams = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "output_unigrams", true, deprecationLogger);
+        Boolean outputUnigramsIfNoShingles = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "output_unigrams_if_no_shingles", false, deprecationLogger);
         String tokenSeparator = settings.get("token_separator", ShingleFilter.DEFAULT_TOKEN_SEPARATOR);
         String fillerToken = settings.get("filler_token", ShingleFilter.DEFAULT_FILLER_TOKEN);
         factory = new Factory("shingle", minShingleSize, maxShingleSize, outputUnigrams, outputUnigramsIfNoShingles, tokenSeparator, fillerToken);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SnowballAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/SnowballAnalyzerProvider.java
index 84f19316331..bd3201e3c8a 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SnowballAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SnowballAnalyzerProvider.java
@@ -65,7 +65,7 @@ public class SnowballAnalyzerProvider extends AbstractIndexAnalyzerProvider<Snow
 
         String language = settings.get("language", settings.get("name", "English"));
         CharArraySet defaultStopwords = DEFAULT_LANGUAGE_STOPWORDS.getOrDefault(language, CharArraySet.EMPTY_SET);
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
+        CharArraySet stopWords = Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, defaultStopwords);
 
         analyzer = new SnowballAnalyzer(language, stopWords);
         analyzer.setVersion(version);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SoraniAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/SoraniAnalyzerProvider.java
index 6b5cf153f32..d3b9fcd3f5c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SoraniAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SoraniAnalyzerProvider.java
@@ -34,8 +34,10 @@ public class SoraniAnalyzerProvider extends AbstractIndexAnalyzerProvider<Sorani
 
     public SoraniAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new SoraniAnalyzer(Analysis.parseStopWords(env, settings, SoraniAnalyzer.getDefaultStopSet()),
-                                      Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new SoraniAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, SoraniAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SpanishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/SpanishAnalyzerProvider.java
index 23f1b734469..a32d9a0f3ca 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SpanishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SpanishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class SpanishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Spani
 
     public SpanishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new SpanishAnalyzer(Analysis.parseStopWords(env, settings, SpanishAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new SpanishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, SpanishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StandardAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/StandardAnalyzerProvider.java
index a4ee6457b52..01a0dc00d58 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/StandardAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/StandardAnalyzerProvider.java
@@ -34,7 +34,7 @@ public class StandardAnalyzerProvider extends AbstractIndexAnalyzerProvider<Stan
     public StandardAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
         final CharArraySet defaultStopwords = CharArraySet.EMPTY_SET;
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
+        CharArraySet stopWords = Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, defaultStopwords);
         int maxTokenLength = settings.getAsInt("max_token_length", StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
         standardAnalyzer = new StandardAnalyzer(stopWords);
         standardAnalyzer.setVersion(version);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StandardHtmlStripAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/StandardHtmlStripAnalyzerProvider.java
index edc837293f9..065c65f1cf0 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/StandardHtmlStripAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/StandardHtmlStripAnalyzerProvider.java
@@ -33,7 +33,7 @@ public class StandardHtmlStripAnalyzerProvider extends AbstractIndexAnalyzerProv
     public StandardHtmlStripAnalyzerProvider(IndexSettings indexSettings, Environment env,  String name, Settings settings) {
         super(indexSettings, name, settings);
         final CharArraySet defaultStopwords = CharArraySet.EMPTY_SET;
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, defaultStopwords);
+        CharArraySet stopWords = Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, defaultStopwords);
         analyzer = new StandardHtmlStripAnalyzer(stopWords);
         analyzer.setVersion(version);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java
index 8c969238b47..7b7cf3e7c42 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/StopAnalyzerProvider.java
@@ -31,7 +31,8 @@ public class StopAnalyzerProvider extends AbstractIndexAnalyzerProvider<StopAnal
 
     public StopAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        CharArraySet stopWords = Analysis.parseStopWords(env, settings, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+        CharArraySet stopWords = Analysis.parseStopWords(
+            env, indexSettings.getIndexVersionCreated(), settings, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
         this.stopAnalyzer = new StopAnalyzer(stopWords);
         this.stopAnalyzer.setVersion(version);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
index 86650e81915..e6d95e6781f 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.StopAnalyzer;
 import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter;
-import org.apache.lucene.util.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -41,8 +40,10 @@ public class StopTokenFilterFactory extends AbstractTokenFilterFactory {
 
     public StopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
-        this.removeTrailing = settings.getAsBoolean("remove_trailing", true);
+        this.ignoreCase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "ignore_case", false, deprecationLogger);
+        this.removeTrailing = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), "remove_trailing", true, deprecationLogger);
         this.stopWords = Analysis.parseStopWords(env, settings, StopAnalyzer.ENGLISH_STOP_WORDS_SET, ignoreCase);
         if (settings.get("enable_position_increments") != null) {
             throw new IllegalArgumentException("enable_position_increments is not supported anymore. Please fix your analysis chain");
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SwedishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/SwedishAnalyzerProvider.java
index b77e19efcaa..066d2eef78b 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SwedishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SwedishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class SwedishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Swedi
 
     public SwedishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new SwedishAnalyzer(Analysis.parseStopWords(env, settings, SwedishAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new SwedishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, SwedishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphFilterFactory.java
rename to core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
index da9b11b9785..cfb37f0b075 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SynonymGraphTokenFilterFactory.java
@@ -27,8 +27,8 @@ import org.elasticsearch.index.IndexSettings;
 
 import java.io.IOException;
 
-public class SynonymGraphFilterFactory extends SynonymTokenFilterFactory {
-    public SynonymGraphFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
+public class SynonymGraphTokenFilterFactory extends SynonymTokenFilterFactory {
+    public SynonymGraphTokenFilterFactory(IndexSettings indexSettings, Environment env, AnalysisRegistry analysisRegistry,
                                      String name, Settings settings) throws IOException {
         super(indexSettings, env, analysisRegistry, name, settings);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
index d32c66e0dfe..0e23089827c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java
@@ -61,8 +61,10 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
             throw new IllegalArgumentException("synonym requires either `synonyms` or `synonyms_path` to be configured");
         }
 
-        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
-        boolean expand = settings.getAsBoolean("expand", true);
+        this.ignoreCase =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "ignore_case", false, deprecationLogger);
+        boolean expand =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "expand", true, deprecationLogger);
 
         String tokenizerName = settings.get("tokenizer", "whitespace");
         AnalysisModule.AnalysisProvider<TokenizerFactory> tokenizerFactoryFactory =
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java
index f1a69c62b1e..119eb81d748 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ThaiAnalyzerProvider.java
@@ -30,7 +30,8 @@ public class ThaiAnalyzerProvider extends AbstractIndexAnalyzerProvider<ThaiAnal
 
     public ThaiAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new ThaiAnalyzer(Analysis.parseStopWords(env, settings, ThaiAnalyzer.getDefaultStopSet()));
+        analyzer = new ThaiAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, ThaiAnalyzer.getDefaultStopSet()));
         analyzer.setVersion(version);
     }
 
@@ -38,4 +39,4 @@ public class ThaiAnalyzerProvider extends AbstractIndexAnalyzerProvider<ThaiAnal
     public ThaiAnalyzer get() {
         return this.analyzer;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/TurkishAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/TurkishAnalyzerProvider.java
index 2b2696a362b..0eaf49afdc8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/TurkishAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/TurkishAnalyzerProvider.java
@@ -31,8 +31,10 @@ public class TurkishAnalyzerProvider extends AbstractIndexAnalyzerProvider<Turki
 
     public TurkishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new TurkishAnalyzer(Analysis.parseStopWords(env, settings, TurkishAnalyzer.getDefaultStopSet()),
-                                       Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new TurkishAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, TurkishAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/UniqueTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/UniqueTokenFilterFactory.java
index 986153a4577..8606a60292c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/UniqueTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/UniqueTokenFilterFactory.java
@@ -31,7 +31,8 @@ public class UniqueTokenFilterFactory extends AbstractTokenFilterFactory {
 
     public UniqueTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.onlyOnSamePosition = settings.getAsBoolean("only_on_same_position", false);
+        this.onlyOnSamePosition = settings.getAsBooleanLenientForPreEs6Indices(
+            indexSettings.getIndexVersionCreated(), "only_on_same_position", false, deprecationLogger);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
index ccc60f4ce70..387c2c9e7f1 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/WordDelimiterTokenFilterFactory.java
@@ -87,7 +87,7 @@ public class WordDelimiterTokenFilterFactory extends AbstractTokenFilterFactory
         // If set, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil"
         flags |= getFlag(STEM_ENGLISH_POSSESSIVE, settings, "stem_english_possessive", true);
         // If not null is the set of tokens to protect from being delimited
-        Set<?> protectedWords = Analysis.getWordSet(env, settings, "protected_words");
+        Set<?> protectedWords = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, "protected_words");
         this.protoWords = protectedWords == null ? null : CharArraySet.copy(protectedWords);
         this.flags = flags;
     }
@@ -101,7 +101,7 @@ public class WordDelimiterTokenFilterFactory extends AbstractTokenFilterFactory
     }
 
     public int getFlag(int flag, Settings settings, String key, boolean defaultValue) {
-        if (settings.getAsBoolean(key, defaultValue)) {
+        if (settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), key, defaultValue, deprecationLogger)) {
             return flag;
         }
         return 0;
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java
index e7147334a7e..91c984c7a6b 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/compound/AbstractCompoundWordTokenFilterFactory.java
@@ -44,8 +44,9 @@ public abstract class AbstractCompoundWordTokenFilterFactory extends AbstractTok
         minWordSize = settings.getAsInt("min_word_size", CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE);
         minSubwordSize = settings.getAsInt("min_subword_size", CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE);
         maxSubwordSize = settings.getAsInt("max_subword_size", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);
-        onlyLongestMatch = settings.getAsBoolean("only_longest_match", false);
-        wordList = Analysis.getWordSet(env, settings, "word_list");
+        onlyLongestMatch = settings
+            .getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "only_longest_match", false, deprecationLogger);
+        wordList = Analysis.getWordSet(env, indexSettings.getIndexVersionCreated(), settings, "word_list");
         if (wordList == null) {
             throw new IllegalArgumentException("word_list must be provided for [" + name + "], either as a path to a file, or directly");
         }
diff --git a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
index 318059d6a3f..7a033acff9b 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/query/QueryCache.java
@@ -25,7 +25,7 @@ import java.io.Closeable;
 
 public interface QueryCache extends IndexComponent, Closeable, org.apache.lucene.search.QueryCache {
 
-    static class EntriesStats {
+    class EntriesStats {
         public final long sizeInBytes;
         public final long count;
 
diff --git a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
index b145a86e43d..dbfb2416bae 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/DeleteVersionValue.java
@@ -29,7 +29,7 @@ class DeleteVersionValue extends VersionValue {
 
     private final long time;
 
-    public DeleteVersionValue(long version, long time) {
+    DeleteVersionValue(long version, long time) {
         super(version);
         this.time = time;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
index 466da06dec4..11cc838660f 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ElasticsearchConcurrentMergeScheduler.java
@@ -67,7 +67,7 @@ class ElasticsearchConcurrentMergeScheduler extends ConcurrentMergeScheduler {
     private final Set<OnGoingMerge> readOnlyOnGoingMerges = Collections.unmodifiableSet(onGoingMerges);
     private final MergeSchedulerConfig config;
 
-    public ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings) {
+    ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings) {
         this.config = indexSettings.getMergeSchedulerConfig();
         this.shardId = shardId;
         this.indexSettings = indexSettings.getSettings();
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 0f1c05e59ee..03ea99b1bd5 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -47,7 +47,6 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -65,7 +64,6 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -284,7 +282,7 @@ public abstract class Engine implements Closeable {
      *
      * Note: engine level failures (i.e. persistent engine failures) are thrown
      */
-    public abstract IndexResult index(final Index index);
+    public abstract IndexResult index(Index index) throws IOException;
 
     /**
      * Perform document delete operation on the engine
@@ -294,7 +292,9 @@ public abstract class Engine implements Closeable {
      *
      * Note: engine level failures (i.e. persistent engine failures) are thrown
      */
-    public abstract DeleteResult delete(final Delete delete);
+    public abstract DeleteResult delete(Delete delete) throws IOException;
+
+    public abstract NoOpResult noOp(NoOp noOp);
 
     /**
      * Base class for index and delete operation results
@@ -379,9 +379,11 @@ public abstract class Engine implements Closeable {
         void freeze() {
             freeze.set(true);
         }
+
     }
 
     public static class IndexResult extends Result {
+
         private final boolean created;
 
         public IndexResult(long version, long seqNo, boolean created) {
@@ -397,9 +399,11 @@ public abstract class Engine implements Closeable {
         public boolean isCreated() {
             return created;
         }
+
     }
 
     public static class DeleteResult extends Result {
+
         private final boolean found;
 
         public DeleteResult(long version, long seqNo, boolean found) {
@@ -415,6 +419,19 @@ public abstract class Engine implements Closeable {
         public boolean isFound() {
             return found;
         }
+
+    }
+
+    static class NoOpResult extends Result {
+
+        NoOpResult(long seqNo) {
+            super(Operation.TYPE.NO_OP, 0, seqNo);
+        }
+
+        NoOpResult(long seqNo, Exception failure) {
+            super(Operation.TYPE.NO_OP, failure, 0, seqNo);
+        }
+
     }
 
     /**
@@ -496,7 +513,7 @@ public abstract class Engine implements Closeable {
                     manager.release(searcher);
                 }
             }
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             throw ex;
         } catch (Exception ex) {
             ensureOpen(); // throw EngineCloseException here if we are already closed
@@ -514,7 +531,7 @@ public abstract class Engine implements Closeable {
 
     protected void ensureOpen() {
         if (isClosed.get()) {
-            throw new EngineClosedException(shardId, failedEngine.get());
+            throw new AlreadyClosedException(shardId + " engine is closed", failedEngine.get());
         }
     }
 
@@ -552,7 +569,7 @@ public abstract class Engine implements Closeable {
      */
     public final SegmentsStats segmentsStats(boolean includeSegmentFileSizes) {
         ensureOpen();
-        try (final Searcher searcher = acquireSearcher("segments_stats")) {
+        try (Searcher searcher = acquireSearcher("segments_stats")) {
             SegmentsStats stats = new SegmentsStats();
             for (LeafReaderContext reader : searcher.reader().leaves()) {
                 final SegmentReader segmentReader = segmentReader(reader.reader());
@@ -910,7 +927,7 @@ public abstract class Engine implements Closeable {
 
         /** type of operation (index, delete), subclasses use static types */
         public enum TYPE {
-            INDEX, DELETE;
+            INDEX, DELETE, NO_OP;
 
             private final String lowercase;
 
@@ -1001,6 +1018,7 @@ public abstract class Engine implements Closeable {
         public Index(Term uid, ParsedDocument doc, long seqNo, long primaryTerm, long version, VersionType versionType, Origin origin,
                      long startTime, long autoGeneratedIdTimestamp, boolean isRetry) {
             super(uid, seqNo, primaryTerm, version, versionType, origin, startTime);
+            assert uid.bytes().equals(doc.uid()) : "term uid " + uid + " doesn't match doc uid " + doc.uid();
             this.doc = doc;
             this.isRetry = isRetry;
             this.autoGeneratedIdTimestamp = autoGeneratedIdTimestamp;
@@ -1114,6 +1132,50 @@ public abstract class Engine implements Closeable {
         public int estimatedSizeInBytes() {
             return (uid().field().length() + uid().text().length()) * 2 + 20;
         }
+
+    }
+
+    public static class NoOp extends Operation {
+
+        private final String reason;
+
+        public String reason() {
+            return reason;
+        }
+
+        public NoOp(
+            final Term uid,
+            final long seqNo,
+            final long primaryTerm,
+            final long version,
+            final VersionType versionType,
+            final Origin origin,
+            final long startTime,
+            final String reason) {
+            super(uid, seqNo, primaryTerm, version, versionType, origin, startTime);
+            this.reason = reason;
+        }
+
+        @Override
+        public String type() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        String id() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        TYPE operationType() {
+            return TYPE.NO_OP;
+        }
+
+        @Override
+        public int estimatedSizeInBytes() {
+            return 2 * reason.length() + 2 * Long.BYTES;
+        }
+
     }
 
     public static class Get {
@@ -1222,7 +1284,7 @@ public abstract class Engine implements Closeable {
                     logger.debug("flushing shard on close - this might take some time to sync files to disk");
                     try {
                         flush(); // TODO we might force a flush in the future since we have the write lock already even though recoveries are running.
-                    } catch (EngineClosedException ex) {
+                    } catch (AlreadyClosedException ex) {
                         logger.debug("engine already closed - skipping flushAndClose");
                     }
                 } finally {
@@ -1313,16 +1375,6 @@ public abstract class Engine implements Closeable {
         return this.lastWriteNanos;
     }
 
-    /**
-     * Returns the engines current document statistics
-     */
-    public DocsStats getDocStats() {
-        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
-            IndexReader reader = searcher.reader();
-            return new DocsStats(reader.numDocs(), reader.numDeletedDocs());
-        }
-    }
-
     /**
      * Called for each new opened engine searcher to warm new segments
      *
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 2957b9ab064..8b9cc8564d1 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -63,8 +63,8 @@ import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.merge.OnGoingMerge;
 import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
@@ -120,8 +120,6 @@ public class InternalEngine extends Engine {
     private final IndexThrottle throttle;
 
     private final SequenceNumbersService seqNoService;
-    static final String LOCAL_CHECKPOINT_KEY = "local_checkpoint";
-    static final String MAX_SEQ_NO = "max_seq_no";
 
     // How many callers are currently requesting index throttling.  Currently there are only two situations where we do this: when merges
     // are falling behind and when writing indexing buffer to disk is too slow.  When this is 0, there is no throttling, else we throttling
@@ -160,11 +158,12 @@ public class InternalEngine extends Engine {
                 switch (openMode) {
                     case OPEN_INDEX_AND_TRANSLOG:
                         writer = createWriter(false);
-                        seqNoStats = loadSeqNoStatsFromLuceneAndTranslog(engineConfig.getTranslogConfig(), writer);
+                        final long globalCheckpoint = Translog.readGlobalCheckpoint(engineConfig.getTranslogConfig().getTranslogPath());
+                        seqNoStats = store.loadSeqNoStats(globalCheckpoint);
                         break;
                     case OPEN_INDEX_CREATE_TRANSLOG:
                         writer = createWriter(false);
-                        seqNoStats = loadSeqNoStatsFromLucene(SequenceNumbersService.UNASSIGNED_SEQ_NO, writer);
+                        seqNoStats = store.loadSeqNoStats(SequenceNumbersService.UNASSIGNED_SEQ_NO);
                         break;
                     case CREATE_INDEX_AND_TRANSLOG:
                         writer = createWriter(true);
@@ -277,7 +276,7 @@ public class InternalEngine extends Engine {
         }
         // flush if we recovered something or if we have references to older translogs
         // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0 length.
-        assert pendingTranslogRecovery.get(): "translogRecovery is not pending but should be";
+        assert pendingTranslogRecovery.get() : "translogRecovery is not pending but should be";
         pendingTranslogRecovery.set(false); // we are good - now we can commit
         if (opsRecovered > 0) {
             logger.trace("flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]",
@@ -354,47 +353,6 @@ public class InternalEngine extends Engine {
         return null;
     }
 
-    /**
-     * Reads the sequence number stats from the Lucene commit point (maximum sequence number and local checkpoint) and the translog
-     * checkpoint (global checkpoint).
-     *
-     * @param translogConfig the translog config (for the global checkpoint)
-     * @param indexWriter    the index writer (for the Lucene commit point)
-     * @return the sequence number stats
-     * @throws IOException if an I/O exception occurred reading the Lucene commit point or the translog checkpoint
-     */
-    private static SeqNoStats loadSeqNoStatsFromLuceneAndTranslog(
-        final TranslogConfig translogConfig,
-        final IndexWriter indexWriter) throws IOException {
-        long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath());
-        return loadSeqNoStatsFromLucene(globalCheckpoint, indexWriter);
-    }
-
-    /**
-     * Reads the sequence number stats from the Lucene commit point (maximum sequence number and local checkpoint) and uses the
-     * specified global checkpoint.
-     *
-     * @param globalCheckpoint the global checkpoint to use
-     * @param indexWriter the index writer (for the Lucene commit point)
-     * @return the sequence number stats
-     */
-    private static SeqNoStats loadSeqNoStatsFromLucene(final long globalCheckpoint, final IndexWriter indexWriter) {
-        long maxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
-        long localCheckpoint = SequenceNumbersService.NO_OPS_PERFORMED;
-        for (Map.Entry<String, String> entry : indexWriter.getLiveCommitData()) {
-            final String key = entry.getKey();
-            if (key.equals(LOCAL_CHECKPOINT_KEY)) {
-                assert localCheckpoint == SequenceNumbersService.NO_OPS_PERFORMED;
-                localCheckpoint = Long.parseLong(entry.getValue());
-            } else if (key.equals(MAX_SEQ_NO)) {
-                assert maxSeqNo == SequenceNumbersService.NO_OPS_PERFORMED : localCheckpoint;
-                maxSeqNo = Long.parseLong(entry.getValue());
-            }
-        }
-
-        return new SeqNoStats(maxSeqNo, localCheckpoint, globalCheckpoint);
-    }
-
     private SearcherManager createSearcherManager() throws EngineException {
         boolean success = false;
         SearcherManager searcherManager = null;
@@ -434,7 +392,7 @@ public class InternalEngine extends Engine {
                     if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) {
                         Uid uid = Uid.createUid(get.uid().text());
                         throw new VersionConflictEngineException(shardId, uid.type(), uid.id(),
-                                get.versionType().explainConflictForReads(versionValue.version(), get.version()));
+                            get.versionType().explainConflictForReads(versionValue.version(), get.version()));
                     }
                     refresh("realtime_get");
                 }
@@ -446,27 +404,18 @@ public class InternalEngine extends Engine {
     }
 
     /**
-     * Checks for version conflicts. If a version conflict exists, the optional return value represents the operation result. Otherwise, if
-     * no conflicts are found, the optional return value is not present.
+     * Checks for version conflicts. If a non-critical version conflict exists <code>true</code> is returned. In the case of a critical
+     * version conflict (if operation origin is primary) a {@link VersionConflictEngineException} is thrown.
      *
-     * @param <T>             the result type
      * @param op              the operation
      * @param currentVersion  the current version
      * @param expectedVersion the expected version
      * @param deleted         {@code true} if the current version is not found or represents a delete
-     * @param onSuccess       if there is a version conflict that can be ignored, the result of the operation
-     * @param onFailure       if there is a version conflict that can not be ignored, the result of the operation
-     * @return if there is a version conflict, the optional value is present and represents the operation result, otherwise the return value
-     * is not present
+     * @return <code>true</code> iff a non-critical version conflict (origin recovery or replica) is found otherwise <code>false</code>
+     * @throws VersionConflictEngineException if a critical version conflict was found where the operation origin is primary
+     * @throws IllegalArgumentException if an unsupported version type is used.
      */
-    private <T extends Result> Optional<T> checkVersionConflict(
-        final Operation op,
-        final long currentVersion,
-        final long expectedVersion,
-        final boolean deleted,
-        final Supplier<T> onSuccess,
-        final Function<VersionConflictEngineException, T> onFailure) {
-        final T result;
+    private boolean checkVersionConflict(final Operation op, final long currentVersion, final long expectedVersion, final boolean deleted) {
         if (op.versionType() == VersionType.FORCE) {
             if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
                 // If index was created in 5.0 or later, 'force' is not allowed at all
@@ -478,23 +427,21 @@ public class InternalEngine extends Engine {
         }
 
         if (op.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
-            if (op.origin().isRecovery()) {
-                // version conflict, but okay
-                result = onSuccess.get();
-            } else {
+            if (op.origin() == Operation.Origin.PRIMARY) {
                 // fatal version conflict
-                final VersionConflictEngineException e =
-                    new VersionConflictEngineException(
+                throw new VersionConflictEngineException(
                         shardId,
                         op.type(),
                         op.id(),
                         op.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
-                result = onFailure.apply(e);
-            }
 
-            return Optional.of(result);
+            } else {
+                /* Version conflicts during recovery and on replicas are normal due to asynchronous execution; as such, we should return a
+                 * successful result.*/
+                return true;
+            }
         } else {
-            return Optional.empty();
+            return false;
         }
     }
 
@@ -508,63 +455,6 @@ public class InternalEngine extends Engine {
         return currentVersion;
     }
 
-    @Override
-    public IndexResult index(Index index) {
-        IndexResult result;
-        try (ReleasableLock ignored = readLock.acquire()) {
-            ensureOpen();
-            if (index.origin().isRecovery()) {
-                // Don't throttle recovery operations
-                result = innerIndex(index);
-            } else {
-                try (Releasable r = throttle.acquireThrottle()) {
-                    result = innerIndex(index);
-                }
-            }
-        } catch (Exception e) {
-            result = new IndexResult(checkIfDocumentFailureOrThrow(index, e), index.version(), SequenceNumbersService.UNASSIGNED_SEQ_NO);
-        }
-        return result;
-    }
-
-    /**
-     * Inspects exception thrown when executing index or delete operations
-     *
-     * @return failure if the failure is a document specific failure (e.g. analysis chain failure)
-     * or throws Exception if the failure caused the engine to fail (e.g. out of disk, lucene tragic event)
-     *
-     * Note: pkg-private for testing
-     */
-    final Exception checkIfDocumentFailureOrThrow(final Operation operation, final Exception failure) {
-        boolean isDocumentFailure;
-        try {
-            // When indexing a document into Lucene, Lucene distinguishes between environment related errors
-            // (like out of disk space) and document specific errors (like analysis chain problems) by setting
-            // the IndexWriter.getTragicEvent() value for the former. maybeFailEngine checks for these kind of
-            // errors and returns true if that is the case. We use that to indicate a document level failure
-            // and set the error in operation.setFailure. In case of environment related errors, the failure
-            // is bubbled up
-            isDocumentFailure = maybeFailEngine(operation.operationType().getLowercase(), failure) == false;
-        } catch (Exception inner) {
-            // we failed checking whether the failure can fail the engine, treat it as a persistent engine failure
-            isDocumentFailure = false;
-            failure.addSuppressed(inner);
-        }
-        if (isDocumentFailure) {
-            return failure;
-        } else {
-            // throw original exception in case the exception caused the engine to fail
-            rethrow(failure);
-            return null;
-        }
-    }
-
-    // hack to rethrow original exception in case of engine level failures during index/delete operation
-    @SuppressWarnings("unchecked")
-    private static <T extends Throwable> void rethrow(Throwable t) throws T {
-        throw (T) t;
-    }
-
     private boolean canOptimizeAddDocument(Index index) {
         if (index.getAutoGeneratedIdTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
             assert index.getAutoGeneratedIdTimestamp() >= 0 : "autoGeneratedIdTimestamp must be positive but was: "
@@ -577,7 +467,7 @@ public class InternalEngine extends Engine {
                 case PEER_RECOVERY:
                 case REPLICA:
                     assert index.version() == 1 && index.versionType() == VersionType.EXTERNAL
-                    : "version: " + index.version() + " type: " + index.versionType();
+                        : "version: " + index.version() + " type: " + index.versionType();
                     return true;
                 case LOCAL_TRANSLOG_RECOVERY:
                     assert index.isRetry();
@@ -592,138 +482,175 @@ public class InternalEngine extends Engine {
     private boolean assertSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) {
         if (engineConfig.getIndexSettings().getIndexVersionCreated().before(Version.V_6_0_0_alpha1_UNRELEASED) && origin == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
             // legacy support
-            assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "old op recovering but it already has a seq no." +
-                " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated() + ". seq no: " + seqNo;
+            assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "old op recovering but it already has a seq no.;" +
+                " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated() + ", seqNo: " + seqNo;
         } else if (origin == Operation.Origin.PRIMARY) {
             // sequence number should not be set when operation origin is primary
-            assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "primary ops should never an assigned seq no. got: " + seqNo;
-        } else {
+            assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "primary ops should never have an assigned seq no.; seqNo: " + seqNo;
+        } else if (engineConfig.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
             // sequence number should be set when operation origin is not primary
-            assert seqNo >= 0 : "replica ops should an assigned seq no. origin: " + origin +
-                " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated();
+            assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin;
         }
         return true;
     }
 
-    private IndexResult innerIndex(Index index) throws IOException {
-        assert assertSequenceNumber(index.origin(), index.seqNo());
-        final Translog.Location location;
-        final long updatedVersion;
-        long seqNo = index.seqNo();
-        try (Releasable ignored = acquireLock(index.uid())) {
-            lastWriteNanos = index.startTime();
-            /* if we have an autoGeneratedID that comes into the engine we can potentially optimize
-             * and just use addDocument instead of updateDocument and skip the entire version and index lookup across the board.
-             * Yet, we have to deal with multiple document delivery, for this we use a property of the document that is added
-             * to detect if it has potentially been added before. We use the documents timestamp for this since it's something
-             * that:
-             *  - doesn't change per document
-             *  - is preserved in the transaction log
-             *  - and is assigned before we start to index / replicate
-             * NOTE: it's not important for this timestamp to be consistent across nodes etc. it's just a number that is in the common
-             * case increasing and can be used in the failure case when we retry and resent documents to establish a happens before relationship.
-             * for instance:
-             *  - doc A has autoGeneratedIdTimestamp = 10, isRetry = false
-             *  - doc B has autoGeneratedIdTimestamp = 9, isRetry = false
-             *
-             *  while both docs are in in flight, we disconnect on one node, reconnect and send doc A again
-             *  - now doc A' has autoGeneratedIdTimestamp = 10, isRetry = true
-             *
-             *  if A' arrives on the shard first we update maxUnsafeAutoIdTimestamp to 10 and use update document. All subsequent
-             *  documents that arrive (A and B) will also use updateDocument since their timestamps are less than maxUnsafeAutoIdTimestamp.
-             *  While this is not strictly needed for doc B it is just much simpler to implement since it will just de-optimize some doc in the worst case.
-             *
-             *  if A arrives on the shard first we use addDocument since maxUnsafeAutoIdTimestamp is < 10. A` will then just be skipped or calls
-             *  updateDocument.
-             */
-            long currentVersion;
-            final boolean deleted;
-            // if anything is fishy here ie. there is a retry we go and force updateDocument below so we are updating the document in the
-            // lucene index without checking the version map but we still do the version check
-            final boolean forceUpdateDocument;
-            final boolean canOptimizeAddDocument = canOptimizeAddDocument(index);
-            if (canOptimizeAddDocument) {
-                long deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
-                if (index.isRetry()) {
-                    forceUpdateDocument = true;
-                    do {
-                        deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
-                        if (deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp()) {
-                            break;
-                        }
-                    } while(maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp,
-                        index.getAutoGeneratedIdTimestamp()) == false);
-                    assert maxUnsafeAutoIdTimestamp.get() >= index.getAutoGeneratedIdTimestamp();
-                } else {
-                    // in this case we force
-                    forceUpdateDocument = deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp();
-                }
-                currentVersion = Versions.NOT_FOUND;
-                deleted = true;
-            } else {
-                // update the document
-                forceUpdateDocument = false; // we don't force it - it depends on the version
-                final VersionValue versionValue = versionMap.getUnderLock(index.uid());
-                assert incrementVersionLookup();
-                if (versionValue == null) {
-                    currentVersion = loadCurrentVersionFromIndex(index.uid());
-                    deleted = currentVersion == Versions.NOT_FOUND;
-                } else {
-                    currentVersion = checkDeletedAndGCed(versionValue);
-                    deleted = versionValue.delete();
-                }
-            }
-            final long expectedVersion = index.version();
-            final Optional<IndexResult> checkVersionConflictResult =
-                checkVersionConflict(
-                    index,
-                    currentVersion,
-                    expectedVersion,
-                    deleted,
-                    () -> new IndexResult(currentVersion, index.seqNo(), false),
-                    e -> new IndexResult(e, currentVersion, index.seqNo()));
-
-            final IndexResult indexResult;
-            if (checkVersionConflictResult.isPresent()) {
-                indexResult = checkVersionConflictResult.get();
-            } else {
-                // no version conflict
-                if (index.origin() == Operation.Origin.PRIMARY) {
-                    seqNo = seqNoService().generateSeqNo();
-                }
-
-                /**
-                 * Update the document's sequence number and primary term; the sequence number here is derived here from either the sequence
-                 * number service if this is on the primary, or the existing document's sequence number if this is on the replica. The
-                 * primary term here has already been set, see IndexShard#prepareIndex where the Engine$Index operation is created.
+    @Override
+    public IndexResult index(Index index) throws IOException {
+        final boolean doThrottle = index.origin().isRecovery() == false;
+        try (ReleasableLock releasableLock = readLock.acquire()) {
+            ensureOpen();
+            assert assertSequenceNumber(index.origin(), index.seqNo());
+            final Translog.Location location;
+            long seqNo = index.seqNo();
+            try (Releasable ignored = acquireLock(index.uid());
+                Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) {
+                lastWriteNanos = index.startTime();
+                /* if we have an autoGeneratedID that comes into the engine we can potentially optimize
+                 * and just use addDocument instead of updateDocument and skip the entire version and index lookup across the board.
+                 * Yet, we have to deal with multiple document delivery, for this we use a property of the document that is added
+                 * to detect if it has potentially been added before. We use the documents timestamp for this since it's something
+                 * that:
+                 *  - doesn't change per document
+                 *  - is preserved in the transaction log
+                 *  - and is assigned before we start to index / replicate
+                 * NOTE: it's not important for this timestamp to be consistent across nodes etc. it's just a number that is in the common
+                 * case increasing and can be used in the failure case when we retry and resent documents to establish a happens before relationship.
+                 * for instance:
+                 *  - doc A has autoGeneratedIdTimestamp = 10, isRetry = false
+                 *  - doc B has autoGeneratedIdTimestamp = 9, isRetry = false
+                 *
+                 *  while both docs are in in flight, we disconnect on one node, reconnect and send doc A again
+                 *  - now doc A' has autoGeneratedIdTimestamp = 10, isRetry = true
+                 *
+                 *  if A' arrives on the shard first we update maxUnsafeAutoIdTimestamp to 10 and use update document. All subsequent
+                 *  documents that arrive (A and B) will also use updateDocument since their timestamps are less than maxUnsafeAutoIdTimestamp.
+                 *  While this is not strictly needed for doc B it is just much simpler to implement since it will just de-optimize some doc in the worst case.
+                 *
+                 *  if A arrives on the shard first we use addDocument since maxUnsafeAutoIdTimestamp is < 10. A` will then just be skipped or calls
+                 *  updateDocument.
                  */
-                index.parsedDoc().updateSeqID(seqNo, index.primaryTerm());
-                updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion);
-                index.parsedDoc().version().setLongValue(updatedVersion);
-
-                if (currentVersion == Versions.NOT_FOUND && forceUpdateDocument == false) {
-                    // document does not exists, we can optimize for create, but double check if assertions are running
-                    assert assertDocDoesNotExist(index, canOptimizeAddDocument == false);
-                    index(index.docs(), indexWriter);
+                long currentVersion;
+                final boolean deleted;
+                // if anything is fishy here ie. there is a retry we go and force updateDocument below so we are updating the document in the
+                // lucene index without checking the version map but we still do the version check
+                final boolean forceUpdateDocument;
+                final boolean canOptimizeAddDocument = canOptimizeAddDocument(index);
+                if (canOptimizeAddDocument) {
+                    forceUpdateDocument = isForceUpdateDocument(index);
+                    currentVersion = Versions.NOT_FOUND;
+                    deleted = true;
                 } else {
-                    update(index.uid(), index.docs(), indexWriter);
+                    // update the document
+                    forceUpdateDocument = false; // we don't force it - it depends on the version
+                    final VersionValue versionValue = versionMap.getUnderLock(index.uid());
+                    assert incrementVersionLookup();
+                    if (versionValue == null) {
+                        currentVersion = loadCurrentVersionFromIndex(index.uid());
+                        deleted = currentVersion == Versions.NOT_FOUND;
+                    } else {
+                        currentVersion = checkDeletedAndGCed(versionValue);
+                        deleted = versionValue.delete();
+                    }
+                }
+                final long expectedVersion = index.version();
+                Optional<IndexResult> resultOnVersionConflict;
+                try {
+                    final boolean isVersionConflict = checkVersionConflict(index, currentVersion, expectedVersion, deleted);
+                    resultOnVersionConflict = isVersionConflict ? Optional.of(new IndexResult(currentVersion, index.seqNo(), false))
+                        : Optional.empty();
+                } catch (IllegalArgumentException | VersionConflictEngineException ex) {
+                    resultOnVersionConflict = Optional.of(new IndexResult(ex, currentVersion, index.seqNo()));
+                }
+
+                final IndexResult indexResult;
+                if (resultOnVersionConflict.isPresent()) {
+                    indexResult = resultOnVersionConflict.get();
+                } else {
+                    // no version conflict
+                    if (index.origin() == Operation.Origin.PRIMARY) {
+                        seqNo = seqNoService().generateSeqNo();
+                    }
+                    indexResult = indexIntoLucene(index, seqNo, currentVersion, deleted, forceUpdateDocument, canOptimizeAddDocument, expectedVersion);
+                }
+                if (indexResult.hasFailure() == false) {
+                    location = index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY
+                        ? translog.add(new Translog.Index(index, indexResult))
+                        : null;
+                    indexResult.setTranslogLocation(location);
+                }
+                indexResult.setTook(System.nanoTime() - index.startTime());
+                indexResult.freeze();
+                return indexResult;
+            } finally {
+                if (seqNo != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
+                    seqNoService().markSeqNoAsCompleted(seqNo);
                 }
-                indexResult = new IndexResult(updatedVersion, seqNo, deleted);
-                location = index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY
-                    ? translog.add(new Translog.Index(index, indexResult))
-                    : null;
-                versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion));
-                indexResult.setTranslogLocation(location);
             }
-            indexResult.setTook(System.nanoTime() - index.startTime());
-            indexResult.freeze();
-            return indexResult;
-        } finally {
-            if (seqNo != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
-                seqNoService().markSeqNoAsCompleted(seqNo);
+        } catch (RuntimeException | IOException e) {
+            try {
+                maybeFailEngine("index", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw e;
+        }
+    }
+
+    private IndexResult indexIntoLucene(Index index, long seqNo, long currentVersion, boolean deleted, boolean forceUpdateDocument, boolean canOptimizeAddDocument, long expectedVersion) throws IOException {
+        /* Update the document's sequence number and primary term; the sequence number here is derived here from either the sequence
+         * number service if this is on the primary, or the existing document's sequence number if this is on the replica. The
+         * primary term here has already been set, see IndexShard#prepareIndex where the Engine$Index operation is created.
+         */
+        index.parsedDoc().updateSeqID(seqNo, index.primaryTerm());
+        final long updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion);
+        index.parsedDoc().version().setLongValue(updatedVersion);
+        try {
+            if (currentVersion == Versions.NOT_FOUND && forceUpdateDocument == false) {
+                // document does not exists, we can optimize for create, but double check if assertions are running
+                assert assertDocDoesNotExist(index, canOptimizeAddDocument == false);
+                index(index.docs(), indexWriter);
+            } else {
+                update(index.uid(), index.docs(), indexWriter);
+            }
+            versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion));
+            return new IndexResult(updatedVersion, seqNo, deleted);
+        } catch (Exception ex) {
+            if (indexWriter.getTragicException() == null) {
+                /* There is no tragic event recorded so this must be a document failure.
+                 *
+                 * The handling inside IW doesn't guarantee that an tragic / aborting exception
+                 * will be used as THE tragicEventException since if there are multiple exceptions causing an abort in IW
+                 * only one wins. Yet, only the one that wins will also close the IW and in turn fail the engine such that
+                 * we can potentially handle the exception before the engine is failed.
+                 * Bottom line is that we can only rely on the fact that if it's a document failure then
+                 * `indexWriter.getTragicException()` will be null otherwise we have to rethrow and treat it as fatal or rather
+                 * non-document failure
+                 */
+                return new IndexResult(ex, currentVersion, index.seqNo());
+            } else {
+                throw ex;
             }
         }
+    }
 
+    private boolean isForceUpdateDocument(Index index) {
+        boolean forceUpdateDocument;
+        long deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
+        if (index.isRetry()) {
+            forceUpdateDocument = true;
+            do {
+                deOptimizeTimestamp = maxUnsafeAutoIdTimestamp.get();
+                if (deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp()) {
+                    break;
+                }
+            } while (maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp,
+                index.getAutoGeneratedIdTimestamp()) == false);
+            assert maxUnsafeAutoIdTimestamp.get() >= index.getAutoGeneratedIdTimestamp();
+        } else {
+            // in this case we force
+            forceUpdateDocument = deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp();
+        }
+        return forceUpdateDocument;
     }
 
     private static void index(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
@@ -744,7 +671,7 @@ public class InternalEngine extends Engine {
                 throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists in version map (version " + versionValue + ")");
             }
         } else {
-            try (final Searcher searcher = acquireSearcher("assert doc doesn't exist")) {
+            try (Searcher searcher = acquireSearcher("assert doc doesn't exist")) {
                 final long docsWithId = searcher.searcher().count(new TermQuery(index.uid()));
                 if (docsWithId > 0) {
                     throw new AssertionError("doc [" + index.type() + "][" + index.id() + "] exists [" + docsWithId + "] times in index");
@@ -763,14 +690,19 @@ public class InternalEngine extends Engine {
     }
 
     @Override
-    public DeleteResult delete(Delete delete) {
+    public DeleteResult delete(Delete delete) throws IOException {
         DeleteResult result;
         try (ReleasableLock ignored = readLock.acquire()) {
             ensureOpen();
             // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
             result = innerDelete(delete);
-        } catch (Exception e) {
-            result = new DeleteResult(checkIfDocumentFailureOrThrow(delete, e), delete.version(), delete.seqNo());
+        } catch (RuntimeException | IOException e) {
+            try {
+                maybeFailEngine("index", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw e;
         }
         maybePruneDeletedTombstones();
         return result;
@@ -805,32 +737,32 @@ public class InternalEngine extends Engine {
             }
 
             final long expectedVersion = delete.version();
-
-            final Optional<DeleteResult> result =
-                checkVersionConflict(
-                    delete,
-                    currentVersion,
-                    expectedVersion,
-                    deleted,
-                    () -> new DeleteResult(expectedVersion, delete.seqNo(), true),
-                    e -> new DeleteResult(e, expectedVersion, delete.seqNo()));
-
+            Optional<DeleteResult> resultOnVersionConflict;
+            try {
+                final boolean isVersionConflict = checkVersionConflict(delete, currentVersion, expectedVersion, deleted);
+                resultOnVersionConflict = isVersionConflict ? Optional.of(new DeleteResult(expectedVersion, delete.seqNo(), true))
+                    : Optional.empty();
+            } catch (IllegalArgumentException | VersionConflictEngineException ex) {
+                resultOnVersionConflict = Optional.of(new DeleteResult(ex, expectedVersion, delete.seqNo()));
+            }
             final DeleteResult deleteResult;
-            if (result.isPresent()) {
-                deleteResult = result.get();
+            if (resultOnVersionConflict.isPresent()) {
+                deleteResult = resultOnVersionConflict.get();
             } else {
                 if (delete.origin() == Operation.Origin.PRIMARY) {
                     seqNo = seqNoService().generateSeqNo();
                 }
-
                 updatedVersion = delete.versionType().updateVersion(currentVersion, expectedVersion);
                 found = deleteIfFound(delete.uid(), currentVersion, deleted, versionValue);
                 deleteResult = new DeleteResult(updatedVersion, seqNo, found);
+
+                versionMap.putUnderLock(delete.uid().bytes(),
+                    new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis()));
+            }
+            if (!deleteResult.hasFailure()) {
                 location = delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY
                     ? translog.add(new Translog.Delete(delete, deleteResult))
                     : null;
-                versionMap.putUnderLock(delete.uid().bytes(),
-                    new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis()));
                 deleteResult.setTranslogLocation(location);
             }
             deleteResult.setTook(System.nanoTime() - delete.startTime());
@@ -844,6 +776,7 @@ public class InternalEngine extends Engine {
     }
 
     private boolean deleteIfFound(Term uid, long currentVersion, boolean deleted, VersionValue versionValue) throws IOException {
+        assert uid != null : "uid must not be null";
         final boolean found;
         if (currentVersion == Versions.NOT_FOUND) {
             // doc does not exist and no prior deletes
@@ -853,12 +786,42 @@ public class InternalEngine extends Engine {
             found = false;
         } else {
             // we deleted a currently existing document
+            // any exception that comes from this is a either an ACE or a fatal exception there can't be any document failures coming
+            // from this.
             indexWriter.deleteDocuments(uid);
             found = true;
         }
         return found;
     }
 
+    @Override
+    public NoOpResult noOp(final NoOp noOp) {
+        NoOpResult noOpResult;
+        try (ReleasableLock ignored = readLock.acquire()) {
+            noOpResult = innerNoOp(noOp);
+        } catch (final Exception e) {
+            noOpResult = new NoOpResult(noOp.seqNo(), e);
+        }
+        return noOpResult;
+    }
+
+    private NoOpResult innerNoOp(final NoOp noOp) throws IOException {
+        assert noOp.seqNo() > SequenceNumbersService.NO_OPS_PERFORMED;
+        final long seqNo = noOp.seqNo();
+        try {
+            final NoOpResult noOpResult = new NoOpResult(noOp.seqNo());
+            final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason()));
+            noOpResult.setTranslogLocation(location);
+            noOpResult.setTook(System.nanoTime() - noOp.startTime());
+            noOpResult.freeze();
+            return noOpResult;
+        } finally {
+            if (seqNo != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
+                seqNoService().markSeqNoAsCompleted(seqNo);
+            }
+        }
+    }
+
     @Override
     public void refresh(String source) throws EngineException {
         // we obtain a read lock here, since we don't want a flush to happen while we are refreshing
@@ -869,8 +832,6 @@ public class InternalEngine extends Engine {
         } catch (AlreadyClosedException e) {
             failOnTragicEvent(e);
             throw e;
-        } catch (EngineClosedException e) {
-            throw e;
         } catch (Exception e) {
             try {
                 failEngine("refresh failed", e);
@@ -917,8 +878,6 @@ public class InternalEngine extends Engine {
         } catch (AlreadyClosedException e) {
             failOnTragicEvent(e);
             throw e;
-        } catch (EngineClosedException e) {
-            throw e;
         } catch (Exception e) {
             try {
                 failEngine("writeIndexingBuffer failed", e);
@@ -1097,7 +1056,7 @@ public class InternalEngine extends Engine {
 
     @Override
     public void forceMerge(final boolean flush, int maxNumSegments, boolean onlyExpungeDeletes,
-                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, EngineClosedException, IOException {
+                           final boolean upgrade, final boolean upgradeOnlyAncientSegments) throws EngineException, IOException {
         /*
          * We do NOT acquire the readlock here since we are waiting on the merges to finish
          * that's fine since the IW.rollback should stop all the threads and trigger an IOException
@@ -1183,7 +1142,8 @@ public class InternalEngine extends Engine {
     }
 
     @SuppressWarnings("finally")
-    private void failOnTragicEvent(AlreadyClosedException ex) {
+    private boolean failOnTragicEvent(AlreadyClosedException ex) {
+        final boolean engineFailed;
         // if we are already closed due to some tragic exception
         // we need to fail the engine. it might have already been failed before
         // but we are double-checking it's failed and closed
@@ -1196,14 +1156,19 @@ public class InternalEngine extends Engine {
                 }
             } else {
                 failEngine("already closed by tragic event on the index writer", (Exception) indexWriter.getTragicException());
+                engineFailed = true;
             }
         } else if (translog.isOpen() == false && translog.getTragicException() != null) {
             failEngine("already closed by tragic event on the translog", translog.getTragicException());
-        } else if (failedEngine.get() == null) { // we are closed but the engine is not failed yet?
+            engineFailed = true;
+        } else if (failedEngine.get() == null && isClosed.get() == false) { // we are closed but the engine is not failed yet?
             // this smells like a bug - we only expect ACE if we are in a fatal case ie. either translog or IW is closed by
             // a tragic event or has closed itself. if that is not the case we are in a buggy state and raise an assertion error
             throw new AssertionError("Unexpected AlreadyClosedException", ex);
+        } else {
+            engineFailed = false;
         }
+        return engineFailed;
     }
 
     @Override
@@ -1216,8 +1181,7 @@ public class InternalEngine extends Engine {
         // exception that should only be thrown in a tragic event. we pass on the checks to failOnTragicEvent which will
         // throw and AssertionError if the tragic event condition is not met.
         if (e instanceof AlreadyClosedException) {
-            failOnTragicEvent((AlreadyClosedException)e);
-            return true;
+            return failOnTragicEvent((AlreadyClosedException)e);
         } else if (e != null &&
                 ((indexWriter.isOpen() == false && indexWriter.getTragicException() == e)
                         || (translog.isOpen() == false && translog.getTragicException() == e))) {
@@ -1322,7 +1286,7 @@ public class InternalEngine extends Engine {
 
     private long loadCurrentVersionFromIndex(Term uid) throws IOException {
         assert incrementIndexVersionLookup();
-        try (final Searcher searcher = acquireSearcher("load_version")) {
+        try (Searcher searcher = acquireSearcher("load_version")) {
             return Versions.loadVersion(searcher.reader(), uid);
         }
     }
@@ -1526,11 +1490,11 @@ public class InternalEngine extends Engine {
                 final Map<String, String> commitData = new HashMap<>(6);
                 commitData.put(Translog.TRANSLOG_GENERATION_KEY, translogFileGen);
                 commitData.put(Translog.TRANSLOG_UUID_KEY, translogUUID);
-                commitData.put(LOCAL_CHECKPOINT_KEY, localCheckpoint);
+                commitData.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, localCheckpoint);
                 if (syncId != null) {
                     commitData.put(Engine.SYNC_COMMIT_ID, syncId);
                 }
-                commitData.put(MAX_SEQ_NO, Long.toString(seqNoService().getMaxSeqNo()));
+                commitData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(seqNoService().getMaxSeqNo()));
                 if (logger.isTraceEnabled()) {
                     logger.trace("committing writer with commit data [{}]", commitData);
                 }
@@ -1593,14 +1557,6 @@ public class InternalEngine extends Engine {
         return seqNoService;
     }
 
-    @Override
-    public DocsStats getDocStats() {
-        final int numDocs = indexWriter.numDocs();
-        final int maxDoc = indexWriter.maxDoc();
-        return new DocsStats(numDocs, maxDoc-numDocs);
-    }
-
-
     /**
      * Returns the number of times a version was looked up either from the index.
      * Note this is only available if assertions are enabled
diff --git a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
index b489cec5768..44b302e85cd 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
@@ -43,12 +43,12 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
         // Used while refresh is running, and to hold adds/deletes until refresh finishes.  We read from both current and old on lookup:
         final Map<BytesRef,VersionValue> old;
 
-        public Maps(Map<BytesRef,VersionValue> current, Map<BytesRef,VersionValue> old) {
+        Maps(Map<BytesRef,VersionValue> current, Map<BytesRef,VersionValue> old) {
            this.current = current;
            this.old = old;
         }
 
-        public Maps() {
+        Maps() {
             this(ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency(),
                  ConcurrentCollections.<BytesRef,VersionValue>newConcurrentMapWithAggressiveConcurrency());
         }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index 39bff89b7de..68f89426d6e 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -116,6 +116,11 @@ public class ShadowEngine extends Engine {
         throw new UnsupportedOperationException(shardId + " delete operation not allowed on shadow engine");
     }
 
+    @Override
+    public NoOpResult noOp(NoOp noOp) {
+        throw new UnsupportedOperationException(shardId + " no-op operation not allowed on shadow engine");
+    }
+
     @Override
     public SyncedFlushResult syncFlush(String syncId, CommitId expectedCommitId) {
         throw new UnsupportedOperationException(shardId + " sync commit operation not allowed on shadow engine");
@@ -192,9 +197,6 @@ public class ShadowEngine extends Engine {
             ensureOpen();
             searcherManager.maybeRefreshBlocking();
         } catch (AlreadyClosedException e) {
-            // This means there's a bug somewhere: don't suppress it
-            throw new AssertionError(e);
-        } catch (EngineClosedException e) {
             throw e;
         } catch (Exception e) {
             try {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
index 5258b270091..6ee484b2b38 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
@@ -31,7 +31,7 @@ class VersionValue implements Accountable {
 
     private final long version;
 
-    public VersionValue(long version) {
+    VersionValue(long version) {
         this.version = version;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
index feacfe59996..97f10932e0f 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -52,7 +52,7 @@ import java.io.IOException;
  */
 public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
 
-    public static class CommonSettings {
+    class CommonSettings {
         public static final String SETTING_MEMORY_STORAGE_HINT = "memory_storage_hint";
 
         public enum MemoryStorageFormat {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
index 948b19a8afb..5238f06a790 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
@@ -31,7 +31,7 @@ public interface IndexFieldDataCache {
 
     <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
 
-    <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception;
+    <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception;
 
     /**
      * Clears all the field data stored cached in on this index.
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
index d6573b930fa..2ae6c24af7c 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata;
 
 public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericFieldData> {
 
-    public static enum NumericType {
+    enum NumericType {
         BOOLEAN(false),
         BYTE(false),
         SHORT(false),
@@ -33,7 +33,7 @@ public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericField
 
         private final boolean floatingPoint;
 
-        private NumericType(boolean floatingPoint) {
+        NumericType(boolean floatingPoint) {
             this.floatingPoint = floatingPoint;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index 308fd7dd0a7..a1e62979cae 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -25,32 +25,62 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
 import org.joda.time.ReadableDateTime;
 
 import java.util.AbstractList;
-import java.util.Collections;
+import java.util.Comparator;
 import java.util.List;
+import java.util.function.UnaryOperator;
 
 
 /**
  * Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
  * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
  */
-public interface ScriptDocValues<T> extends List<T> {
+public abstract class ScriptDocValues<T> extends AbstractList<T> {
 
     /**
      * Set the current doc ID.
      */
-    void setNextDocId(int docId);
+    public abstract void setNextDocId(int docId);
 
     /**
      * Return a copy of the list of the values for the current document.
      */
-    List<T> getValues();
+    public final List<T> getValues() {
+        return this;
+    }
 
-    public static final class Strings extends AbstractList<String> implements ScriptDocValues<String> {
+    // Throw meaningful exceptions if someone tries to modify the ScriptDocValues.
+    @Override
+    public final void add(int index, T element) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final boolean remove(Object o) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final void replaceAll(UnaryOperator<T> operator) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final T set(int index, T element) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    @Override
+    public final void sort(Comparator<? super T> c) {
+        throw new UnsupportedOperationException("doc values are unmodifiable");
+    }
+
+    public static final class Strings extends ScriptDocValues<String> {
 
         private final SortedBinaryDocValues values;
 
@@ -84,11 +114,6 @@ public interface ScriptDocValues<T> extends List<T> {
             }
         }
 
-        @Override
-        public List<String> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public String get(int index) {
             return values.valueAt(index).utf8ToString();
@@ -101,10 +126,10 @@ public interface ScriptDocValues<T> extends List<T> {
 
     }
 
-    public static class Longs extends AbstractList<Long> implements ScriptDocValues<Long> {
+    public static final class Longs extends ScriptDocValues<Long> {
 
         private final SortedNumericDocValues values;
-        private final MutableDateTime date = new MutableDateTime(0, DateTimeZone.UTC);
+        private Dates dates;
 
         public Longs(SortedNumericDocValues values) {
             this.values = values;
@@ -113,6 +138,9 @@ public interface ScriptDocValues<T> extends List<T> {
         @Override
         public void setNextDocId(int docId) {
             values.setDocument(docId);
+            if (dates != null) {
+                dates.refreshArray();
+            }
         }
 
         public SortedNumericDocValues getInternalValues() {
@@ -127,14 +155,20 @@ public interface ScriptDocValues<T> extends List<T> {
             return values.valueAt(0);
         }
 
-        @Override
-        public List<Long> getValues() {
-            return Collections.unmodifiableList(this);
+        public ReadableDateTime getDate() {
+            if (dates == null) {
+                dates = new Dates(values);
+                dates.refreshArray();
+            }
+            return dates.getValue();
         }
 
-        public ReadableDateTime getDate() {
-            date.setMillis(getValue());
-            return date;
+        public List<ReadableDateTime> getDates() {
+            if (dates == null) {
+                dates = new Dates(values);
+                dates.refreshArray();
+            }
+            return dates;
         }
 
         @Override
@@ -146,10 +180,87 @@ public interface ScriptDocValues<T> extends List<T> {
         public int size() {
             return values.count();
         }
-
     }
 
-    public static class Doubles extends AbstractList<Double> implements ScriptDocValues<Double> {
+    public static final class Dates extends ScriptDocValues<ReadableDateTime> {
+        private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
+
+        private final SortedNumericDocValues values;
+        /**
+         * Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep
+         * this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document.
+         */
+        private MutableDateTime[] dates;
+
+        public Dates(SortedNumericDocValues values) {
+            this.values = values;
+        }
+
+        /**
+         * Fetch the first field value or 0 millis after epoch if there are no values.
+         */
+        public ReadableDateTime getValue() {
+            if (values.count() == 0) {
+                return EPOCH;
+            }
+            return get(0);
+        }
+
+        @Override
+        public ReadableDateTime get(int index) {
+            if (index >= values.count()) {
+                throw new IndexOutOfBoundsException(
+                        "attempted to fetch the [" + index + "] date when there are only [" + values.count() + "] dates.");
+            }
+            return dates[index];
+        }
+
+        @Override
+        public int size() {
+            return values.count();
+        }
+
+        @Override
+        public void setNextDocId(int docId) {
+            values.setDocument(docId);
+            refreshArray();
+        }
+
+        /**
+         * Refresh the backing array. Package private so it can be called when {@link Longs} loads dates.
+         */
+        void refreshArray() {
+            if (values.count() == 0) {
+                return;
+            }
+            if (dates == null) {
+                // Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
+                dates = new MutableDateTime[values.count()];
+                for (int i = 0; i < dates.length; i++) {
+                    dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
+                }
+                return;
+            }
+            if (values.count() > dates.length) {
+                // Happens when we move to a new document and it has more dates than any documents before it.
+                MutableDateTime[] backup = dates;
+                dates = new MutableDateTime[values.count()];
+                System.arraycopy(backup, 0, dates, 0, backup.length);
+                for (int i = 0; i < backup.length; i++) {
+                    dates[i].setMillis(values.valueAt(i));
+                }
+                for (int i = backup.length; i < dates.length; i++) {
+                    dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
+                }
+                return;
+            }
+            for (int i = 0; i < values.count(); i++) {
+                dates[i].setMillis(values.valueAt(i));
+            }
+        }
+    }
+
+    public static final class Doubles extends ScriptDocValues<Double> {
 
         private final SortedNumericDoubleValues values;
 
@@ -174,11 +285,6 @@ public interface ScriptDocValues<T> extends List<T> {
             return values.valueAt(0);
         }
 
-        @Override
-        public List<Double> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public Double get(int index) {
             return values.valueAt(index);
@@ -190,7 +296,7 @@ public interface ScriptDocValues<T> extends List<T> {
         }
     }
 
-    class GeoPoints extends AbstractList<GeoPoint> implements ScriptDocValues<GeoPoint> {
+    public static final class GeoPoints extends ScriptDocValues<GeoPoint> {
 
         private final MultiGeoPointValues values;
 
@@ -237,11 +343,6 @@ public interface ScriptDocValues<T> extends List<T> {
             return getValue().lon();
         }
 
-        @Override
-        public List<GeoPoint> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public GeoPoint get(int index) {
             final GeoPoint point = values.valueAt(index);
@@ -291,7 +392,7 @@ public interface ScriptDocValues<T> extends List<T> {
         }
     }
 
-    final class Booleans extends AbstractList<Boolean> implements ScriptDocValues<Boolean> {
+    public static final class Booleans extends ScriptDocValues<Boolean> {
 
         private final SortedNumericDocValues values;
 
@@ -304,11 +405,6 @@ public interface ScriptDocValues<T> extends List<T> {
             values.setDocument(docId);
         }
 
-        @Override
-        public List<Boolean> getValues() {
-            return this;
-        }
-
         public boolean getValue() {
             return values.count() != 0 && values.valueAt(0) == 1;
         }
@@ -325,7 +421,7 @@ public interface ScriptDocValues<T> extends List<T> {
 
     }
 
-    public static class BytesRefs extends AbstractList<BytesRef> implements ScriptDocValues<BytesRef> {
+    public static final class BytesRefs extends ScriptDocValues<BytesRef> {
 
         private final SortedBinaryDocValues values;
 
@@ -350,11 +446,6 @@ public interface ScriptDocValues<T> extends List<T> {
             return values.valueAt(0);
         }
 
-        @Override
-        public List<BytesRef> getValues() {
-            return Collections.unmodifiableList(this);
-        }
-
         @Override
         public BytesRef get(int index) {
             return values.valueAt(index);
@@ -365,5 +456,4 @@ public interface ScriptDocValues<T> extends List<T> {
             return values.count();
         }
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
index 6c3c5a4f161..4207ac73a1a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/SingletonSortedNumericDoubleValues.java
@@ -34,7 +34,7 @@ final class SingletonSortedNumericDoubleValues extends SortedNumericDoubleValues
   private double value;
   private int count;
 
-  public SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) {
+  SingletonSortedNumericDoubleValues(NumericDoubleValues in, Bits docsWithField) {
     this.in = in;
     this.docsWithField = docsWithField instanceof MatchAllBits ? null : docsWithField;
   }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
index cfc0a0f3313..f7db724c356 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java
@@ -67,7 +67,7 @@ public class SinglePackedOrdinals extends Ordinals {
         private final PackedInts.Reader reader;
         private final ValuesHolder values;
 
-        public Docs(SinglePackedOrdinals parent, ValuesHolder values) {
+        Docs(SinglePackedOrdinals parent, ValuesHolder values) {
             this.maxOrd = parent.valueCount;
             this.reader = parent.reader;
             this.values = values;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
index 90554bd1308..c5858afaf22 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
@@ -99,7 +99,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<Ato
         }
     }
 
-    public AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
+    AbstractIndexGeoPointFieldData(IndexSettings indexSettings, String fieldName, IndexFieldDataCache cache) {
         super(indexSettings, fieldName, cache);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 6a66765a200..84fa585b6b2 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -132,7 +132,7 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
 
         private int minFreq;
         private int maxFreq;
-        public FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) {
+        FrequencyFilter(TermsEnum delegate, int minFreq, int maxFreq) {
             super(delegate, false);
             this.minFreq = minFreq;
             this.maxFreq = maxFreq;
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
index d483baa777e..71d1e04604b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
@@ -179,7 +179,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
         final OrdinalMap ordMap;
         final AtomicParentChildFieldData[] fieldData;
 
-        public OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) {
+        OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) {
             this.ordMap = ordMap;
             this.fieldData = fieldData;
         }
@@ -223,7 +223,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
         private final Map<String, OrdinalMapAndAtomicFieldData> atomicFD;
         private final int segmentIndex;
 
-        public GlobalAtomicFieldData(Set<String> types, Map<String, OrdinalMapAndAtomicFieldData> atomicFD, int segmentIndex) {
+        GlobalAtomicFieldData(Set<String> types, Map<String, OrdinalMapAndAtomicFieldData> atomicFD, int segmentIndex) {
             this.types = types;
             this.atomicFD = atomicFD;
             this.segmentIndex = segmentIndex;
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetField.java b/core/src/main/java/org/elasticsearch/index/get/GetField.java
index be3b8d6a257..928988e3d3d 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetField.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetField.java
@@ -22,14 +22,21 @@ package org.elasticsearch.index.get;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
-public class GetField implements Streamable, Iterable<Object> {
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseStoredFieldsValue;
+
+public class GetField implements Streamable, ToXContent, Iterable<Object> {
 
     private String name;
     private List<Object> values;
@@ -38,8 +45,8 @@ public class GetField implements Streamable, Iterable<Object> {
     }
 
     public GetField(String name, List<Object> values) {
-        this.name = name;
-        this.values = values;
+        this.name = Objects.requireNonNull(name, "name must not be null");
+        this.values = Objects.requireNonNull(values, "values must not be null");
     }
 
     public String getName() {
@@ -90,4 +97,55 @@ public class GetField implements Streamable, Iterable<Object> {
             out.writeGenericValue(obj);
         }
     }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(name);
+        for (Object value : values) {
+            //this call doesn't really need to support writing any kind of object.
+            //Stored fields values are converted using MappedFieldType#valueForDisplay.
+            //As a result they can either be Strings, Numbers, Booleans, or BytesReference, that's all.
+            builder.value(value);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    public static GetField fromXContent(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+        String fieldName = parser.currentName();
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation);
+        List<Object> values = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            values.add(parseStoredFieldsValue(parser));
+        }
+        return new GetField(fieldName, values);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetField objects = (GetField) o;
+        return Objects.equals(name, objects.name) &&
+                Objects.equals(values, objects.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, values);
+    }
+
+    @Override
+    public String toString() {
+        return "GetField{" +
+                "name='" + name + '\'' +
+                ", values=" + values +
+                '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetResult.java b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
index 0f02885a251..3837a39c5c5 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -25,9 +25,10 @@ import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -38,11 +39,21 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
 import static org.elasticsearch.index.get.GetField.readGetField;
 
-public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
+public class GetResult implements Streamable, Iterable<GetField>, ToXContentObject {
+
+    private static final String _INDEX = "_index";
+    private static final String _TYPE = "_type";
+    private static final String _ID = "_id";
+    private static final String _VERSION = "_version";
+    private static final String FOUND = "found";
+    private static final String FIELDS = "fields";
 
     private String index;
     private String type;
@@ -57,7 +68,8 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
     GetResult() {
     }
 
-    public GetResult(String index, String type, String id, long version, boolean exists, BytesReference source, Map<String, GetField> fields) {
+    public GetResult(String index, String type, String id, long version, boolean exists, BytesReference source,
+                     Map<String, GetField> fields) {
         this.index = index;
         this.type = type;
         this.id = id;
@@ -123,6 +135,10 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
      * Returns bytes reference, also un compress the source if needed.
      */
     public BytesReference sourceRef() {
+        if (source == null) {
+            return null;
+        }
+
         try {
             this.source = CompressorFactory.uncompressIfNeeded(this.source);
             return this.source;
@@ -196,15 +212,6 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
         return fields.values().iterator();
     }
 
-    static final class Fields {
-        static final String _INDEX = "_index";
-        static final String _TYPE = "_type";
-        static final String _ID = "_id";
-        static final String _VERSION = "_version";
-        static final String FOUND = "found";
-        static final String FIELDS = "fields";
-    }
-
     public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params) throws IOException {
         List<GetField> metaFields = new ArrayList<>();
         List<GetField> otherFields = new ArrayList<>();
@@ -225,20 +232,16 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
             builder.field(field.getName(), field.getValue());
         }
 
-        builder.field(Fields.FOUND, exists);
+        builder.field(FOUND, exists);
 
         if (source != null) {
             XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params);
         }
 
         if (!otherFields.isEmpty()) {
-            builder.startObject(Fields.FIELDS);
+            builder.startObject(FIELDS);
             for (GetField field : otherFields) {
-                builder.startArray(field.getName());
-                for (Object value : field.getValues()) {
-                    builder.value(value);
-                }
-                builder.endArray();
+                field.toXContent(builder, params);
             }
             builder.endObject();
         }
@@ -247,23 +250,77 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        if (!isExists()) {
-            builder.field(Fields._INDEX, index);
-            builder.field(Fields._TYPE, type);
-            builder.field(Fields._ID, id);
-            builder.field(Fields.FOUND, false);
-        } else {
-            builder.field(Fields._INDEX, index);
-            builder.field(Fields._TYPE, type);
-            builder.field(Fields._ID, id);
+        builder.startObject();
+        builder.field(_INDEX, index);
+        builder.field(_TYPE, type);
+        builder.field(_ID, id);
+        if (isExists()) {
             if (version != -1) {
-                builder.field(Fields._VERSION, version);
+                builder.field(_VERSION, version);
             }
             toXContentEmbedded(builder, params);
+        } else {
+            builder.field(FOUND, false);
         }
+        builder.endObject();
         return builder;
     }
 
+    public static GetResult fromXContentEmbedded(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation);
+
+        String currentFieldName = parser.currentName();
+        String index = null, type = null, id = null;
+        long version = -1;
+        boolean found = false;
+        BytesReference source = null;
+        Map<String, GetField> fields = new HashMap<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (_INDEX.equals(currentFieldName)) {
+                    index = parser.text();
+                } else if (_TYPE.equals(currentFieldName)) {
+                    type = parser.text();
+                } else if (_ID.equals(currentFieldName)) {
+                    id = parser.text();
+                }  else if (_VERSION.equals(currentFieldName)) {
+                    version = parser.longValue();
+                } else if (FOUND.equals(currentFieldName)) {
+                    found = parser.booleanValue();
+                } else {
+                    fields.put(currentFieldName, new GetField(currentFieldName, Collections.singletonList(parser.objectText())));
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (SourceFieldMapper.NAME.equals(currentFieldName)) {
+                    try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
+                        //the original document gets slightly modified: whitespaces or pretty printing are not preserved,
+                        //it all depends on the current builder settings
+                        builder.copyCurrentStructure(parser);
+                        source = builder.bytes();
+                    }
+                } else if (FIELDS.equals(currentFieldName)) {
+                    while(parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                        GetField getField = GetField.fromXContent(parser);
+                        fields.put(getField.getName(), getField);
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            }
+        }
+        return new GetResult(index, type, id, version, found, source, fields);
+    }
+
+    public static GetResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        return fromXContentEmbedded(parser);
+    }
+
     public static GetResult readGetResult(StreamInput in) throws IOException {
         GetResult result = new GetResult();
         result.readFrom(in);
@@ -314,5 +371,28 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
             }
         }
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        GetResult getResult = (GetResult) o;
+        return version == getResult.version &&
+                exists == getResult.exists &&
+                Objects.equals(index, getResult.index) &&
+                Objects.equals(type, getResult.type) &&
+                Objects.equals(id, getResult.id) &&
+                Objects.equals(fields, getResult.fields) &&
+                Objects.equals(sourceAsMap(), getResult.sourceAsMap());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(index, type, id, version, exists, fields, sourceAsMap());
+    }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
index a0f7ad57d04..85403ccc03b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.lucene.all.AllField;
@@ -39,7 +40,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseTextField;
 
@@ -52,7 +52,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
     public static class Defaults {
         public static final String NAME = AllFieldMapper.NAME;
         public static final String INDEX_NAME = AllFieldMapper.NAME;
-        public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_ENABLED;
+        public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
         public static final int POSITION_INCREMENT_GAP = 100;
 
         public static final MappedFieldType FIELD_TYPE = new AllFieldType();
@@ -103,6 +103,11 @@ public class AllFieldMapper extends MetadataFieldMapper {
         @Override
         public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
                                                  ParserContext parserContext) throws MapperParsingException {
+            if (node.isEmpty() == false &&
+                    parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                throw new IllegalArgumentException("[_all] is disabled in 6.0. As a replacement, you can use an [copy_to] " +
+                                "on mapping fields to create your own catch all field.");
+            }
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
             builder.fieldType().setIndexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
             builder.fieldType().setSearchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
@@ -112,7 +117,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
             // the AllFieldMapper ctor in the builder since it is not valid. Here we validate
             // the doc values settings (old and new) are rejected
             Object docValues = node.get("doc_values");
-            if (docValues != null && lenientNodeBooleanValue(docValues)) {
+            if (docValues != null && TypeParsers.nodeBooleanValueLenient(name, "doc_values", docValues)) {
                 throw new MapperParsingException("Field [" + name +
                     "] is always tokenized and cannot have doc values");
             }
@@ -133,8 +138,8 @@ public class AllFieldMapper extends MetadataFieldMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED :
-                        EnabledAttributeMapper.DISABLED);
+                    boolean enabled = TypeParsers.nodeBooleanValueLenient(name, "enabled", fieldNode);
+                    builder.enabled(enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();
                 }
             }
@@ -155,7 +160,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
 
     static final class AllFieldType extends StringFieldType {
 
-        public AllFieldType() {
+        AllFieldType() {
         }
 
         protected AllFieldType(AllFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
index c4f9e7a1cb3..f09d841ec9a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BaseGeoPointFieldMapper.java
@@ -19,14 +19,17 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.geo.GeoHashUtils;
+import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -122,7 +125,7 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
                 Object propNode = entry.getValue();
 
                 if (propName.equals(Names.IGNORE_MALFORMED)) {
-                    builder.ignoreMalformed(XContentMapValues.lenientNodeBooleanValue(propNode));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, Names.IGNORE_MALFORMED, propNode, parserContext));
                     iterator.remove();
                 }
             }
@@ -170,6 +173,24 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + name() + "]");
         }
+
+        @Override
+        public FieldStats.GeoPoint stats(IndexReader reader) throws IOException {
+            String field = name();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(field);
+            if (fi == null) {
+                return null;
+            }
+
+            Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, field);
+            if (terms == null) {
+                return new FieldStats.GeoPoint(reader.maxDoc(), 0L, -1L, -1L, isSearchable(), isAggregatable());
+            }
+            GeoPoint minPt = GeoPoint.fromGeohash(NumericUtils.sortableBytesToLong(terms.getMin().bytes, terms.getMin().offset));
+            GeoPoint maxPt = GeoPoint.fromGeohash(NumericUtils.sortableBytesToLong(terms.getMax().bytes, terms.getMax().offset));
+            return new FieldStats.GeoPoint(reader.maxDoc(), terms.getDocCount(), -1L, terms.getSumTotalTermFreq(), isSearchable(),
+                isAggregatable(), minPt, maxPt);
+        }
     }
 
     protected Explicit<Boolean> ignoreMalformed;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
index c32f6219a3c..aa60bfcd9a0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java
@@ -83,7 +83,7 @@ public class BinaryFieldMapper extends FieldMapper {
 
     static final class BinaryFieldType extends MappedFieldType {
 
-        public BinaryFieldType() {}
+        BinaryFieldType() {}
 
         protected BinaryFieldType(BinaryFieldType ref) {
             super(ref);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
index fc9520beeab..7bd8c79ee67 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -26,8 +26,10 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Booleans;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -44,13 +46,13 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
 /**
  * A field mapper for boolean fields.
  */
 public class BooleanFieldMapper extends FieldMapper {
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(BooleanFieldMapper.class));
 
     public static final String CONTENT_TYPE = "boolean";
 
@@ -108,7 +110,7 @@ public class BooleanFieldMapper extends FieldMapper {
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(lenientNodeBooleanValue(propNode));
+                    builder.nullValue(TypeParsers.nodeBooleanValue(name, "null_value", propNode, parserContext));
                     iterator.remove();
                 }
             }
@@ -231,7 +233,15 @@ public class BooleanFieldMapper extends FieldMapper {
                     value = fieldType().nullValue();
                 }
             } else {
-                value = context.parser().booleanValue();
+                if (indexCreatedVersion.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                    value = context.parser().booleanValue();
+                } else {
+                    value = context.parser().booleanValueLenient();
+                    if (context.parser().isBooleanValueLenient() != context.parser().isBooleanValue()) {
+                        String rawValue = context.parser().text();
+                        deprecationLogger.deprecated("Expected a boolean for property [{}] but got [{}]", fieldType().name(), rawValue);
+                    }
+                }
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
index c499c30c607..3855489efe3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.suggest.document.Completion50PostingsFormat;
@@ -125,22 +124,22 @@ public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapp
                 if (fieldName.equals("type")) {
                     continue;
                 }
-                if (parserContext.parseFieldMatcher().match(fieldName, Fields.ANALYZER)) {
+                if (Fields.ANALYZER.match(fieldName)) {
                     indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.SEARCH_ANALYZER)) {
+                } else if (Fields.SEARCH_ANALYZER.match(fieldName)) {
                     searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_SEPARATORS)) {
+                } else if (Fields.PRESERVE_SEPARATORS.match(fieldName)) {
                     builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.PRESERVE_POSITION_INCREMENTS)) {
+                } else if (Fields.PRESERVE_POSITION_INCREMENTS.match(fieldName)) {
                     builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.MAX_INPUT_LENGTH)) {
+                } else if (Fields.MAX_INPUT_LENGTH.match(fieldName)) {
                     builder.maxInputLength(Integer.parseInt(fieldNode.toString()));
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, Fields.CONTEXTS)) {
+                } else if (Fields.CONTEXTS.match(fieldName)) {
                     builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated()));
                     iterator.remove();
                 } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
index a8b27d11245..8e6cee222d4 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/CustomDocValuesField.java
@@ -39,7 +39,7 @@ abstract class CustomDocValuesField implements IndexableField {
 
     private final String name;
 
-    public CustomDocValuesField(String  name) {
+    CustomDocValuesField(String  name) {
         this.name = name;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
index ce03fc0a6b4..52e5de0a176 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -70,6 +70,7 @@ public class DateFieldMapper extends FieldMapper {
 
         private Boolean ignoreMalformed;
         private Locale locale;
+        private boolean dateTimeFormatterSet = false;
 
         public Builder(String name) {
             super(name, new DateFieldType(), new DateFieldType());
@@ -97,8 +98,14 @@ public class DateFieldMapper extends FieldMapper {
             return Defaults.IGNORE_MALFORMED;
         }
 
+        /** Whether an explicit format for this date field has been set already. */
+        public boolean isDateTimeFormatterSet() {
+            return dateTimeFormatterSet;
+        }
+
         public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
             fieldType().setDateTimeFormatter(dateTimeFormatter);
+            dateTimeFormatterSet = true;
             return this;
         }
 
@@ -144,7 +151,7 @@ public class DateFieldMapper extends FieldMapper {
                     builder.nullValue(propNode.toString());
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode, parserContext));
                     iterator.remove();
                 } else if (propName.equals("locale")) {
                     builder.locale(LocaleUtils.parse(propNode.toString()));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
index 12f968bb38b..2a19cb3f8bd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentFieldMappers.java
@@ -75,8 +75,6 @@ public final class DocumentFieldMappers implements Iterable<FieldMapper> {
         for (FieldMapper fieldMapper : this) {
             if (Regex.simpleMatch(pattern, fieldMapper.fieldType().name())) {
                 fields.add(fieldMapper.fieldType().name());
-            } else if (Regex.simpleMatch(pattern, fieldMapper.fieldType().name())) {
-                fields.add(fieldMapper.fieldType().name());
             }
         }
         return fields;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index fdc45530b99..a8c74101bff 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -256,8 +257,9 @@ public class DocumentMapper implements ToXContent {
         return this.objectMappers;
     }
 
+    // TODO this method looks like it is only used in tests...
     public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException {
-        return parse(SourceToParse.source(index, type, id, source));
+        return parse(SourceToParse.source(index, type, id, source, XContentType.JSON));
     }
 
     public ParsedDocument parse(SourceToParse source) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
index b3e967bdad3..be25775c135 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java
@@ -21,12 +21,12 @@ package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -44,23 +44,23 @@ public class DocumentMapperParser {
 
     final MapperService mapperService;
     final IndexAnalyzers indexAnalyzers;
+    private final NamedXContentRegistry xContentRegistry;
     private final SimilarityService similarityService;
     private final Supplier<QueryShardContext> queryShardContextSupplier;
 
     private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
 
     private final Version indexVersionCreated;
-    private final ParseFieldMatcher parseFieldMatcher;
 
     private final Map<String, Mapper.TypeParser> typeParsers;
     private final Map<String, MetadataFieldMapper.TypeParser> rootTypeParsers;
 
     public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, IndexAnalyzers indexAnalyzers,
-                                SimilarityService similarityService, MapperRegistry mapperRegistry,
+                                NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry,
                                 Supplier<QueryShardContext> queryShardContextSupplier) {
-        this.parseFieldMatcher = new ParseFieldMatcher(indexSettings.getSettings());
         this.mapperService = mapperService;
         this.indexAnalyzers = indexAnalyzers;
+        this.xContentRegistry = xContentRegistry;
         this.similarityService = similarityService;
         this.queryShardContextSupplier = queryShardContextSupplier;
         this.typeParsers = mapperRegistry.getMapperParsers();
@@ -69,7 +69,8 @@ public class DocumentMapperParser {
     }
 
     public Mapper.TypeParser.ParserContext parserContext(String type) {
-        return new Mapper.TypeParser.ParserContext(type, indexAnalyzers, similarityService::getSimilarity, mapperService, typeParsers::get, indexVersionCreated, parseFieldMatcher, queryShardContextSupplier);
+        return new Mapper.TypeParser.ParserContext(type, indexAnalyzers, similarityService::getSimilarity, mapperService,
+                typeParsers::get, indexVersionCreated, queryShardContextSupplier);
     }
 
     public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException {
@@ -79,7 +80,7 @@ public class DocumentMapperParser {
     public DocumentMapper parse(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException {
         Map<String, Object> mapping = null;
         if (source != null) {
-            Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true).v2();
+            Map<String, Object> root = XContentHelper.convertToMap(source.compressedReference(), true, XContentType.JSON).v2();
             Tuple<String, Map<String, Object>> t = extractMapping(type, root);
             type = t.v1();
             mapping = t.v2();
@@ -106,7 +107,8 @@ public class DocumentMapperParser {
 
         Mapper.TypeParser.ParserContext parserContext = parserContext(type);
         // parse RootObjectMapper
-        DocumentMapper.Builder docBuilder = new DocumentMapper.Builder((RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService);
+        DocumentMapper.Builder docBuilder = new DocumentMapper.Builder(
+                (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService);
         Iterator<Map.Entry<String, Object>> iterator = mapping.entrySet().iterator();
         // parse DocumentMapper
         while(iterator.hasNext()) {
@@ -140,7 +142,8 @@ public class DocumentMapperParser {
     }
 
     public static void checkNoRemainingFields(String fieldName, Map<?, ?> fieldNodeMap, Version indexVersionCreated) {
-        checkNoRemainingFields(fieldNodeMap, indexVersionCreated, "Mapping definition for [" + fieldName + "] has unsupported parameters: ");
+        checkNoRemainingFields(fieldNodeMap, indexVersionCreated,
+                "Mapping definition for [" + fieldName + "] has unsupported parameters: ");
     }
 
     public static void checkNoRemainingFields(Map<?, ?> fieldNodeMap, Version indexVersionCreated, String message) {
@@ -159,7 +162,7 @@ public class DocumentMapperParser {
 
     private Tuple<String, Map<String, Object>> extractMapping(String type, String source) throws MapperParsingException {
         Map<String, Object> root;
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, source)) {
             root = parser.mapOrdered();
         } catch (Exception e) {
             throw new MapperParsingException("failed to parse mapping definition", e);
@@ -182,4 +185,8 @@ public class DocumentMapperParser {
         }
         return mapping;
     }
+
+    NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 72d021c0e3e..5ec1eab2115 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -47,7 +47,7 @@ final class DocumentParser {
     private final DocumentMapperParser docMapperParser;
     private final DocumentMapper docMapper;
 
-    public DocumentParser(IndexSettings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper) {
+    DocumentParser(IndexSettings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper) {
         this.indexSettings = indexSettings;
         this.docMapperParser = docMapperParser;
         this.docMapper = docMapper;
@@ -58,7 +58,7 @@ final class DocumentParser {
 
         final Mapping mapping = docMapper.mapping();
         final ParseContext.InternalParseContext context;
-        try (XContentParser parser = XContentHelper.createParser(source.source())) {
+        try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), source.source())) {
             context = new ParseContext.InternalParseContext(indexSettings.getSettings(),
                     docMapperParser, docMapper, source, parser);
             validateStart(parser);
@@ -154,6 +154,7 @@ final class DocumentParser {
             source.routing(),
             context.docs(),
             context.sourceToParse().source(),
+            context.sourceToParse().getXContentType(),
             update
         ).parent(source.parent());
     }
@@ -173,6 +174,17 @@ final class DocumentParser {
         return new MapperParsingException("failed to parse", e);
     }
 
+    private static String[] splitAndValidatePath(String fullFieldPath) {
+        String[] parts = fullFieldPath.split("\\.");
+        for (String part : parts) {
+            if (Strings.hasText(part) == false) {
+                throw new IllegalArgumentException(
+                    "object field starting or ending with a [.] makes object resolution ambiguous: [" + fullFieldPath + "]");
+            }
+        }
+        return parts;
+    }
+
     /** Creates a Mapping containing any dynamically added fields, or returns null if there were no dynamic mappings. */
     static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, List<Mapper> dynamicMappers) {
         if (dynamicMappers.isEmpty()) {
@@ -185,7 +197,7 @@ final class DocumentParser {
         Iterator<Mapper> dynamicMapperItr = dynamicMappers.iterator();
         List<ObjectMapper> parentMappers = new ArrayList<>();
         Mapper firstUpdate = dynamicMapperItr.next();
-        parentMappers.add(createUpdate(mapping.root(), firstUpdate.name().split("\\."), 0, firstUpdate));
+        parentMappers.add(createUpdate(mapping.root(), splitAndValidatePath(firstUpdate.name()), 0, firstUpdate));
         Mapper previousMapper = null;
         while (dynamicMapperItr.hasNext()) {
             Mapper newMapper = dynamicMapperItr.next();
@@ -197,7 +209,7 @@ final class DocumentParser {
                 continue;
             }
             previousMapper = newMapper;
-            String[] nameParts = newMapper.name().split("\\.");
+            String[] nameParts = splitAndValidatePath(newMapper.name());
 
             // We first need the stack to only contain mappers in common with the previously processed mapper
             // For example, if the first mapper processed was a.b.c, and we now have a.d, the stack will contain
@@ -454,7 +466,7 @@ final class DocumentParser {
             context.path().remove();
         } else {
 
-            final String[] paths = currentFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
             ObjectMapper parentMapper = parentMapperTuple.v2();
@@ -498,7 +510,7 @@ final class DocumentParser {
             }
         } else {
 
-            final String[] paths = arrayFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(arrayFieldName);
             arrayFieldName = paths[paths.length - 1];
             lastFieldName = arrayFieldName;
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
@@ -562,7 +574,7 @@ final class DocumentParser {
             parseObjectOrField(context, mapper);
         } else {
 
-            final String[] paths = currentFieldName.split("\\.");
+            final String[] paths = splitAndValidatePath(currentFieldName);
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
             parentMapper = parentMapperTuple.v2();
@@ -650,47 +662,61 @@ final class DocumentParser {
 
     private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseContext context, XContentParser.Token token, String currentFieldName) throws IOException {
         if (token == XContentParser.Token.VALUE_STRING) {
-            if (context.root().dateDetection()) {
-                String text = context.parser().text();
-                // a safe check since "1" gets parsed as well
-                if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) {
-                    for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) {
-                        try {
-                            dateTimeFormatter.parser().parseMillis(text);
-                            Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE);
-                            if (builder == null) {
-                                builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings()));
-                            }
-                            return builder;
-                        } catch (Exception e) {
-                            // failure to parse this, continue
+            String text = context.parser().text();
+
+            boolean parseableAsLong = false;
+            try {
+                Long.parseLong(text);
+                parseableAsLong = true;
+            } catch (NumberFormatException e) {
+                // not a long number
+            }
+
+            boolean parseableAsDouble = false;
+            try {
+                Double.parseDouble(text);
+                parseableAsDouble = true;
+            } catch (NumberFormatException e) {
+                // not a double number
+            }
+
+            if (parseableAsLong && context.root().numericDetection()) {
+                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
+                if (builder == null) {
+                    builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
+                }
+                return builder;
+            } else if (parseableAsDouble && context.root().numericDetection()) {
+                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
+                if (builder == null) {
+                    builder = newFloatBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
+                }
+                return builder;
+            } else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) {
+                // We refuse to match pure numbers, which are too likely to be
+                // false positives with date formats that include eg.
+                // `epoch_millis` or `YYYY`
+                for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) {
+                    try {
+                        dateTimeFormatter.parser().parseMillis(text);
+                    } catch (IllegalArgumentException e) {
+                        // failure to parse this, continue
+                        continue;
+                    }
+                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE);
+                    if (builder == null) {
+                        builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings()));
+                    }
+                    if (builder instanceof DateFieldMapper.Builder) {
+                        DateFieldMapper.Builder dateBuilder = (DateFieldMapper.Builder) builder;
+                        if (dateBuilder.isDateTimeFormatterSet() == false) {
+                            dateBuilder.dateTimeFormatter(dateTimeFormatter);
                         }
                     }
+                    return builder;
                 }
             }
-            if (context.root().numericDetection()) {
-                String text = context.parser().text();
-                try {
-                    Long.parseLong(text);
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG);
-                    if (builder == null) {
-                        builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
-                    }
-                    return builder;
-                } catch (NumberFormatException e) {
-                    // not a long number
-                }
-                try {
-                    Double.parseDouble(text);
-                    Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE);
-                    if (builder == null) {
-                        builder = newFloatBuilder(currentFieldName, Version.indexCreated(context.indexSettings()));
-                    }
-                    return builder;
-                } catch (NumberFormatException e) {
-                    // not a long number
-                }
-            }
+
             Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
             if (builder == null) {
                 builder = new TextFieldMapper.Builder(currentFieldName)
@@ -800,7 +826,7 @@ final class DocumentParser {
             // The path of the dest field might be completely different from the current one so we need to reset it
             context = context.overridePath(new ContentPath(0));
 
-            final String[] paths = field.split("\\.");
+            final String[] paths = splitAndValidatePath(field);
             final String fieldName = paths[paths.length-1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, null);
             ObjectMapper mapper = parentMapperTuple.v2();
@@ -884,7 +910,7 @@ final class DocumentParser {
 
     // looks up a child mapper, but takes into account field names that expand to objects
     static Mapper getMapper(ObjectMapper objectMapper, String fieldName) {
-        String[] subfields = fieldName.split("\\.");
+        String[] subfields = splitAndValidatePath(fieldName);
         for (int i = 0; i < subfields.length - 1; ++i) {
             Mapper mapper = objectMapper.getMapper(subfields[i]);
             if (mapper == null || (mapper instanceof ObjectMapper) == false) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
index dcb991095b2..b07492740ae 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
@@ -38,7 +38,7 @@ public class DynamicTemplate implements ToXContent {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DynamicTemplate.class));
 
-    public static enum MatchType {
+    public enum MatchType {
         SIMPLE {
             @Override
             public boolean matches(String pattern, String value) {
@@ -152,7 +152,7 @@ public class DynamicTemplate implements ToXContent {
                     return v;
                 }
             }
-            throw new IllegalArgumentException("No xcontent type matched on [" + value + "], possible values are "
+            throw new IllegalArgumentException("No field type matched on [" + value + "], possible values are "
                     + Arrays.toString(values()));
         }
 
@@ -208,6 +208,8 @@ public class DynamicTemplate implements ToXContent {
                 if (indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
                     throw e;
                 } else {
+                    DEPRECATION_LOGGER.deprecated("match_mapping_type [" + matchMappingType + "] is invalid and will be ignored: "
+                            + e.getMessage());
                     // this template is on an unknown type so it will never match anything
                     // null indicates that the template should be ignored
                     return null;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index b2152e41b75..0ed093b3a88 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -237,7 +237,7 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         }
     }
 
-    private final Version indexCreatedVersion;
+    protected final Version indexCreatedVersion;
     protected MappedFieldType fieldType;
     protected final MappedFieldType defaultFieldType;
     protected MultiFields multiFields;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
index 764586562d2..4396cba58cc 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
@@ -36,8 +36,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 /**
  * A mapper that indexes the field names of a document under <code>_field_names</code>. This mapper is typically useful in order
  * to have fast <code>exists</code> and <code>missing</code> queries/filters.
@@ -107,7 +105,7 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode));
+                    builder.enabled(TypeParsers.nodeBooleanValue(name, "enabled", fieldNode, parserContext));
                     iterator.remove();
                 }
             }
@@ -245,7 +243,7 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
                     }
 
                     @Override
-                    public final void remove() {
+                    public void remove() {
                         throw new UnsupportedOperationException();
                     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
index d05077fe5cf..fee41e43f2a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
@@ -43,7 +43,7 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
     final CopyOnWriteHashMap<String, Set<String>> fullNameToTypes;
 
     /** Create a new empty instance. */
-    public FieldTypeLookup() {
+    FieldTypeLookup() {
         fullNameToFieldType = new CopyOnWriteHashMap<>();
         fullNameToTypes = new CopyOnWriteHashMap<>();
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index f584c216709..c46fe227d6f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -18,12 +18,11 @@
  */
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.index.IndexableField;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Shape;
-import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
@@ -33,8 +32,8 @@ import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -42,9 +41,11 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Shape;
+import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -52,9 +53,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
-
 /**
  * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
  * <p>
@@ -183,11 +181,12 @@ public class GeoShapeFieldMapper extends FieldMapper {
                     builder.fieldType().setStrategyName(fieldNode.toString());
                     iterator.remove();
                 } else if (Names.COERCE.equals(fieldName)) {
-                    builder.coerce(lenientNodeBooleanValue(fieldNode));
+                    builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
                     iterator.remove();
                 } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
                     && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) {
-                    builder.fieldType().setPointsOnly(XContentMapValues.lenientNodeBooleanValue(fieldNode));
+                    boolean pointsOnly = TypeParsers.nodeBooleanValue(fieldName, Names.STRATEGY_POINTS_ONLY, fieldNode, parserContext);
+                    builder.fieldType().setPointsOnly(pointsOnly);
                     iterator.remove();
                 }
             }
@@ -415,6 +414,20 @@ public class GeoShapeFieldMapper extends FieldMapper {
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead");
         }
+
+        @Override
+        public FieldStats stats(IndexReader reader) throws IOException {
+            int maxDoc = reader.maxDoc();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
+            /**
+             * we don't have a specific type for geo_shape so we use an empty {@link FieldStats.Text}.
+             * TODO: we should maybe support a new type that knows how to (de)encode the min/max information
+             */
+            return new FieldStats.Text(maxDoc, -1, -1, -1, isSearchable(), isAggregatable());
+        }
     }
 
     protected Explicit<Boolean> coerce;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
index 4f74278014b..6cac16d2fce 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java
@@ -88,7 +88,7 @@ public class IdFieldMapper extends MetadataFieldMapper {
 
     static final class IdFieldType extends TermBasedFieldType {
 
-        public IdFieldType() {
+        IdFieldType() {
         }
 
         protected IdFieldType(IdFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
index 164f18075a4..c961b74d77a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java
@@ -92,7 +92,7 @@ public class IndexFieldMapper extends MetadataFieldMapper {
 
     static final class IndexFieldType extends MappedFieldType {
 
-        public IndexFieldType() {}
+        IndexFieldType() {}
 
         protected IndexFieldType(IndexFieldType ref) {
             super(ref);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
index a892fbe2214..df35253b8ec 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.document.Field;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -32,7 +31,6 @@ import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
@@ -41,7 +39,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
@@ -49,13 +46,9 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.net.InetAddress;
-import java.util.AbstractList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.ListIterator;
 import java.util.Map;
 
 /** A {@link FieldMapper} for ip addresses. */
@@ -119,7 +112,7 @@ public class IpFieldMapper extends FieldMapper {
                     builder.nullValue(InetAddresses.forString(propNode.toString()));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode, parserContext));
                     iterator.remove();
                 } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) {
                     iterator.remove();
@@ -238,11 +231,11 @@ public class IpFieldMapper extends FieldMapper {
                 InetAddressPoint.decode(min), InetAddressPoint.decode(max));
         }
 
-        private static class IpScriptDocValues extends AbstractList<String> implements ScriptDocValues<String> {
+        public static final class IpScriptDocValues extends ScriptDocValues<String> {
 
             private final RandomAccessOrds values;
 
-            IpScriptDocValues(RandomAccessOrds values) {
+            public IpScriptDocValues(RandomAccessOrds values) {
                 this.values = values;
             }
 
@@ -259,11 +252,6 @@ public class IpFieldMapper extends FieldMapper {
                 }
             }
 
-            @Override
-            public List<String> getValues() {
-                return Collections.unmodifiableList(this);
-            }
-
             @Override
             public String get(int index) {
                 BytesRef encoded = values.lookupOrd(values.ordAt(0));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
index 68807215027..f988a68d5ef 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -19,16 +19,20 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
 
@@ -36,6 +40,7 @@ import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.TypeParsers.parseField;
 
@@ -70,6 +75,11 @@ public final class KeywordFieldMapper extends FieldMapper {
             builder = this;
         }
 
+        @Override
+        public KeywordFieldType fieldType() {
+            return (KeywordFieldType) super.fieldType();
+        }
+
         public Builder ignoreAbove(int ignoreAbove) {
             if (ignoreAbove < 0) {
                 throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
@@ -92,6 +102,12 @@ public final class KeywordFieldMapper extends FieldMapper {
             return builder;
         }
 
+        public Builder normalizer(NamedAnalyzer normalizer) {
+            fieldType().setNormalizer(normalizer);
+            fieldType().setSearchAnalyzer(normalizer);
+            return builder;
+        }
+
         @Override
         public KeywordFieldMapper build(BuilderContext context) {
             setupFieldType(context);
@@ -103,7 +119,7 @@ public final class KeywordFieldMapper extends FieldMapper {
 
     public static class TypeParser implements Mapper.TypeParser {
         @Override
-        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder(name);
             parseField(builder, name, node, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
@@ -120,10 +136,19 @@ public final class KeywordFieldMapper extends FieldMapper {
                     builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
                     iterator.remove();
                 } else if (propName.equals("norms")) {
-                    builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode) == false);
+                    builder.omitNorms(XContentMapValues.nodeBooleanValue(propNode, "norms") == false);
                     iterator.remove();
                 } else if (propName.equals("eager_global_ordinals")) {
-                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
+                    iterator.remove();
+                } else if (propName.equals("normalizer")) {
+                    if (propNode != null) {
+                        NamedAnalyzer normalizer = parserContext.getIndexAnalyzers().getNormalizer(propNode.toString());
+                        if (normalizer == null) {
+                            throw new MapperParsingException("normalizer [" + propNode.toString() + "] not found for field [" + name + "]");
+                        }
+                        builder.normalizer(normalizer);
+                    }
                     iterator.remove();
                 }
             }
@@ -133,21 +158,58 @@ public final class KeywordFieldMapper extends FieldMapper {
 
     public static final class KeywordFieldType extends StringFieldType {
 
-        public KeywordFieldType() {}
+        private NamedAnalyzer normalizer = null;
+
+        public KeywordFieldType() {
+            setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+            setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
+        }
 
         protected KeywordFieldType(KeywordFieldType ref) {
             super(ref);
+            this.normalizer = ref.normalizer;
         }
 
         public KeywordFieldType clone() {
             return new KeywordFieldType(this);
         }
 
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            return Objects.equals(normalizer, ((KeywordFieldType) o).normalizer);
+        }
+
+        @Override
+        public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts, boolean strict) {
+            super.checkCompatibility(otherFT, conflicts, strict);
+            KeywordFieldType other = (KeywordFieldType) otherFT;
+            if (Objects.equals(normalizer, other.normalizer) == false) {
+                conflicts.add("mapper [" + name() + "] has different [normalizer]");
+            }
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * super.hashCode() + Objects.hashCode(normalizer);
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
         }
 
+        public NamedAnalyzer normalizer() {
+            return normalizer;
+        }
+
+        public void setNormalizer(NamedAnalyzer normalizer) {
+            checkIfFrozen();
+            this.normalizer = normalizer;
+        }
+
         @Override
         public Query nullValueQuery() {
             if (nullValue() == null) {
@@ -171,13 +233,25 @@ public final class KeywordFieldMapper extends FieldMapper {
             BytesRef binaryValue = (BytesRef) value;
             return binaryValue.utf8ToString();
         }
+
+        @Override
+        protected BytesRef indexedValueForSearch(Object value) {
+            if (value == null) {
+                return null;
+            }
+            if (value instanceof BytesRef) {
+                value = ((BytesRef) value).utf8ToString();
+            }
+            return searchAnalyzer().normalize(name(), value.toString());
+        }
     }
 
     private Boolean includeInAll;
     private int ignoreAbove;
 
     protected KeywordFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
-                                int ignoreAbove, Boolean includeInAll, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+                                int ignoreAbove, Boolean includeInAll,
+                                Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;
         this.ignoreAbove = ignoreAbove;
@@ -196,6 +270,11 @@ public final class KeywordFieldMapper extends FieldMapper {
         return (KeywordFieldMapper) super.clone();
     }
 
+    @Override
+    public KeywordFieldType fieldType() {
+        return (KeywordFieldType) super.fieldType();
+    }
+
     // pkg-private for testing
     Boolean includeInAll() {
         return includeInAll;
@@ -203,7 +282,7 @@ public final class KeywordFieldMapper extends FieldMapper {
 
     @Override
     protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
-        final String value;
+        String value;
         if (context.externalValueSet()) {
             value = context.externalValue().toString();
         } else {
@@ -219,6 +298,27 @@ public final class KeywordFieldMapper extends FieldMapper {
             return;
         }
 
+        final NamedAnalyzer normalizer = fieldType().normalizer();
+        if (normalizer != null) {
+            try (TokenStream ts = normalizer.tokenStream(name(), value)) {
+                final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+                ts.reset();
+                if (ts.incrementToken() == false) {
+                  throw new IllegalStateException("The normalization token stream is "
+                      + "expected to produce exactly 1 token, but got 0 for analyzer "
+                      + normalizer + " and input \"" + value + "\"");
+                }
+                final String newValue = termAtt.toString();
+                if (ts.incrementToken()) {
+                  throw new IllegalStateException("The normalization token stream is "
+                      + "expected to produce exactly 1 token, but got 2+ for analyzer "
+                      + normalizer + " and input \"" + value + "\"");
+                }
+                ts.end();
+                value = newValue;
+            }
+        }
+
         if (context.includeInAll(includeInAll, this)) {
             context.allEntries().addText(fieldType().name(), value, fieldType().boost());
         }
@@ -263,5 +363,11 @@ public final class KeywordFieldMapper extends FieldMapper {
         if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
             builder.field("ignore_above", ignoreAbove);
         }
+
+        if (fieldType().normalizer() != null) {
+            builder.field("normalizer", fieldType().normalizer().name());
+        } else if (includeDefaults) {
+            builder.nullField("normalizer");
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java
index f5579751e5b..919f6590c14 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/LatLonPointFieldMapper.java
@@ -21,8 +21,13 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.document.LatLonPoint;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
@@ -120,6 +125,26 @@ public class LatLonPointFieldMapper extends BaseGeoPointFieldMapper {
             throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: ["
                 + name() + "]");
         }
+
+        @Override
+        public FieldStats.GeoPoint stats(IndexReader reader) throws IOException {
+            String field = name();
+            FieldInfo fi = org.apache.lucene.index.MultiFields.getMergedFieldInfos(reader).fieldInfo(name());
+            if (fi == null) {
+                return null;
+            }
+            final long size = PointValues.size(reader, field);
+            if (size == 0) {
+                return new FieldStats.GeoPoint(reader.maxDoc(), -1L, -1L, -1L, isSearchable(), isAggregatable());
+            }
+            final int docCount = PointValues.getDocCount(reader, field);
+            byte[] min = PointValues.getMinPackedValue(reader, field);
+            byte[] max = PointValues.getMaxPackedValue(reader, field);
+            GeoPoint minPt = new GeoPoint(GeoEncodingUtils.decodeLatitude(min, 0), GeoEncodingUtils.decodeLongitude(min, Integer.BYTES));
+            GeoPoint maxPt = new GeoPoint(GeoEncodingUtils.decodeLatitude(max, 0), GeoEncodingUtils.decodeLongitude(max, Integer.BYTES));
+            return new FieldStats.GeoPoint(reader.maxDoc(), docCount, -1L, size, isSearchable(), isAggregatable(),
+                minPt, maxPt);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 64f3f06a110..93f22d42a80 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -393,7 +393,7 @@ public abstract class MappedFieldType extends FieldType {
      * An enum used to describe the relation between the range of terms in a
      * shard when compared with a query range
      */
-    public static enum Relation {
+    public enum Relation {
         WITHIN,
         INTERSECTS,
         DISJOINT;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 384331c2d9e..db1060d3be6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -91,21 +90,17 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
 
             private final Version indexVersionCreated;
 
-            private final ParseFieldMatcher parseFieldMatcher;
-
             private final Supplier<QueryShardContext> queryShardContextSupplier;
 
             public ParserContext(String type, IndexAnalyzers indexAnalyzers, Function<String, SimilarityProvider> similarityLookupService,
                                  MapperService mapperService, Function<String, TypeParser> typeParsers,
-                                 Version indexVersionCreated, ParseFieldMatcher parseFieldMatcher,
-                                 Supplier<QueryShardContext> queryShardContextSupplier) {
+                                 Version indexVersionCreated, Supplier<QueryShardContext> queryShardContextSupplier) {
                 this.type = type;
                 this.indexAnalyzers = indexAnalyzers;
                 this.similarityLookupService = similarityLookupService;
                 this.mapperService = mapperService;
                 this.typeParsers = typeParsers;
                 this.indexVersionCreated = indexVersionCreated;
-                this.parseFieldMatcher = parseFieldMatcher;
                 this.queryShardContextSupplier = queryShardContextSupplier;
             }
 
@@ -133,10 +128,6 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
                 return indexVersionCreated;
             }
 
-            public ParseFieldMatcher parseFieldMatcher() {
-                return parseFieldMatcher;
-            }
-
             public Supplier<QueryShardContext> queryShardContextSupplier() {
                 return queryShardContextSupplier;
             }
@@ -156,7 +147,8 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
 
             static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
-                    super(in.type(), in.indexAnalyzers, in.similarityLookupService(), in.mapperService(), in.typeParsers(), in.indexVersionCreated(), in.parseFieldMatcher(), in.queryShardContextSupplier());
+                    super(in.type(), in.indexAnalyzers, in.similarityLookupService(), in.mapperService(), in.typeParsers(),
+                            in.indexVersionCreated(), in.queryShardContextSupplier());
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 1e3f96fbe2c..41b29f3c67f 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.ObjectHashSet;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
@@ -28,12 +29,16 @@ import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -51,6 +56,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -61,7 +67,6 @@ import java.util.stream.Collectors;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
 
 public class MapperService extends AbstractIndexComponent implements Closeable {
 
@@ -126,13 +131,14 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
 
     final MapperRegistry mapperRegistry;
 
-    public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
+    public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry,
                          SimilarityService similarityService, MapperRegistry mapperRegistry,
                          Supplier<QueryShardContext> queryShardContextSupplier) {
         super(indexSettings);
         this.indexAnalyzers = indexAnalyzers;
         this.fieldTypes = new FieldTypeLookup();
-        this.documentParser = new DocumentMapperParser(indexSettings, this, indexAnalyzers, similarityService, mapperRegistry, queryShardContextSupplier);
+        this.documentParser = new DocumentMapperParser(indexSettings, this, indexAnalyzers, xContentRegistry, similarityService,
+                mapperRegistry, queryShardContextSupplier);
         this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), p -> p.indexAnalyzer());
         this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer());
         this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
@@ -185,159 +191,246 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return this.documentParser;
     }
 
-    public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
+    /**
+     * Parses the mappings (formatted as JSON) into a map
+     */
+    public static Map<String, Object> parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception {
+        try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry, mappingSource)) {
             return parser.map();
         }
     }
 
+    /**
+     * Update mapping by only merging the metadata that is different between received and stored entries
+     */
     public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
         assert indexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index() + " but was " + indexMetaData.getIndex();
         // go over and add the relevant mappings (or update them)
+        final Set<String> existingMappers = new HashSet<>(mappers.keySet());
+        final Map<String, DocumentMapper> updatedEntries;
+        try {
+            // only update entries if needed
+            updatedEntries = internalMerge(indexMetaData, MergeReason.MAPPING_RECOVERY, true, true);
+        } catch (Exception e) {
+            logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("[{}] failed to apply mappings", index()), e);
+            throw e;
+        }
+
         boolean requireRefresh = false;
-        for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
-            MappingMetaData mappingMd = cursor.value;
-            String mappingType = mappingMd.type();
-            CompressedXContent mappingSource = mappingMd.source();
+
+        for (DocumentMapper documentMapper : updatedEntries.values()) {
+            String mappingType = documentMapper.type();
+            CompressedXContent incomingMappingSource = indexMetaData.mapping(mappingType).source();
+
+            String op = existingMappers.contains(mappingType) ? "updated" : "added";
+            if (logger.isDebugEnabled() && incomingMappingSource.compressed().length < 512) {
+                logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, incomingMappingSource.string());
+            } else if (logger.isTraceEnabled()) {
+                logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, incomingMappingSource.string());
+            } else {
+                logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op, mappingType);
+            }
+
             // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same
             // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the
             // merge version of it, which it does when refreshing the mappings), and warn log it.
-            try {
-                DocumentMapper existingMapper = documentMapper(mappingType);
+            if (documentMapper(mappingType).mappingSource().equals(incomingMappingSource) == false) {
+                logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(), mappingType,
+                    incomingMappingSource, documentMapper(mappingType).mappingSource());
 
-                if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) {
-                    String op = existingMapper == null ? "adding" : "updating";
-                    if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) {
-                        logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
-                    } else if (logger.isTraceEnabled()) {
-                        logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
-                    } else {
-                        logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op,
-                            mappingType);
-                    }
-                    merge(mappingType, mappingSource, MergeReason.MAPPING_RECOVERY, true);
-                    if (!documentMapper(mappingType).mappingSource().equals(mappingSource)) {
-                        logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(),
-                            mappingType, mappingSource, documentMapper(mappingType).mappingSource());
-                        requireRefresh = true;
-                    }
-                }
-            } catch (Exception e) {
-                logger.warn(
-                    (org.apache.logging.log4j.util.Supplier<?>)
-                        () -> new ParameterizedMessage("[{}] failed to add mapping [{}], source [{}]", index(), mappingType, mappingSource),
-                    e);
-                throw e;
+                requireRefresh = true;
             }
         }
+
         return requireRefresh;
     }
 
-    //TODO: make this atomic
-    public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
-        // first, add the default mapping
-        if (mappings.containsKey(DEFAULT_MAPPING)) {
-            try {
-                this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
-            } catch (Exception e) {
-                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
-            }
-        }
+    public void merge(Map<String, Map<String, Object>> mappings, MergeReason reason, boolean updateAllTypes) {
+        Map<String, CompressedXContent> mappingSourcesCompressed = new LinkedHashMap<>(mappings.size());
         for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
-            if (entry.getKey().equals(DEFAULT_MAPPING)) {
-                continue;
-            }
             try {
-                // apply the default here, its the first time we parse it
-                this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
+                mappingSourcesCompressed.put(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()));
             } catch (Exception e) {
                 throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
             }
         }
+
+        internalMerge(mappingSourcesCompressed, reason, updateAllTypes);
+    }
+
+    public void merge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes) {
+        internalMerge(indexMetaData, reason, updateAllTypes, false);
     }
 
     public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
-        if (DEFAULT_MAPPING.equals(type)) {
+        return internalMerge(Collections.singletonMap(type, mappingSource), reason, updateAllTypes).get(type);
+    }
+
+    private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes,
+                                                                   boolean onlyUpdateIfNeeded) {
+        Map<String, CompressedXContent> map = new LinkedHashMap<>();
+        for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
+            MappingMetaData mappingMetaData = cursor.value;
+            if (onlyUpdateIfNeeded) {
+                DocumentMapper existingMapper = documentMapper(mappingMetaData.type());
+                if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) {
+                    map.put(mappingMetaData.type(), mappingMetaData.source());
+                }
+            } else {
+                map.put(mappingMetaData.type(), mappingMetaData.source());
+            }
+        }
+        return internalMerge(map, reason, updateAllTypes);
+    }
+
+    private synchronized Map<String, DocumentMapper> internalMerge(Map<String, CompressedXContent> mappings, MergeReason reason, boolean updateAllTypes) {
+        DocumentMapper defaultMapper = null;
+        String defaultMappingSource = null;
+
+        if (mappings.containsKey(DEFAULT_MAPPING)) {
             // verify we can parse it
             // NOTE: never apply the default here
-            DocumentMapper mapper = documentParser.parse(type, mappingSource);
-            // still add it as a document mapper so we have it registered and, for example, persisted back into
-            // the cluster meta data if needed, or checked for existence
-            synchronized (this) {
-                mappers = newMapBuilder(mappers).put(type, mapper).map();
+            try {
+                defaultMapper = documentParser.parse(DEFAULT_MAPPING, mappings.get(DEFAULT_MAPPING));
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
             }
             try {
-                defaultMappingSource = mappingSource.string();
+                defaultMappingSource = mappings.get(DEFAULT_MAPPING).string();
             } catch (IOException e) {
                 throw new ElasticsearchGenerationException("failed to un-compress", e);
             }
-            return mapper;
+        }
+
+        final String defaultMappingSourceOrLastStored;
+        if (defaultMappingSource != null) {
+            defaultMappingSourceOrLastStored = defaultMappingSource;
         } else {
-            synchronized (this) {
-                final boolean applyDefault =
-                        // the default was already applied if we are recovering
-                        reason != MergeReason.MAPPING_RECOVERY
-                        // only apply the default mapping if we don't have the type yet
-                        && mappers.containsKey(type) == false;
-                DocumentMapper mergeWith = parse(type, mappingSource, applyDefault);
-                return merge(mergeWith, reason, updateAllTypes);
+            defaultMappingSourceOrLastStored = this.defaultMappingSource;
+        }
+
+        List<DocumentMapper> documentMappers = new ArrayList<>();
+        for (Map.Entry<String, CompressedXContent> entry : mappings.entrySet()) {
+            String type = entry.getKey();
+            if (type.equals(DEFAULT_MAPPING)) {
+                continue;
+            }
+
+            final boolean applyDefault =
+                // the default was already applied if we are recovering
+                reason != MergeReason.MAPPING_RECOVERY
+                    // only apply the default mapping if we don't have the type yet
+                    && mappers.containsKey(type) == false;
+
+            try {
+                DocumentMapper documentMapper =
+                    documentParser.parse(type, entry.getValue(), applyDefault ? defaultMappingSourceOrLastStored : null);
+                documentMappers.add(documentMapper);
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
             }
         }
+
+        return internalMerge(defaultMapper, defaultMappingSource, documentMappers, reason, updateAllTypes);
     }
 
-    private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason reason, boolean updateAllTypes) {
-        if (mapper.type().length() == 0) {
-            throw new InvalidTypeNameException("mapping type name is empty");
-        }
-        if (mapper.type().length() > 255) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]");
-        }
-        if (mapper.type().charAt(0) == '_') {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] can't start with '_'");
-        }
-        if (mapper.type().contains("#")) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include '#' in it");
-        }
-        if (mapper.type().contains(",")) {
-            throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it");
-        }
-        if (mapper.type().equals(mapper.parentFieldMapper().type())) {
-            throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
-        }
-        if (typeNameStartsWithIllegalDot(mapper)) {
-            throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'");
-        }
-
-        // 1. compute the merged DocumentMapper
-        DocumentMapper oldMapper = mappers.get(mapper.type());
-        DocumentMapper newMapper;
-        if (oldMapper != null) {
-            newMapper = oldMapper.merge(mapper.mapping(), updateAllTypes);
-        } else {
-            newMapper = mapper;
-        }
-
-        // 2. check basic sanity of the new mapping
-        List<ObjectMapper> objectMappers = new ArrayList<>();
-        List<FieldMapper> fieldMappers = new ArrayList<>();
-        Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
-        MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
-        checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers);
-        checkObjectsCompatibility(objectMappers, updateAllTypes);
-
-        // 3. update lookup data-structures
-        // this will in particular make sure that the merged fields are compatible with other types
-        FieldTypeLookup fieldTypes = this.fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, updateAllTypes);
-
+    private synchronized Map<String, DocumentMapper> internalMerge(@Nullable DocumentMapper defaultMapper, @Nullable String defaultMappingSource,
+                                                                   List<DocumentMapper> documentMappers, MergeReason reason, boolean updateAllTypes) {
         boolean hasNested = this.hasNested;
-        Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers);
-        for (ObjectMapper objectMapper : objectMappers) {
-            fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
-            if (objectMapper.nested().isNested()) {
-                hasNested = true;
-            }
+        boolean allEnabled = this.allEnabled;
+        Map<String, ObjectMapper> fullPathObjectMappers = this.fullPathObjectMappers;
+        FieldTypeLookup fieldTypes = this.fieldTypes;
+        Set<String> parentTypes = this.parentTypes;
+        Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
+
+        Map<String, DocumentMapper> results = new LinkedHashMap<>(documentMappers.size() + 1);
+
+        if (defaultMapper != null) {
+            assert defaultMapper.type().equals(DEFAULT_MAPPING);
+            mappers.put(DEFAULT_MAPPING, defaultMapper);
+            results.put(DEFAULT_MAPPING, defaultMapper);
+        }
+
+        for (DocumentMapper mapper : documentMappers) {
+            // check naming
+            if (mapper.type().length() == 0) {
+                throw new InvalidTypeNameException("mapping type name is empty");
+            }
+            if (mapper.type().length() > 255) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] is too long; limit is length 255 but was [" + mapper.type().length() + "]");
+            }
+            if (mapper.type().charAt(0) == '_') {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] can't start with '_'");
+            }
+            if (mapper.type().contains("#")) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include '#' in it");
+            }
+            if (mapper.type().contains(",")) {
+                throw new InvalidTypeNameException("mapping type name [" + mapper.type() + "] should not include ',' in it");
+            }
+            if (mapper.type().equals(mapper.parentFieldMapper().type())) {
+                throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
+            }
+            if (typeNameStartsWithIllegalDot(mapper)) {
+                throw new IllegalArgumentException("mapping type name [" + mapper.type() + "] must not start with a '.'");
+            }
+
+            // compute the merged DocumentMapper
+            DocumentMapper oldMapper = mappers.get(mapper.type());
+            DocumentMapper newMapper;
+            if (oldMapper != null) {
+                newMapper = oldMapper.merge(mapper.mapping(), updateAllTypes);
+            } else {
+                newMapper = mapper;
+            }
+
+            // check basic sanity of the new mapping
+            List<ObjectMapper> objectMappers = new ArrayList<>();
+            List<FieldMapper> fieldMappers = new ArrayList<>();
+            Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
+            MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
+            checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers, fullPathObjectMappers, fieldTypes);
+            checkObjectsCompatibility(objectMappers, updateAllTypes, fullPathObjectMappers);
+            checkPartitionedIndexConstraints(newMapper);
+
+            // update lookup data-structures
+            // this will in particular make sure that the merged fields are compatible with other types
+            fieldTypes = fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, updateAllTypes);
+
+            for (ObjectMapper objectMapper : objectMappers) {
+                if (fullPathObjectMappers == this.fullPathObjectMappers) {
+                    fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers);
+                }
+                fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
+
+                if (objectMapper.nested().isNested()) {
+                    hasNested = true;
+                }
+            }
+
+            if (reason == MergeReason.MAPPING_UPDATE) {
+                // this check will only be performed on the master node when there is
+                // a call to the update mapping API. For all other cases like
+                // the master node restoring mappings from disk or data nodes
+                // deserializing cluster state that was sent by the master node,
+                // this check will be skipped.
+                checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
+            }
+
+            if (oldMapper == null && newMapper.parentFieldMapper().active()) {
+                if (parentTypes == this.parentTypes) {
+                    parentTypes = new HashSet<>(this.parentTypes);
+                }
+                parentTypes.add(mapper.parentFieldMapper().type());
+            }
+
+            // this is only correct because types cannot be removed and we do not
+            // allow to disable an existing _all field
+            allEnabled |= mapper.allFieldMapper().enabled();
+
+            results.put(newMapper.type(), newMapper);
+            mappers.put(newMapper.type(), newMapper);
         }
-        fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
 
         if (reason == MergeReason.MAPPING_UPDATE) {
             // this check will only be performed on the master node when there is
@@ -346,45 +439,46 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
             // deserializing cluster state that was sent by the master node,
             // this check will be skipped.
             checkNestedFieldsLimit(fullPathObjectMappers);
-            checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
             checkDepthLimit(fullPathObjectMappers.keySet());
         }
 
-        Set<String> parentTypes = this.parentTypes;
-        if (oldMapper == null && newMapper.parentFieldMapper().active()) {
-            parentTypes = new HashSet<>(parentTypes.size() + 1);
-            parentTypes.addAll(this.parentTypes);
-            parentTypes.add(mapper.parentFieldMapper().type());
-            parentTypes = Collections.unmodifiableSet(parentTypes);
-        }
-
-        Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
-        mappers.put(newMapper.type(), newMapper);
         for (Map.Entry<String, DocumentMapper> entry : mappers.entrySet()) {
             if (entry.getKey().equals(DEFAULT_MAPPING)) {
                 continue;
             }
-            DocumentMapper m = entry.getValue();
+            DocumentMapper documentMapper = entry.getValue();
             // apply changes to the field types back
-            m = m.updateFieldType(fieldTypes.fullNameToFieldType);
-            entry.setValue(m);
+            DocumentMapper updatedDocumentMapper = documentMapper.updateFieldType(fieldTypes.fullNameToFieldType);
+            if (updatedDocumentMapper != documentMapper) {
+                // update both mappers and result
+                entry.setValue(updatedDocumentMapper);
+                if (results.containsKey(updatedDocumentMapper.type())) {
+                    results.put(updatedDocumentMapper.type(), updatedDocumentMapper);
+                }
+            }
         }
-        mappers = Collections.unmodifiableMap(mappers);
 
-        // 4. commit the change
+        // make structures immutable
+        mappers = Collections.unmodifiableMap(mappers);
+        results = Collections.unmodifiableMap(results);
+        parentTypes = Collections.unmodifiableSet(parentTypes);
+        fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
+
+        // commit the change
+        if (defaultMappingSource != null) {
+            this.defaultMappingSource = defaultMappingSource;
+        }
         this.mappers = mappers;
         this.fieldTypes = fieldTypes;
         this.hasNested = hasNested;
         this.fullPathObjectMappers = fullPathObjectMappers;
         this.parentTypes = parentTypes;
-        // this is only correct because types cannot be removed and we do not
-        // allow to disable an existing _all field
-        this.allEnabled |= mapper.allFieldMapper().enabled();
+        this.allEnabled = allEnabled;
 
-        assert assertSerialization(newMapper);
         assert assertMappersShareSameFieldType();
+        assert results.values().stream().allMatch(this::assertSerialization);
 
-        return newMapper;
+        return results;
     }
 
     private boolean assertMappersShareSameFieldType() {
@@ -421,8 +515,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         return true;
     }
 
-    private void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
-        assert Thread.holdsLock(this);
+    private static void checkFieldUniqueness(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers,
+                                             Map<String, ObjectMapper> fullPathObjectMappers, FieldTypeLookup fieldTypes) {
 
         // first check within mapping
         final Set<String> objectFullNames = new HashSet<>();
@@ -459,9 +553,8 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         }
     }
 
-    private void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers, boolean updateAllTypes) {
-        assert Thread.holdsLock(this);
-
+    private static void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers, boolean updateAllTypes,
+                                                  Map<String, ObjectMapper> fullPathObjectMappers) {
         for (ObjectMapper newObjectMapper : objectMappers) {
             ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
             if (existingObjectMapper != null) {
@@ -513,6 +606,20 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
         }
     }
 
+    private void checkPartitionedIndexConstraints(DocumentMapper newMapper) {
+        if (indexSettings.getIndexMetaData().isRoutingPartitionedIndex()) {
+            if (newMapper.parentFieldMapper().active()) {
+                throw new IllegalArgumentException("mapping type name [" + newMapper.type() + "] cannot have a "
+                        + "_parent field for the partitioned index [" + indexSettings.getIndex().getName() + "]");
+            }
+
+            if (!newMapper.routingFieldMapper().required()) {
+                throw new IllegalArgumentException("mapping type [" + newMapper.type() + "] must have routing "
+                        + "required for partitioned index [" + indexSettings.getIndex().getName() + "]");
+            }
+        }
+    }
+
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
index 9e615e10169..e9043b3c754 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -39,6 +39,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -54,6 +55,7 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -145,13 +147,13 @@ public class NumberFieldMapper extends FieldMapper {
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(type.parse(propNode));
+                    builder.nullValue(type.parse(propNode, false));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name,"ignore_malformed", propNode, parserContext));
                     iterator.remove();
                 } else if (propName.equals("coerce")) {
-                    builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext));
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode, parserContext));
                     iterator.remove();
                 }
             }
@@ -162,8 +164,8 @@ public class NumberFieldMapper extends FieldMapper {
     public enum NumberType {
         HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
             @Override
-            Float parse(Object value) {
-                return (Float) FLOAT.parse(value);
+            Float parse(Object value, boolean coerce) {
+                return (Float) FLOAT.parse(value, false);
             }
 
             @Override
@@ -173,7 +175,7 @@ public class NumberFieldMapper extends FieldMapper {
 
             @Override
             Query termQuery(String field, Object value) {
-                float v = parse(value);
+                float v = parse(value, false);
                 return HalfFloatPoint.newExactQuery(field, v);
             }
 
@@ -181,7 +183,7 @@ public class NumberFieldMapper extends FieldMapper {
             Query termsQuery(String field, List<Object> values) {
                 float[] v = new float[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return HalfFloatPoint.newSetQuery(field, v);
             }
@@ -192,16 +194,16 @@ public class NumberFieldMapper extends FieldMapper {
                 float l = Float.NEGATIVE_INFINITY;
                 float u = Float.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower) {
-                        l = Math.nextDown(l);
+                        l = HalfFloatPoint.nextDown(l);
                     }
                     l = HalfFloatPoint.nextUp(l);
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper) {
-                        u = Math.nextUp(u);
+                        u = HalfFloatPoint.nextUp(u);
                     }
                     u = HalfFloatPoint.nextDown(u);
                 }
@@ -246,7 +248,7 @@ public class NumberFieldMapper extends FieldMapper {
         },
         FLOAT("float", NumericType.FLOAT) {
             @Override
-            Float parse(Object value) {
+            Float parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     return ((Number) value).floatValue();
                 }
@@ -263,7 +265,7 @@ public class NumberFieldMapper extends FieldMapper {
 
             @Override
             Query termQuery(String field, Object value) {
-                float v = parse(value);
+                float v = parse(value, false);
                 return FloatPoint.newExactQuery(field, v);
             }
 
@@ -271,7 +273,7 @@ public class NumberFieldMapper extends FieldMapper {
             Query termsQuery(String field, List<Object> values) {
                 float[] v = new float[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return FloatPoint.newSetQuery(field, v);
             }
@@ -282,15 +284,15 @@ public class NumberFieldMapper extends FieldMapper {
                 float l = Float.NEGATIVE_INFINITY;
                 float u = Float.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower == false) {
-                        l = Math.nextUp(l);
+                        l = FloatPoint.nextUp(l);
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper == false) {
-                        u = Math.nextDown(u);
+                        u = FloatPoint.nextDown(u);
                     }
                 }
                 return FloatPoint.newRangeQuery(field, l, u);
@@ -334,7 +336,7 @@ public class NumberFieldMapper extends FieldMapper {
         },
         DOUBLE("double", NumericType.DOUBLE) {
             @Override
-            Double parse(Object value) {
+            Double parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     return ((Number) value).doubleValue();
                 }
@@ -351,7 +353,7 @@ public class NumberFieldMapper extends FieldMapper {
 
             @Override
             Query termQuery(String field, Object value) {
-                double v = parse(value);
+                double v = parse(value, false);
                 return DoublePoint.newExactQuery(field, v);
             }
 
@@ -359,7 +361,7 @@ public class NumberFieldMapper extends FieldMapper {
             Query termsQuery(String field, List<Object> values) {
                 double[] v = new double[values.size()];
                 for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                    v[i] = parse(values.get(i), false);
                 }
                 return DoublePoint.newSetQuery(field, v);
             }
@@ -370,15 +372,15 @@ public class NumberFieldMapper extends FieldMapper {
                 double l = Double.NEGATIVE_INFINITY;
                 double u = Double.POSITIVE_INFINITY;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
+                    l = parse(lowerTerm, false);
                     if (includeLower == false) {
-                        l = Math.nextUp(l);
+                        l = DoublePoint.nextUp(l);
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
+                    u = parse(upperTerm, false);
                     if (includeUpper == false) {
-                        u = Math.nextDown(u);
+                        u = DoublePoint.nextDown(u);
                     }
                 }
                 return DoublePoint.newRangeQuery(field, l, u);
@@ -422,13 +424,13 @@ public class NumberFieldMapper extends FieldMapper {
         },
         BYTE("byte", NumericType.BYTE) {
             @Override
-            Byte parse(Object value) {
+            Byte parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     double doubleValue = ((Number) value).doubleValue();
                     if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) {
                         throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
                     }
-                    if (doubleValue % 1 != 0) {
+                    if (!coerce && doubleValue % 1 != 0) {
                         throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
                     }
                     return ((Number) value).byteValue();
@@ -483,13 +485,13 @@ public class NumberFieldMapper extends FieldMapper {
         },
         SHORT("short", NumericType.SHORT) {
             @Override
-            Short parse(Object value) {
+            Short parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     double doubleValue = ((Number) value).doubleValue();
                     if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) {
                         throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
                     }
-                    if (doubleValue % 1 != 0) {
+                    if (!coerce && doubleValue % 1 != 0) {
                         throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
                     }
                     return ((Number) value).shortValue();
@@ -544,13 +546,13 @@ public class NumberFieldMapper extends FieldMapper {
         },
         INTEGER("integer", NumericType.INT) {
             @Override
-            Integer parse(Object value) {
+            Integer parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     double doubleValue = ((Number) value).doubleValue();
                     if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) {
                         throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer");
                     }
-                    if (doubleValue % 1 != 0) {
+                    if (!coerce && doubleValue % 1 != 0) {
                         throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
                     }
                     return ((Number) value).intValue();
@@ -568,15 +570,30 @@ public class NumberFieldMapper extends FieldMapper {
 
             @Override
             Query termQuery(String field, Object value) {
-                int v = parse(value);
+                if (hasDecimalPart(value)) {
+                    return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
+                }
+                int v = parse(value, true);
                 return IntPoint.newExactQuery(field, v);
             }
 
             @Override
             Query termsQuery(String field, List<Object> values) {
                 int[] v = new int[values.size()];
-                for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                int upTo = 0;
+
+                for (int i = 0; i < values.size(); i++) {
+                    Object value = values.get(i);
+                    if (!hasDecimalPart(value)) {
+                        v[upTo++] = parse(value, true);
+                    }
+                }
+
+                if (upTo == 0) {
+                    return Queries.newMatchNoDocsQuery("All values have a decimal part");
+                }
+                if (upTo != v.length) {
+                    v = Arrays.copyOf(v, upTo);
                 }
                 return IntPoint.newSetQuery(field, v);
             }
@@ -587,8 +604,15 @@ public class NumberFieldMapper extends FieldMapper {
                 int l = Integer.MIN_VALUE;
                 int u = Integer.MAX_VALUE;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
-                    if (includeLower == false) {
+                    l = parse(lowerTerm, true);
+                    // if the lower bound is decimal:
+                    // - if the bound is positive then we increment it:
+                    //      if lowerTerm=1.5 then the (inclusive) bound becomes 2
+                    // - if the bound is negative then we leave it as is:
+                    //      if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
+                    boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
+                    if ((lowerTermHasDecimalPart == false && includeLower == false) ||
+                            (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
                         if (l == Integer.MAX_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -596,8 +620,10 @@ public class NumberFieldMapper extends FieldMapper {
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
-                    if (includeUpper == false) {
+                    u = parse(upperTerm, true);
+                    boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
+                    if ((upperTermHasDecimalPart == false && includeUpper == false) ||
+                            (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
                         if (u == Integer.MIN_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -644,13 +670,13 @@ public class NumberFieldMapper extends FieldMapper {
         },
         LONG("long", NumericType.LONG) {
             @Override
-            Long parse(Object value) {
+            Long parse(Object value, boolean coerce) {
                 if (value instanceof Number) {
                     double doubleValue = ((Number) value).doubleValue();
                     if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
                         throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
                     }
-                    if (doubleValue % 1 != 0) {
+                    if (!coerce && doubleValue % 1 != 0) {
                         throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
                     }
                     return ((Number) value).longValue();
@@ -668,15 +694,30 @@ public class NumberFieldMapper extends FieldMapper {
 
             @Override
             Query termQuery(String field, Object value) {
-                long v = parse(value);
+                if (hasDecimalPart(value)) {
+                    return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
+                }
+                long v = parse(value, true);
                 return LongPoint.newExactQuery(field, v);
             }
 
             @Override
             Query termsQuery(String field, List<Object> values) {
                 long[] v = new long[values.size()];
-                for (int i = 0; i < values.size(); ++i) {
-                    v[i] = parse(values.get(i));
+                int upTo = 0;
+
+                for (int i = 0; i < values.size(); i++) {
+                    Object value = values.get(i);
+                    if (!hasDecimalPart(value)) {
+                        v[upTo++] = parse(value, true);
+                    }
+                }
+
+                if (upTo == 0) {
+                    return Queries.newMatchNoDocsQuery("All values have a decimal part");
+                }
+                if (upTo != v.length) {
+                    v = Arrays.copyOf(v, upTo);
                 }
                 return LongPoint.newSetQuery(field, v);
             }
@@ -687,8 +728,15 @@ public class NumberFieldMapper extends FieldMapper {
                 long l = Long.MIN_VALUE;
                 long u = Long.MAX_VALUE;
                 if (lowerTerm != null) {
-                    l = parse(lowerTerm);
-                    if (includeLower == false) {
+                    l = parse(lowerTerm, true);
+                    // if the lower bound is decimal:
+                    // - if the bound is positive then we increment it:
+                    //      if lowerTerm=1.5 then the (inclusive) bound becomes 2
+                    // - if the bound is negative then we leave it as is:
+                    //      if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue
+                    boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm);
+                    if ((lowerTermHasDecimalPart == false && includeLower == false) ||
+                            (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) {
                         if (l == Long.MAX_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -696,8 +744,10 @@ public class NumberFieldMapper extends FieldMapper {
                     }
                 }
                 if (upperTerm != null) {
-                    u = parse(upperTerm);
-                    if (includeUpper == false) {
+                    u = parse(upperTerm, true);
+                    boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm);
+                    if ((upperTermHasDecimalPart == false && includeUpper == false) ||
+                            (upperTermHasDecimalPart && signum(upperTerm) < 0)) {
                         if (u == Long.MIN_VALUE) {
                             return new MatchNoDocsQuery();
                         }
@@ -755,7 +805,7 @@ public class NumberFieldMapper extends FieldMapper {
         public final String typeName() {
             return name;
         }
-        /** Get the associated numerit type */
+        /** Get the associated numeric type */
         final NumericType numericType() {
             return numericType;
         }
@@ -764,7 +814,7 @@ public class NumberFieldMapper extends FieldMapper {
         abstract Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
                                   boolean includeLower, boolean includeUpper);
         abstract Number parse(XContentParser parser, boolean coerce) throws IOException;
-        abstract Number parse(Object value);
+        abstract Number parse(Object value, boolean coerce);
         public abstract List<Field> createFields(String name, Number value, boolean indexed,
                                                  boolean docValued, boolean stored);
         abstract FieldStats<? extends Number> stats(IndexReader reader, String fieldName,
@@ -772,6 +822,38 @@ public class NumberFieldMapper extends FieldMapper {
         Number valueForSearch(Number value) {
             return value;
         }
+
+        /**
+         * Returns true if the object is a number and has a decimal part
+         */
+        boolean hasDecimalPart(Object number) {
+            if (number instanceof Number) {
+                double doubleValue = ((Number) number).doubleValue();
+                return doubleValue % 1 != 0;
+            }
+            if (number instanceof BytesRef) {
+                number = ((BytesRef) number).utf8ToString();
+            }
+            if (number instanceof String) {
+                return Double.parseDouble((String) number) % 1 != 0;
+            }
+            return false;
+        }
+
+        /**
+         * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0
+         */
+        double signum(Object value) {
+            if (value instanceof Number) {
+                double doubleValue = ((Number) value).doubleValue();
+                return Math.signum(doubleValue);
+            }
+            if (value instanceof BytesRef) {
+                value = ((BytesRef) value).utf8ToString();
+            }
+            return Math.signum(Double.parseDouble(value.toString()));
+        }
+
     }
 
     public static final class NumberFieldType extends MappedFieldType {
@@ -942,7 +1024,7 @@ public class NumberFieldMapper extends FieldMapper {
         }
 
         if (numericValue == null) {
-            numericValue = fieldType().type.parse(value);
+            numericValue = fieldType().type.parse(value, coerce.value());
         }
 
         if (includeInAll) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
index 60c264e5706..1d3c4791b65 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -42,8 +42,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 public class ObjectMapper extends Mapper implements Cloneable {
 
     public static final String CONTENT_TYPE = "object";
@@ -55,7 +53,7 @@ public class ObjectMapper extends Mapper implements Cloneable {
         public static final Dynamic DYNAMIC = null; // not set, inherited from root
     }
 
-    public static enum Dynamic {
+    public enum Dynamic {
         TRUE,
         FALSE,
         STRICT
@@ -167,7 +165,7 @@ public class ObjectMapper extends Mapper implements Cloneable {
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             ObjectMapper.Builder builder = new Builder(name);
-            parseNested(name, node, builder);
+            parseNested(name, node, builder, parserContext);
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = entry.getKey();
@@ -185,11 +183,12 @@ public class ObjectMapper extends Mapper implements Cloneable {
                 if (value.equalsIgnoreCase("strict")) {
                     builder.dynamic(Dynamic.STRICT);
                 } else {
-                    builder.dynamic(lenientNodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
+                    boolean dynamic = TypeParsers.nodeBooleanValue(fieldName, "dynamic", fieldNode, parserContext);
+                    builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
                 }
                 return true;
             } else if (fieldName.equals("enabled")) {
-                builder.enabled(lenientNodeBooleanValue(fieldNode));
+                builder.enabled(TypeParsers.nodeBooleanValue(fieldName, "enabled", fieldNode, parserContext));
                 return true;
             } else if (fieldName.equals("properties")) {
                 if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
@@ -201,13 +200,14 @@ public class ObjectMapper extends Mapper implements Cloneable {
                 }
                 return true;
             } else if (fieldName.equals("include_in_all")) {
-                builder.includeInAll(lenientNodeBooleanValue(fieldNode));
+                builder.includeInAll(TypeParsers.nodeBooleanValue(fieldName, "include_in_all", fieldNode, parserContext));
                 return true;
             }
             return false;
         }
 
-        protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder) {
+        protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder,
+                                          ParserContext parserContext) {
             boolean nested = false;
             boolean nestedIncludeInParent = false;
             boolean nestedIncludeInRoot = false;
@@ -224,12 +224,12 @@ public class ObjectMapper extends Mapper implements Cloneable {
             }
             fieldNode = node.get("include_in_parent");
             if (fieldNode != null) {
-                nestedIncludeInParent = lenientNodeBooleanValue(fieldNode);
+                nestedIncludeInParent = TypeParsers.nodeBooleanValue(name, "include_in_parent", fieldNode, parserContext);
                 node.remove("include_in_parent");
             }
             fieldNode = node.get("include_in_root");
             if (fieldNode != null) {
-                nestedIncludeInRoot = lenientNodeBooleanValue(fieldNode);
+                nestedIncludeInRoot = TypeParsers.nodeBooleanValue(name, "include_in_root", fieldNode, parserContext);
                 node.remove("include_in_root");
             }
             if (nested) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
index d3f9eb22b72..62877567c3d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java
@@ -114,7 +114,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
                 if (fieldName.equals("type")) {
                     builder.type(fieldNode.toString());
                     iterator.remove();
-                } else if (parserContext.parseFieldMatcher().match(fieldName, FIELDDATA)) {
+                } else if (FIELDDATA.match(fieldName)) {
                     // for bw compat only
                     Map<String, String> fieldDataSettings = SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(fieldNode, "fielddata"));
                     if (fieldDataSettings.containsKey("loading")) {
@@ -122,7 +122,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
                     }
                     iterator.remove();
                 } else if (fieldName.equals("eager_global_ordinals")) {
-                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(fieldNode));
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(fieldNode, "eager_global_ordinals"));
                     iterator.remove();
                 }
             }
@@ -152,7 +152,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
 
         final String documentType;
 
-        public ParentFieldType() {
+        ParentFieldType() {
             documentType = null;
             setEagerGlobalOrdinals(true);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
index db8bdf9df7f..f7d5804be0d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
@@ -22,8 +22,8 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 
 import java.util.List;
 
@@ -43,6 +43,7 @@ public class ParsedDocument {
     private final List<Document> documents;
 
     private BytesReference source;
+    private XContentType xContentType;
 
     private Mapping dynamicMappingsUpdate;
 
@@ -55,6 +56,7 @@ public class ParsedDocument {
                           String routing,
                           List<Document> documents,
                           BytesReference source,
+                          XContentType xContentType,
                           Mapping dynamicMappingsUpdate) {
         this.version = version;
         this.seqID = seqID;
@@ -65,6 +67,7 @@ public class ParsedDocument {
         this.documents = documents;
         this.source = source;
         this.dynamicMappingsUpdate = dynamicMappingsUpdate;
+        this.xContentType = xContentType;
     }
 
     public BytesRef uid() {
@@ -105,8 +108,13 @@ public class ParsedDocument {
         return this.source;
     }
 
-    public void setSource(BytesReference source) {
+    public XContentType getXContentType() {
+        return this.xContentType;
+    }
+
+    public void setSource(BytesReference source, XContentType xContentType) {
         this.source = source;
+        this.xContentType = xContentType;
     }
 
     public ParsedDocument parent(String parent) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
index 214cc52c3ee..f270ae74ca9 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java
@@ -166,7 +166,7 @@ public class RangeFieldMapper extends FieldMapper {
                     throw new MapperParsingException("Property [null_value] is not supported for [" + this.type.name
                             + "] field types.");
                 } else if (propName.equals("coerce")) {
-                    builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext));
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode, parserContext));
                     iterator.remove();
                 } else if (propName.equals("locale")) {
                     builder.locale(LocaleUtils.parse(propNode.toString()));
@@ -405,12 +405,14 @@ public class RangeFieldMapper extends FieldMapper {
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
 
-        if (includeDefaults || (fieldType().dateTimeFormatter() != null
-                && fieldType().dateTimeFormatter().format().equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format()) == false)) {
+        if (fieldType().rangeType == RangeType.DATE
+                && (includeDefaults || (fieldType().dateTimeFormatter() != null
+                && fieldType().dateTimeFormatter().format().equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.format()) == false))) {
             builder.field("format", fieldType().dateTimeFormatter().format());
         }
-        if (includeDefaults || (fieldType().dateTimeFormatter() != null
-                && fieldType().dateTimeFormatter().locale() != Locale.ROOT)) {
+        if (fieldType().rangeType == RangeType.DATE
+                && (includeDefaults || (fieldType().dateTimeFormatter() != null
+                && fieldType().dateTimeFormatter().locale() != Locale.ROOT))) {
             builder.field("locale", fieldType().dateTimeFormatter().locale());
         }
         if (includeDefaults || coerce.explicit()) {
@@ -728,8 +730,8 @@ public class RangeFieldMapper extends FieldMapper {
         public Query rangeQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo,
                 ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser dateMathParser,
                 QueryShardContext context) {
-            Number lower = from == null ? minValue() : numberType.parse(from);
-            Number upper = to == null ? maxValue() : numberType.parse(to);
+            Number lower = from == null ? minValue() : numberType.parse(from, false);
+            Number upper = to == null ? maxValue() : numberType.parse(to, false);
             if (relation == ShapeRelation.WITHIN) {
                 return withinQuery(field, lower, upper, includeFrom, includeTo);
             } else if (relation == ShapeRelation.CONTAINS) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
index 0ae7ce33581..c51f29072c1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -150,10 +150,10 @@ public class RootObjectMapper extends ObjectMapper {
                 builder.dynamicTemplates(templates);
                 return true;
             } else if (fieldName.equals("date_detection")) {
-                ((Builder) builder).dateDetection = new Explicit<>(nodeBooleanValue(fieldNode), true);
+                builder.dateDetection = new Explicit<>(nodeBooleanValue(fieldNode, "date_detection"), true);
                 return true;
             } else if (fieldName.equals("numeric_detection")) {
-                ((Builder) builder).numericDetection = new Explicit<>(nodeBooleanValue(fieldNode), true);
+                builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
                 return true;
             }
             return false;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
index 8640bfaa35b..88679d910b1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java
@@ -32,8 +32,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 public class RoutingFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_routing";
@@ -86,7 +84,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("required")) {
-                    builder.required(lenientNodeBooleanValue(fieldNode));
+                    builder.required(TypeParsers.nodeBooleanValue(name, "required", fieldNode, parserContext));
                     iterator.remove();
                 }
             }
@@ -107,7 +105,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
 
     static final class RoutingFieldType extends TermBasedFieldType {
 
-        public RoutingFieldType() {
+        RoutingFieldType() {
         }
 
         protected RoutingFieldType(RoutingFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
index 2a76aa1addd..cf2564f0382 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java
@@ -144,16 +144,16 @@ public class ScaledFloatFieldMapper extends FieldMapper {
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode));
+                    builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false));
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
-                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
+                    builder.ignoreMalformed(TypeParsers.nodeBooleanValue(name, "ignore_malformed", propNode, parserContext));
                     iterator.remove();
                 } else if (propName.equals("coerce")) {
-                    builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext));
+                    builder.coerce(TypeParsers.nodeBooleanValue(name, "coerce", propNode, parserContext));
                     iterator.remove();
                 } else if (propName.equals("scaling_factor")) {
-                    builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode).doubleValue());
+                    builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false).doubleValue());
                     iterator.remove();
                 }
             }
@@ -207,7 +207,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
         @Override
         public Query termQuery(Object value, QueryShardContext context) {
             failIfNotIndexed();
-            double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue();
+            double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue();
             long scaledValue = Math.round(queryValue * scalingFactor);
             Query query = NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue);
             if (boost() != 1f) {
@@ -221,7 +221,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
             failIfNotIndexed();
             List<Long> scaledValues = new ArrayList<>(values.size());
             for (Object value : values) {
-                double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue();
+                double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue();
                 long scaledValue = Math.round(queryValue * scalingFactor);
                 scaledValues.add(scaledValue);
             }
@@ -237,7 +237,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
             failIfNotIndexed();
             Long lo = null;
             if (lowerTerm != null) {
-                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm).doubleValue();
+                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm, false).doubleValue();
                 if (includeLower == false) {
                     dValue = Math.nextUp(dValue);
                 }
@@ -245,7 +245,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
             }
             Long hi = null;
             if (upperTerm != null) {
-                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm).doubleValue();
+                double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm, false).doubleValue();
                 if (includeUpper == false) {
                     dValue = Math.nextDown(dValue);
                 }
@@ -404,7 +404,7 @@ public class ScaledFloatFieldMapper extends FieldMapper {
         }
 
         if (numericValue == null) {
-            numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value);
+            numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false);
         }
 
         if (includeInAll) {
@@ -504,7 +504,10 @@ public class ScaledFloatFieldMapper extends FieldMapper {
 
         @Override
         public NumericType getNumericType() {
-            return scaledFieldData.getNumericType();
+            /**
+             * {@link ScaledFloatLeafFieldData#getDoubleValues()} transforms the raw long values in `scaled` floats.
+             */
+            return NumericType.DOUBLE;
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java
index e38a2aab840..c160f2b8cb9 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java
@@ -145,7 +145,7 @@ public class SeqNoFieldMapper extends MetadataFieldMapper {
 
     static final class SeqNoFieldType extends MappedFieldType {
 
-        public SeqNoFieldType() {
+        SeqNoFieldType() {
         }
 
         protected SeqNoFieldType(SeqNoFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index efddec90669..5e1b6843940 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -47,8 +47,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 public class SourceFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_source";
@@ -116,7 +114,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode));
+                    builder.enabled(TypeParsers.nodeBooleanValue(name, "enabled", fieldNode, parserContext));
                     iterator.remove();
                 } else if ("format".equals(fieldName) && parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)) {
                     // ignore on old indices, reject on and after 5.0
@@ -151,7 +149,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
     static final class SourceFieldType extends MappedFieldType {
 
-        public SourceFieldType() {}
+        SourceFieldType() {}
 
         protected SourceFieldType(SourceFieldType ref) {
             super(ref);
@@ -243,7 +241,8 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
         if (filter != null) {
             // we don't update the context source if we filter, we want to keep it as is...
-            Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(source, true);
+            Tuple<XContentType, Map<String, Object>> mapTuple =
+                XContentHelper.convertToMap(source, true, context.sourceToParse().getXContentType());
             Map<String, Object> filteredSource = filter.apply(mapTuple.v2());
             BytesStreamOutput bStream = new BytesStreamOutput();
             XContentType contentType = mapTuple.v1();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
index 0cafc50bbe2..a8a983ecde8 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
@@ -23,15 +23,18 @@ import java.util.Objects;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 
 public class SourceToParse {
 
-    public static SourceToParse source(String index, String type, String id, BytesReference source) {
-        return source(Origin.PRIMARY, index, type, id, source);
+    public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) {
+        return source(Origin.PRIMARY, index, type, id, source, contentType);
     }
 
-    public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) {
-        return new SourceToParse(origin, index, type, id, source);
+    public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source,
+                                       XContentType contentType) {
+        return new SourceToParse(origin, index, type, id, source, contentType);
     }
 
     private final Origin origin;
@@ -48,14 +51,17 @@ public class SourceToParse {
 
     private String parentId;
 
-    private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) {
+    private XContentType xContentType;
+
+    private SourceToParse(Origin origin, String index, String type, String id, BytesReference source, XContentType xContentType) {
         this.origin = Objects.requireNonNull(origin);
         this.index = Objects.requireNonNull(index);
         this.type = Objects.requireNonNull(type);
         this.id = Objects.requireNonNull(id);
         // we always convert back to byte array, since we store it and Field only supports bytes..
         // so, we might as well do it here, and improve the performance of working with direct byte arrays
-        this.source = new BytesArray(source.toBytesRef());
+        this.source = new BytesArray(Objects.requireNonNull(source).toBytesRef());
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     public Origin origin() {
@@ -91,6 +97,10 @@ public class SourceToParse {
         return this.routing;
     }
 
+    public XContentType getXContentType() {
+        return this.xContentType;
+    }
+
     public SourceToParse routing(String routing) {
         this.routing = routing;
         return this;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
index 71d07aa385f..e1fd56616f3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
@@ -35,7 +35,7 @@ import org.elasticsearch.index.query.QueryShardContext;
  *  with the inverted index. */
 abstract class TermBasedFieldType extends MappedFieldType {
 
-    public TermBasedFieldType() {}
+    TermBasedFieldType() {}
 
     protected TermBasedFieldType(MappedFieldType ref) {
         super(ref);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
index bb8c4d77a63..b41695eb8bd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -144,10 +144,10 @@ public class TextFieldMapper extends FieldMapper {
                     builder.positionIncrementGap(newPositionIncrementGap);
                     iterator.remove();
                 } else if (propName.equals("fielddata")) {
-                    builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
+                    builder.fielddata(XContentMapValues.nodeBooleanValue(propNode, "fielddata"));
                     iterator.remove();
                 } else if (propName.equals("eager_global_ordinals")) {
-                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
+                    builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode, "eager_global_ordinals"));
                     iterator.remove();
                 } else if (propName.equals("fielddata_frequency_filter")) {
                     Map<?,?> frequencyFilter = (Map<?, ?>) propNode;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
index 67775283d5a..5f5be04a914 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java
@@ -90,7 +90,7 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     static final class TypeFieldType extends StringFieldType {
         private boolean fielddata;
 
-        public TypeFieldType() {
+        TypeFieldType() {
             this.fielddata = false;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
index 475848989d4..3faa135402d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java
@@ -22,28 +22,23 @@ package org.elasticsearch.index.mapper;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.isArray;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeFloatValue;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
 
@@ -56,22 +51,37 @@ public class TypeParsers {
     public static final String INDEX_OPTIONS_OFFSETS = "offsets";
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TypeParsers.class));
-    private static final Set<String> BOOLEAN_STRINGS = new HashSet<>(Arrays.asList("true", "false"));
 
-    public static boolean nodeBooleanValue(String name, Object node, Mapper.TypeParser.ParserContext parserContext) {
-        // Hook onto ParseFieldMatcher so that parsing becomes strict when setting index.query.parse.strict
-        if (parserContext.parseFieldMatcher().isStrict()) {
-            return XContentMapValues.nodeBooleanValue(node);
+    //TODO 22298: Remove this method and have all call-sites use <code>XContentMapValues.nodeBooleanValue(node)</code> directly.
+    public static boolean nodeBooleanValue(String fieldName, String propertyName, Object node,
+                                           Mapper.TypeParser.ParserContext parserContext) {
+        if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            return XContentMapValues.nodeBooleanValue(node, fieldName + "." + propertyName);
         } else {
-            // TODO: remove this leniency in 6.0
-            if (BOOLEAN_STRINGS.contains(node.toString()) == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [{}] but got [{}]", name, node);
-            }
-            return XContentMapValues.lenientNodeBooleanValue(node);
+            return nodeBooleanValueLenient(fieldName, propertyName, node);
         }
     }
 
-    private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
+    //TODO 22298: Remove this method and have all call-sites use <code>XContentMapValues.nodeBooleanValue(node)</code> directly.
+    public static boolean nodeBooleanValueLenient(String fieldName, String propertyName, Object node) {
+        if (Booleans.isBoolean(node.toString()) == false) {
+            DEPRECATION_LOGGER.deprecated("Expected a boolean for property [{}] for field [{}] but got [{}]",
+                propertyName, fieldName, node);
+        }
+        if (node instanceof Boolean) {
+            return (Boolean) node;
+        }
+        if (node instanceof Number) {
+            return ((Number) node).intValue() != 0;
+        }
+        @SuppressWarnings("deprecated")
+        boolean value = Booleans.parseBooleanLenient(node.toString(), false);
+        return value;
+    }
+
+
+    private static void parseAnalyzersAndTermVectors(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode,
+                                                     Mapper.TypeParser.ParserContext parserContext) {
         NamedAnalyzer indexAnalyzer = null;
         NamedAnalyzer searchAnalyzer = null;
         NamedAnalyzer searchQuoteAnalyzer = null;
@@ -84,16 +94,17 @@ public class TypeParsers {
                 parseTermVector(name, propNode.toString(), builder);
                 iterator.remove();
             } else if (propName.equals("store_term_vectors")) {
-                builder.storeTermVectors(nodeBooleanValue("store_term_vectors", propNode, parserContext));
+                builder.storeTermVectors(nodeBooleanValue(name, "store_term_vectors", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_offsets")) {
-                builder.storeTermVectorOffsets(nodeBooleanValue("store_term_vector_offsets", propNode, parserContext));
+                builder.storeTermVectorOffsets(nodeBooleanValue(name, "store_term_vector_offsets", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_positions")) {
-                builder.storeTermVectorPositions(nodeBooleanValue("store_term_vector_positions", propNode, parserContext));
+                builder.storeTermVectorPositions(
+                    nodeBooleanValue(name, "store_term_vector_positions", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("store_term_vector_payloads")) {
-                builder.storeTermVectorPayloads(nodeBooleanValue("store_term_vector_payloads", propNode, parserContext));
+                builder.storeTermVectorPayloads(nodeBooleanValue(name,"store_term_vector_payloads", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("analyzer")) {
                 NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
@@ -124,7 +135,8 @@ public class TypeParsers {
         }
 
         if (searchAnalyzer == null && searchQuoteAnalyzer != null) {
-            throw new MapperParsingException("analyzer and search_analyzer on field [" + name + "] must be set when search_quote_analyzer is set");
+            throw new MapperParsingException("analyzer and search_analyzer on field [" + name +
+                "] must be set when search_quote_analyzer is set");
         }
 
         if (searchAnalyzer == null) {
@@ -146,16 +158,17 @@ public class TypeParsers {
         }
     }
 
-    public static boolean parseNorms(FieldMapper.Builder builder, String propName, Object propNode, Mapper.TypeParser.ParserContext parserContext) {
+    public static boolean parseNorms(FieldMapper.Builder builder, String fieldName, String propName, Object propNode,
+                                     Mapper.TypeParser.ParserContext parserContext) {
         if (propName.equals("norms")) {
             if (propNode instanceof Map) {
                 final Map<String, Object> properties = nodeMapValue(propNode, "norms");
-                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext();) {
+                for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext(); ) {
                     Entry<String, Object> entry2 = propsIterator.next();
                     final String propName2 = entry2.getKey();
                     final Object propNode2 = entry2.getValue();
                     if (propName2.equals("enabled")) {
-                        builder.omitNorms(!lenientNodeBooleanValue(propNode2));
+                        builder.omitNorms(nodeBooleanValue(fieldName, "enabled", propNode2, parserContext) == false);
                         propsIterator.remove();
                     } else if (propName2.equals("loading")) {
                         // ignore for bw compat
@@ -163,13 +176,14 @@ public class TypeParsers {
                     }
                 }
                 DocumentMapperParser.checkNoRemainingFields(propName, properties, parserContext.indexVersionCreated());
-                DEPRECATION_LOGGER.deprecated("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use [norms:true/false] instead");
+                DEPRECATION_LOGGER.deprecated("The [norms{enabled:true/false}] way of specifying norms is deprecated, please use " +
+                    "[norms:true/false] instead");
             } else {
-                builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext) == false);
+                builder.omitNorms(nodeBooleanValue(fieldName,"norms", propNode, parserContext) == false);
             }
             return true;
         } else if (propName.equals("omit_norms")) {
-            builder.omitNorms(nodeBooleanValue("norms", propNode, parserContext));
+            builder.omitNorms(nodeBooleanValue(fieldName,"norms", propNode, parserContext));
             DEPRECATION_LOGGER.deprecated("[omit_norms] is deprecated, please use [norms] instead with the opposite boolean value");
             return true;
         } else {
@@ -181,14 +195,15 @@ public class TypeParsers {
      * Parse text field attributes. In addition to {@link #parseField common attributes}
      * this will parse analysis and term-vectors related settings.
      */
-    public static void parseTextField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
+    public static void parseTextField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode,
+                                      Mapper.TypeParser.ParserContext parserContext) {
         parseField(builder, name, fieldNode, parserContext);
         parseAnalyzersAndTermVectors(builder, name, fieldNode, parserContext);
-        for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
+        for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext(); ) {
             Map.Entry<String, Object> entry = iterator.next();
             final String propName = entry.getKey();
             final Object propNode = entry.getValue();
-            if (parseNorms(builder, propName, propNode, parserContext)) {
+            if (parseNorms(builder, name, propName, propNode, parserContext)) {
                 iterator.remove();
             }
         }
@@ -197,8 +212,8 @@ public class TypeParsers {
     /**
      * Parse common field attributes such as {@code doc_values} or {@code store}.
      */
-    public static void parseField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) {
-        Version indexVersionCreated = parserContext.indexVersionCreated();
+    public static void parseField(FieldMapper.Builder builder, String name, Map<String, Object> fieldNode,
+                                  Mapper.TypeParser.ParserContext parserContext) {
         for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
             Map.Entry<String, Object> entry = iterator.next();
             final String propName = entry.getKey();
@@ -211,25 +226,30 @@ public class TypeParsers {
                 throw new MapperParsingException("[" + propName + "] must not have a [null] value");
             }
             if (propName.equals("store")) {
-                builder.store(parseStore(name, propNode.toString(), parserContext));
+                builder.store(nodeBooleanValue(name,"store", propNode.toString(), parserContext));
                 iterator.remove();
             } else if (propName.equals("index")) {
-                builder.index(parseIndex(name, propNode.toString(), parserContext));
+                builder.index(nodeBooleanValue(name, "index", propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals(DOC_VALUES)) {
-                builder.docValues(nodeBooleanValue(DOC_VALUES, propNode, parserContext));
+                builder.docValues(nodeBooleanValue(name, DOC_VALUES, propNode, parserContext));
                 iterator.remove();
             } else if (propName.equals("boost")) {
                 builder.boost(nodeFloatValue(propNode));
                 iterator.remove();
             } else if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1)
-                    && parseNorms(builder, propName, propNode, parserContext)) {
+                && parseNorms(builder, name, propName, propNode, parserContext)) {
                 iterator.remove();
             } else if (propName.equals("index_options")) {
                 builder.indexOptions(nodeIndexOptionValue(propNode));
                 iterator.remove();
             } else if (propName.equals("include_in_all")) {
-                builder.includeInAll(nodeBooleanValue("include_in_all", propNode, parserContext));
+                if (parserContext.isWithinMultiField()) {
+                    throw new MapperParsingException("include_in_all in multi fields is not allowed. Found the include_in_all in field ["
+                        + name + "] which is within a multi field.");
+                } else {
+                    builder.includeInAll(nodeBooleanValue(name, "include_in_all", propNode, parserContext));
+                }
                 iterator.remove();
             } else if (propName.equals("similarity")) {
                 SimilarityProvider similarityProvider = resolveSimilarity(parserContext, name, propNode.toString());
@@ -244,7 +264,8 @@ public class TypeParsers {
                 iterator.remove();
             } else if (propName.equals("copy_to")) {
                 if (parserContext.isWithinMultiField()) {
-                    throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field.");
+                    throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] " +
+                        "which is within a multi field.");
                 } else {
                     parseCopyFields(propNode, builder);
                 }
@@ -253,7 +274,8 @@ public class TypeParsers {
         }
     }
 
-    public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) {
+    public static boolean parseMultiField(FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext,
+                                          String propName, Object propNode) {
         parserContext = parserContext.createMultiFieldContext(parserContext);
         if (propName.equals("fields")) {
 
@@ -265,13 +287,14 @@ public class TypeParsers {
                 multiFieldsPropNodes = (Map<String, Object>) propNode;
             } else {
                 throw new MapperParsingException("expected map for property [fields] on field [" + propNode + "] or " +
-                        "[" + propName + "] but got a " + propNode.getClass());
+                    "[" + propName + "] but got a " + propNode.getClass());
             }
 
             for (Map.Entry<String, Object> multiFieldEntry : multiFieldsPropNodes.entrySet()) {
                 String multiFieldName = multiFieldEntry.getKey();
                 if (multiFieldName.contains(".")) {
-                    throw new MapperParsingException("Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot contain '.'");
+                    throw new MapperParsingException("Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot" +
+                        " contain '.'");
                 }
                 if (!(multiFieldEntry.getValue() instanceof Map)) {
                     throw new MapperParsingException("illegal field [" + multiFieldName + "], only fields can be specified inside fields");
@@ -346,48 +369,11 @@ public class TypeParsers {
         }
     }
 
-    public static boolean parseIndex(String fieldName, String index, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
-        switch (index) {
-        case "true":
-            return true;
-        case "false":
-            return false;
-        case "not_analyzed":
-        case "analyzed":
-        case "no":
-            if (parserContext.parseFieldMatcher().isStrict() == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [index] but got [{}]", index);
-                return "no".equals(index) == false;
-            } else {
-                throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
-            }
-        default:
-            throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true] or [false]");
-        }
-    }
-
-    public static boolean parseStore(String fieldName, String store, Mapper.TypeParser.ParserContext parserContext) throws MapperParsingException {
-        if (parserContext.parseFieldMatcher().isStrict()) {
-            return XContentMapValues.nodeBooleanValue(store);
-        } else {
-            if (BOOLEAN_STRINGS.contains(store) == false) {
-                DEPRECATION_LOGGER.deprecated("Expected a boolean for property [store] but got [{}]", store);
-            }
-            if ("no".equals(store)) {
-                return false;
-            } else if ("yes".equals(store)) {
-                return true;
-            } else {
-                return lenientNodeBooleanValue(store);
-            }
-        }
-    }
-
     @SuppressWarnings("unchecked")
     public static void parseCopyFields(Object propNode, FieldMapper.Builder builder) {
         FieldMapper.CopyTo.Builder copyToBuilder = new FieldMapper.CopyTo.Builder();
         if (isArray(propNode)) {
-            for(Object node : (List<Object>) propNode) {
+            for (Object node : (List<Object>) propNode) {
                 copyToBuilder.add(nodeStringValue(node, null));
             }
         } else {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
index bb98f44ed1c..a2d76dcf2c0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java
@@ -77,7 +77,7 @@ public class UidFieldMapper extends MetadataFieldMapper {
 
     static final class UidFieldType extends TermBasedFieldType {
 
-        public UidFieldType() {
+        UidFieldType() {
         }
 
         protected UidFieldType(UidFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
index ce3b2c4b8e8..1d2e997acba 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java
@@ -70,7 +70,7 @@ public class VersionFieldMapper extends MetadataFieldMapper {
 
     static final class VersionFieldType extends MappedFieldType {
 
-        public VersionFieldType() {
+        VersionFieldType() {
         }
 
         protected VersionFieldType(VersionFieldType ref) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index e48373a905c..6a3cd2f1ed5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -26,7 +26,6 @@ import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -309,7 +308,7 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
      * {@link MatchAllQueryBuilder} and {@link MatchNoneQueryBuilder} support these fields so they
      * should use this method.
      */
-    protected static void declareStandardFields(AbstractObjectParser<? extends QueryBuilder, ? extends ParseFieldMatcherSupplier> parser) {
+    protected static void declareStandardFields(AbstractObjectParser<? extends QueryBuilder, ?> parser) {
         parser.declareFloat((builder, value) -> builder.boost(value), AbstractQueryBuilder.BOOST_FIELD);
         parser.declareString((builder, value) -> builder.queryName(value), AbstractQueryBuilder.NAME_FIELD);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 7866035dfac..5eb8b81009c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -57,7 +57,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     private static final String MUST = "must";
     private static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord");
     private static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match");
-    private static final ParseField MINIMUM_NUMBER_SHOULD_MATCH = new ParseField("minimum_number_should_match");
     private static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative");
 
     private final List<QueryBuilder> mustClauses = new ArrayList<>();
@@ -167,7 +166,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * <tt>MUST</tt> clauses one or more <code>SHOULD</code> clauses must match a document
      * for the BooleanQuery to match. No <tt>null</tt> value allowed.
      *
-     * @see #minimumNumberShouldMatch(int)
+     * @see #minimumShouldMatch(int)
      */
     public BoolQueryBuilder should(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
@@ -181,7 +180,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * Gets the list of clauses that <b>should</b> be matched by the returned documents.
      *
      * @see #should(QueryBuilder)
-     *  @see #minimumNumberShouldMatch(int)
+     *  @see #minimumShouldMatch(int)
      */
     public List<QueryBuilder> should() {
         return this.shouldClauses;
@@ -202,6 +201,22 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         return this.disableCoord;
     }
 
+    /**
+     * @return the string representation of the minimumShouldMatch settings for this query
+     */
+    public String minimumShouldMatch() {
+        return this.minimumShouldMatch;
+    }
+
+    /**
+     * Sets the minimum should match parameter using the special syntax (for example, supporting percentage).
+     * @see BoolQueryBuilder#minimumShouldMatch(int)
+     */
+    public BoolQueryBuilder minimumShouldMatch(String minimumShouldMatch) {
+        this.minimumShouldMatch = minimumShouldMatch;
+        return this;
+    }
+
     /**
      * Specifies a minimum number of the optional (should) boolean clauses which must be satisfied.
      * <p>
@@ -213,35 +228,10 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * any specific clauses are required (or prohibited).  This number will
      * only be compared against the number of matching optional clauses.
      *
-     * @param minimumNumberShouldMatch the number of optional clauses that must match
+     * @param minimumShouldMatch the number of optional clauses that must match
      */
-    public BoolQueryBuilder minimumNumberShouldMatch(int minimumNumberShouldMatch) {
-        this.minimumShouldMatch = Integer.toString(minimumNumberShouldMatch);
-        return this;
-    }
-
-
-    /**
-     * Specifies a minimum number of the optional (should) boolean clauses which must be satisfied.
-     * @see BoolQueryBuilder#minimumNumberShouldMatch(int)
-     */
-    public BoolQueryBuilder minimumNumberShouldMatch(String minimumNumberShouldMatch) {
-        this.minimumShouldMatch = minimumNumberShouldMatch;
-        return this;
-    }
-
-    /**
-     * @return the string representation of the minimumShouldMatch settings for this query
-     */
-    public String minimumShouldMatch() {
-        return this.minimumShouldMatch;
-    }
-
-    /**
-     * Sets the minimum should match using the special syntax (for example, supporting percentage).
-     */
-    public BoolQueryBuilder minimumShouldMatch(String minimumShouldMatch) {
-        this.minimumShouldMatch = minimumShouldMatch;
+    public BoolQueryBuilder minimumShouldMatch(int minimumShouldMatch) {
+        this.minimumShouldMatch = Integer.toString(minimumShouldMatch);
         return this;
     }
 
@@ -358,17 +348,15 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) {
+                if (DISABLE_COORD_FIELD.match(currentFieldName)) {
                     disableCoord = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH)) {
+                } else if (MINIMUM_SHOULD_MATCH.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_NUMBER_SHOULD_MATCH)) {
-                    minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ADJUST_PURE_NEGATIVE)) {
+                } else if (ADJUST_PURE_NEGATIVE.match(currentFieldName)) {
                     adjustPureNegative = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]");
@@ -391,7 +379,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         boolQuery.boost(boost);
         boolQuery.disableCoord(disableCoord);
         boolQuery.adjustPureNegative(adjustPureNegative);
-        boolQuery.minimumNumberShouldMatch(minimumShouldMatch);
+        boolQuery.minimumShouldMatch(minimumShouldMatch);
         boolQuery.queryName(queryName);
         return boolQuery;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index 19f7a1b6ea1..cb16faf9b39 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -153,21 +153,21 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, POSITIVE_FIELD)) {
+                if (POSITIVE_FIELD.match(currentFieldName)) {
                     positiveQuery = parseContext.parseInnerQueryBuilder();
                     positiveQueryFound = true;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NEGATIVE_FIELD)) {
+                } else if (NEGATIVE_FIELD.match(currentFieldName)) {
                     negativeQuery = parseContext.parseInnerQueryBuilder();
                     negativeQueryFound = true;
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NEGATIVE_BOOST_FIELD)) {
+                if (NEGATIVE_BOOST_FIELD.match(currentFieldName)) {
                     negativeBoost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 596a1875079..a91d8fe44cf 100644
--- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -290,15 +290,15 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             String innerFieldName = null;
                             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     innerFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (parseContext.getParseFieldMatcher().match(innerFieldName, LOW_FREQ_FIELD)) {
+                                    if (LOW_FREQ_FIELD.match(innerFieldName)) {
                                         lowFreqMinimumShouldMatch = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(innerFieldName, HIGH_FREQ_FIELD)) {
+                                    } else if (HIGH_FREQ_FIELD.match(innerFieldName)) {
                                         highFreqMinimumShouldMatch = parser.text();
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
@@ -316,23 +316,23 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
                                     "] query does not support [" + currentFieldName + "]");
                         }
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                        if (QUERY_FIELD.match(currentFieldName)) {
                             text = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                        } else if (ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) {
+                        } else if (DISABLE_COORD_FIELD.match(currentFieldName)) {
                             disableCoord = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, HIGH_FREQ_OPERATOR_FIELD)) {
+                        } else if (HIGH_FREQ_OPERATOR_FIELD.match(currentFieldName)) {
                             highFreqOperator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOW_FREQ_OPERATOR_FIELD)) {
+                        } else if (LOW_FREQ_OPERATOR_FIELD.match(currentFieldName)) {
                             lowFreqOperator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             lowFreqMinimumShouldMatch = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                             cutoffFrequency = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "[" + CommonTermsQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 8efd96c1ea7..96035b94b16 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -101,7 +101,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
             } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_QUERY_FIELD)) {
+                if (INNER_QUERY_FIELD.match(currentFieldName)) {
                     if (queryFound) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + ConstantScoreQueryBuilder.NAME + "]"
                                 + " accepts only one 'filter' element.");
@@ -113,9 +113,9 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
                             "[constant_score] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index 96687777e93..5061206eb1d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -137,14 +137,14 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
+                if (QUERIES_FIELD.match(currentFieldName)) {
                     queriesFound = true;
                     queries.add(parseContext.parseInnerQueryBuilder());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
+                if (QUERIES_FIELD.match(currentFieldName)) {
                     queriesFound = true;
                     while (token != XContentParser.Token.END_ARRAY) {
                         queries.add(parseContext.parseInnerQueryBuilder());
@@ -154,11 +154,11 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index a6f85f96be4..07d652cb333 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -96,11 +96,11 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                if (FIELD_FIELD.match(currentFieldName)) {
                     fieldPattern = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
index 2bd384aff95..a7ef6d9551b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
@@ -115,7 +115,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "[field_masking_span] query must be of type span query");
@@ -126,11 +126,11 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
                             + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                } else if (FIELD_FIELD.match(currentFieldName)) {
                     field = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
index ef376d440a8..5da9edcd2a5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
@@ -276,23 +276,23 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                        } else if (Fuzziness.FIELD.match(currentFieldName)) {
                             fuzziness = Fuzziness.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                        } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                             prefixLength = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansions = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TRANSPOSITIONS_FIELD)) {
+                        } else if (TRANSPOSITIONS_FIELD.match(currentFieldName)) {
                             transpositions = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -337,7 +337,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
             query = new FuzzyQuery(new Term(fieldName, BytesRefs.toBytesRef(value)), maxEdits, prefixLength, maxExpansions, transpositions);
         }
         if (query instanceof MultiTermQuery) {
-            MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+            MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null);
             QueryParsers.setRewriteMethod((MultiTermQuery) query, rewriteMethod);
         }
         return query;
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index ca61a66066c..00371ce7a63 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -62,9 +62,9 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
     private static final ParseField TYPE_FIELD = new ParseField("type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
     private static final ParseField COERCE_FIELD =new ParseField("coerce", "normalize")
-            .withAllDeprecated("use field validation_method instead");
+            .withAllDeprecated("validation_method");
     private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use field validation_method instead");
+            .withAllDeprecated("validation_method");
     private static final ParseField FIELD_FIELD = new ParseField("field");
     private static final ParseField TOP_FIELD = new ParseField("top");
     private static final ParseField BOTTOM_FIELD = new ParseField("bottom");
@@ -408,30 +408,30 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
                         token = parser.nextToken();
                         if (parseContext.isDeprecatedSetting(currentFieldName)) {
                             // skip
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELD_FIELD)) {
+                        } else if (FIELD_FIELD.match(currentFieldName)) {
                             fieldName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_FIELD)) {
+                        } else if (TOP_FIELD.match(currentFieldName)) {
                             top = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_FIELD)) {
+                        } else if (BOTTOM_FIELD.match(currentFieldName)) {
                             bottom = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LEFT_FIELD)) {
+                        } else if (LEFT_FIELD.match(currentFieldName)) {
                             left = parser.doubleValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, RIGHT_FIELD)) {
+                        } else if (RIGHT_FIELD.match(currentFieldName)) {
                             right = parser.doubleValue();
                         } else {
-                            if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_LEFT_FIELD)) {
+                            if (TOP_LEFT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 top = sparse.getLat();
                                 left = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_RIGHT_FIELD)) {
+                            } else if (BOTTOM_RIGHT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 bottom = sparse.getLat();
                                 right = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TOP_RIGHT_FIELD)) {
+                            } else if (TOP_RIGHT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 top = sparse.getLat();
                                 right = sparse.getLon();
-                            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOTTOM_LEFT_FIELD)) {
+                            } else if (BOTTOM_LEFT_FIELD.match(currentFieldName)) {
                                 GeoUtils.parseGeoPoint(parser, sparse);
                                 bottom = sparse.getLat();
                                 left = sparse.getLon();
@@ -446,22 +446,22 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                } else if (TYPE_FIELD.match(currentFieldName)) {
                     type = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. unexpected field [{}]",
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index db66694add4..0ba1836cab9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -55,7 +55,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     /** Default for distance unit computation. */
     public static final DistanceUnit DEFAULT_DISTANCE_UNIT = DistanceUnit.DEFAULT;
     /** Default for geo distance computation. */
-    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.DEFAULT;
+    public static final GeoDistance DEFAULT_GEO_DISTANCE = GeoDistance.ARC;
     /** Default for optimising query through pre computed bounding box query. */
     @Deprecated
     public static final String DEFAULT_OPTIMIZE_BBOX = "memory";
@@ -66,10 +66,8 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
     @Deprecated
     private static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox")
             .withAllDeprecated("no replacement: `optimize_bbox` is no longer supported due to recent improvements");
@@ -84,7 +82,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     /** Point to use as center. */
     private GeoPoint center = new GeoPoint(Double.NaN, Double.NaN);
     /** Algorithm to use for distance computation. */
-    private GeoDistance geoDistance = DEFAULT_GEO_DISTANCE;
+    private GeoDistance geoDistance = GeoDistance.ARC;
     /** Whether or not to use a bbox for pre-filtering. TODO change to enum? */
     private String optimizeBbox = null;
     /** How strict should geo coordinate validation be? */
@@ -289,9 +287,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
             GeoUtils.normalizePoint(center, true, true);
         }
 
-        double normDistance = geoDistance.normalize(this.distance, DistanceUnit.DEFAULT);
-
-        return LatLonPoint.newDistanceQuery(fieldType.name(), center.lat(), center.lon(), normDistance);
+        return LatLonPoint.newDistanceQuery(fieldType.name(), center.lat(), center.lon(), this.distance);
     }
 
     @Override
@@ -358,15 +354,15 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DISTANCE_FIELD)) {
+                if (DISTANCE_FIELD.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_STRING) {
                         vDistance = parser.text(); // a String
                     } else {
                         vDistance = parser.numberValue(); // a Number
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentFieldName)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DISTANCE_TYPE_FIELD)) {
+                } else if (DISTANCE_TYPE_FIELD.match(currentFieldName)) {
                     geoDistance = GeoDistance.fromString(parser.text());
                 } else if (currentFieldName.endsWith(".lat")) {
                     point.resetLat(parser.doubleValue());
@@ -374,22 +370,22 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
                 } else if (currentFieldName.endsWith(".lon")) {
                     point.resetLon(parser.doubleValue());
                     fieldName = currentFieldName.substring(0, currentFieldName.length() - ".lon".length());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPTIMIZE_BBOX_FIELD)) {
+                } else if (OPTIMIZE_BBOX_FIELD.match(currentFieldName)) {
                     optimizeBbox = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else {
                     if (fieldName == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index da0a0fe18de..ca85a9d809f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -49,10 +49,8 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
      */
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
     private static final ParseField VALIDATION_METHOD = new ParseField("validation_method");
     private static final ParseField POINTS_FIELD = new ParseField("points");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
@@ -254,7 +252,7 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_ARRAY) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, POINTS_FIELD)) {
+                        if (POINTS_FIELD.match(currentFieldName)) {
                             shell = new ArrayList<GeoPoint>();
                             while ((token = parser.nextToken()) != Token.END_ARRAY) {
                                 shell.add(GeoUtils.parseGeoPoint(parser));
@@ -273,16 +271,16 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
                     queryName = parser.text();
                 } else if ("boost".equals(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentFieldName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentFieldName)) {
                     ignoreMalformed = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALIDATION_METHOD)) {
+                } else if (VALIDATION_METHOD.match(currentFieldName)) {
                     validationMethod = GeoValidationMethod.fromString(parser.text());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index c54cf2af07f..138f62a92c2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.geo.SpatialStrategy;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -383,7 +384,8 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
         String[] pathElements = path.split("\\.");
         int currentPathSlot = 0;
 
-        try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
+        // It is safe to use EMPTY here because this never uses namedObject
+        try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, response.getSourceAsBytesRef())) {
             XContentParser.Token currentToken;
             while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (currentToken == XContentParser.Token.FIELD_NAME) {
@@ -487,31 +489,31 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                         token = parser.nextToken();
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_FIELD)) {
+                        if (SHAPE_FIELD.match(currentFieldName)) {
                             shape = ShapeBuilder.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, STRATEGY_FIELD)) {
+                        } else if (STRATEGY_FIELD.match(currentFieldName)) {
                             String strategyName = parser.text();
                             strategy = SpatialStrategy.fromString(strategyName);
                             if (strategy == null) {
                                 throw new ParsingException(parser.getTokenLocation(), "Unknown strategy [" + strategyName + " ]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, RELATION_FIELD)) {
+                        } else if (RELATION_FIELD.match(currentFieldName)) {
                             shapeRelation = ShapeRelation.getRelationByName(parser.text());
                             if (shapeRelation == null) {
                                 throw new ParsingException(parser.getTokenLocation(), "Unknown shape operation [" + parser.text() + " ]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_SHAPE_FIELD)) {
+                        } else if (INDEXED_SHAPE_FIELD.match(currentFieldName)) {
                             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     currentFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_ID_FIELD)) {
+                                    if (SHAPE_ID_FIELD.match(currentFieldName)) {
                                         id = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_TYPE_FIELD)) {
+                                    } else if (SHAPE_TYPE_FIELD.match(currentFieldName)) {
                                         type = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_INDEX_FIELD)) {
+                                    } else if (SHAPE_INDEX_FIELD.match(currentFieldName)) {
                                         index = parser.text();
-                                    } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SHAPE_PATH_FIELD)) {
+                                    } else if (SHAPE_PATH_FIELD.match(currentFieldName)) {
                                         shapePath = parser.text();
                                     }
                                 } else {
@@ -526,11 +528,11 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + GeoShapeQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 85b4fd18da5..37d06f79eb2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -145,8 +145,8 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         return innerHitBuilder;
     }
 
-    public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHitBuilder = new InnerHitBuilder(Objects.requireNonNull(innerHit), query, type);
+    public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) {
+        this.innerHitBuilder = new InnerHitBuilder(Objects.requireNonNull(innerHit), query, type, ignoreUnmapped);
         return this;
     }
 
@@ -240,27 +240,27 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
             } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     iqb = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
                     innerHitBuilder = InnerHitBuilder.fromXContent(parseContext);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                if (TYPE_FIELD.match(currentFieldName)) {
                     childType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
                     scoreMode = parseScoreMode(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MIN_CHILDREN_FIELD)) {
+                } else if (MIN_CHILDREN_FIELD.match(currentFieldName)) {
                     minChildren = parser.intValue(true);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_CHILDREN_FIELD)) {
+                } else if (MAX_CHILDREN_FIELD.match(currentFieldName)) {
                     maxChildren = parser.intValue(true);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
@@ -268,13 +268,13 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
             }
         }
         HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb, scoreMode);
-        if (innerHitBuilder != null) {
-            hasChildQueryBuilder.innerHit(innerHitBuilder);
-        }
         hasChildQueryBuilder.minMaxChildren(minChildren, maxChildren);
         hasChildQueryBuilder.queryName(queryName);
         hasChildQueryBuilder.boost(boost);
         hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+        if (innerHitBuilder != null) {
+            hasChildQueryBuilder.innerHit(innerHitBuilder, ignoreUnmapped);
+        }
         return hasChildQueryBuilder;
     }
 
@@ -318,7 +318,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
             context.setTypes(previousTypes);
         }
 
-        DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
+        DocumentMapper childDocMapper = context.documentMapper(type);
         if (childDocMapper == null) {
             if (ignoreUnmapped) {
                 return new MatchNoDocsQuery();
@@ -473,7 +473,10 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         QueryBuilder rewrittenQuery = query.rewrite(queryRewriteContext);
         if (rewrittenQuery != query) {
             InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHitBuilder, rewrittenQuery);
-            return new HasChildQueryBuilder(type, rewrittenQuery, minChildren, maxChildren, scoreMode, rewrittenInnerHit);
+            HasChildQueryBuilder hasChildQueryBuilder =
+                new HasChildQueryBuilder(type, rewrittenQuery, minChildren, maxChildren, scoreMode, rewrittenInnerHit);
+            hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+            return hasChildQueryBuilder;
         }
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
index f8644a87528..0dd0aea15c3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
@@ -124,8 +124,8 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
         return innerHit;
     }
 
-    public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHit = new InnerHitBuilder(innerHit, query, type);
+    public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) {
+        this.innerHit = new InnerHitBuilder(innerHit, query, type, ignoreUnmapped);
         return this;
     }
 
@@ -159,7 +159,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
             context.setTypes(previousTypes);
         }
 
-        DocumentMapper parentDocMapper = context.getMapperService().documentMapper(type);
+        DocumentMapper parentDocMapper = context.documentMapper(type);
         if (parentDocMapper == null) {
             if (ignoreUnmapped) {
                 return new MatchNoDocsQuery();
@@ -238,17 +238,17 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     iqb = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
                     innerHits = InnerHitBuilder.fromXContent(parseContext);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                if (TYPE_FIELD.match(currentFieldName)) {
                     parentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
                     String scoreModeValue = parser.text();
                     if ("score".equals(scoreModeValue)) {
                         score = true;
@@ -258,13 +258,13 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
                         throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" +
                                 scoreModeValue + "] as an option for score_mode");
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_FIELD)) {
+                } else if (SCORE_FIELD.match(currentFieldName)) {
                     score = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]");
@@ -276,7 +276,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
                 .queryName(queryName)
                 .boost(boost);
         if (innerHits != null) {
-            queryBuilder.innerHit(innerHits);
+            queryBuilder.innerHit(innerHits, ignoreUnmapped);
         }
         return queryBuilder;
     }
@@ -305,7 +305,9 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
         QueryBuilder rewrittenQuery = query.rewrite(queryShardContext);
         if (rewrittenQuery != query) {
             InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHit, rewrittenQuery);
-            return new HasParentQueryBuilder(type, rewrittenQuery, score, rewrittenInnerHit);
+            HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder(type, rewrittenQuery, score, rewrittenInnerHit);
+            hasParentQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+            return hasParentQueryBuilder;
         }
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 3a98ba36d20..c1b240066ab 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.query;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -45,9 +46,9 @@ import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -59,6 +60,7 @@ import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT;
 public final class InnerHitBuilder extends ToXContentToBytes implements Writeable {
 
     public static final ParseField NAME_FIELD = new ParseField("name");
+    public static final ParseField IGNORE_UNMAPPED = new ParseField("ignore_unmapped");
     public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
     public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder();
 
@@ -66,6 +68,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     static {
         PARSER.declareString(InnerHitBuilder::setName, NAME_FIELD);
+        PARSER.declareBoolean((innerHitBuilder, value) -> innerHitBuilder.ignoreUnmapped = value, IGNORE_UNMAPPED);
         PARSER.declareInt(InnerHitBuilder::setFrom, SearchSourceBuilder.FROM_FIELD);
         PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD);
@@ -94,7 +97,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
                 ObjectParser.ValueType.OBJECT_ARRAY);
         PARSER.declareField((p, i, c) -> {
             try {
-                i.setFetchSourceContext(FetchSourceContext.parse(c.parser()));
+                i.setFetchSourceContext(FetchSourceContext.fromXContent(c.parser()));
             } catch (IOException e) {
                 throw new ParsingException(p.getTokenLocation(), "Could not parse inner _source definition", e);
             }
@@ -130,6 +133,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private String name;
     private String nestedPath;
     private String parentChildType;
+    private boolean ignoreUnmapped;
 
     private int from;
     private int size = 3;
@@ -151,6 +155,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     private InnerHitBuilder(InnerHitBuilder other) {
         name = other.name;
+        this.ignoreUnmapped = other.ignoreUnmapped;
         from = other.from;
         size = other.size;
         explain = other.explain;
@@ -180,19 +185,21 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     }
 
 
-    InnerHitBuilder(InnerHitBuilder other, String nestedPath, QueryBuilder query) {
+    InnerHitBuilder(InnerHitBuilder other, String nestedPath, QueryBuilder query, boolean ignoreUnmapped) {
         this(other);
         this.query = query;
         this.nestedPath = nestedPath;
+        this.ignoreUnmapped = ignoreUnmapped;
         if (name == null) {
             this.name = nestedPath;
         }
     }
 
-    InnerHitBuilder(InnerHitBuilder other, QueryBuilder query, String parentChildType) {
+    InnerHitBuilder(InnerHitBuilder other, QueryBuilder query, String parentChildType, boolean ignoreUnmapped) {
         this(other);
         this.query = query;
         this.parentChildType = parentChildType;
+        this.ignoreUnmapped = ignoreUnmapped;
         if (name == null) {
             this.name = parentChildType;
         }
@@ -205,6 +212,9 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         name = in.readOptionalString();
         nestedPath = in.readOptionalString();
         parentChildType = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            ignoreUnmapped = in.readBoolean();
+        }
         from = in.readVInt();
         size = in.readVInt();
         explain = in.readBoolean();
@@ -243,6 +253,9 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeOptionalString(name);
         out.writeOptionalString(nestedPath);
         out.writeOptionalString(parentChildType);
+        if (out.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            out.writeBoolean(ignoreUnmapped);
+        }
         out.writeVInt(from);
         out.writeVInt(size);
         out.writeBoolean(explain);
@@ -254,8 +267,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
             out.writeVInt(scriptFields.size());
-            for (ScriptField scriptField : scriptFields) {
-                scriptField.writeTo(out);
+            Iterator<ScriptField> iterator = scriptFields.stream()
+                    .sorted((a, b) -> a.fieldName().compareTo(b.fieldName())).iterator();
+            while (iterator.hasNext()) {
+                iterator.next().writeTo(out);
             }
         }
         out.writeOptionalWriteable(fetchSourceContext);
@@ -273,7 +288,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeBoolean(hasChildInnerHits);
         if (hasChildInnerHits) {
             out.writeVInt(childInnerHits.size());
-            for (Map.Entry<String, InnerHitBuilder> entry : childInnerHits.entrySet()) {
+            Iterator<Map.Entry<String, InnerHitBuilder>> iterator = childInnerHits.entrySet().stream()
+                    .sorted((a, b) -> a.getKey().compareTo(b.getKey())).iterator();
+            while (iterator.hasNext()) {
+                Map.Entry<String, InnerHitBuilder> entry = iterator.next();
                 out.writeString(entry.getKey());
                 entry.getValue().writeTo(out);
             }
@@ -289,6 +307,13 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
+    /**
+     * Whether to include inner hits in the search response hits if required mappings is missing
+     */
+    public boolean isIgnoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
     public int getFrom() {
         return from;
     }
@@ -523,6 +548,14 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
         if (nestedPath != null) {
             ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(nestedPath);
+            if (nestedObjectMapper == null) {
+                if (ignoreUnmapped == false) {
+                    throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + nestedPath + "]");
+                } else {
+                    return null;
+                }
+            }
+
             ObjectMapper parentObjectMapper = queryShardContext.nestedScope().nextLevel(nestedObjectMapper);
             InnerHitsContext.NestedInnerHits nestedInnerHits = new InnerHitsContext.NestedInnerHits(
                     name, parentSearchContext, parentObjectMapper, nestedObjectMapper
@@ -535,7 +568,15 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
             innerHitsContext.addInnerHitDefinition(nestedInnerHits);
             return nestedInnerHits;
         } else if (parentChildType != null) {
-            DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(parentChildType);
+            DocumentMapper documentMapper = queryShardContext.documentMapper(parentChildType);
+            if (documentMapper == null) {
+                if (ignoreUnmapped == false) {
+                    throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + parentChildType + "]");
+                } else {
+                    return null;
+                }
+            }
+
             InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits(
                     name, parentSearchContext, queryShardContext.getMapperService(), documentMapper
             );
@@ -556,7 +597,9 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
             InnerHitsContext.BaseInnerHits childInnerHit = entry.getValue().build(
                     parentSearchContext, new InnerHitsContext()
             );
-            childInnerHits.put(entry.getKey(), childInnerHit);
+            if (childInnerHit != null) {
+                childInnerHits.put(entry.getKey(), childInnerHit);
+            }
         }
         innerHits.setChildInnerHits(childInnerHits);
     }
@@ -617,6 +660,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         if (name != null) {
             builder.field(NAME_FIELD.getPreferredName(), name);
         }
+        builder.field(IGNORE_UNMAPPED.getPreferredName(), ignoreUnmapped);
         builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from);
         builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size);
         builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version);
@@ -672,6 +716,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return Objects.equals(name, that.name) &&
                 Objects.equals(nestedPath, that.nestedPath) &&
                 Objects.equals(parentChildType, that.parentChildType) &&
+                Objects.equals(ignoreUnmapped, that.ignoreUnmapped) &&
                 Objects.equals(from, that.from) &&
                 Objects.equals(size, that.size) &&
                 Objects.equals(explain, that.explain) &&
@@ -689,8 +734,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldsContext,
-            docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
+        return Objects.hash(name, nestedPath, parentChildType, ignoreUnmapped, from, size, explain, version, trackScores,
+                storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
     }
 
     public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
index 4a86854e7bf..63580eca3e7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
@@ -58,7 +58,7 @@ public class MatchAllQueryBuilder extends AbstractQueryBuilder<MatchAllQueryBuil
         builder.endObject();
     }
 
-    private static ObjectParser<MatchAllQueryBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME, MatchAllQueryBuilder::new);
+    private static final ObjectParser<MatchAllQueryBuilder, QueryParseContext> PARSER = new ObjectParser<>(NAME, MatchAllQueryBuilder::new);
 
     static {
         declareStandardFields(PARSER);
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
index 674d7c4f698..79c64b32bb5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
@@ -68,9 +68,9 @@ public class MatchNoneQueryBuilder extends AbstractQueryBuilder<MatchNoneQueryBu
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "["+MatchNoneQueryBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
index b7ef225ad58..17495642cc2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
@@ -212,17 +212,17 @@ public class MatchPhrasePrefixQueryBuilder extends AbstractQueryBuilder<MatchPhr
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.QUERY_FIELD)) {
+                        if (MatchQueryBuilder.QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.ANALYZER_FIELD)) {
+                        } else if (MatchQueryBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchPhraseQueryBuilder.SLOP_FIELD)) {
+                        } else if (MatchPhraseQueryBuilder.SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansion = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
index 6adb1a22de2..e5227570b3f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
@@ -183,15 +183,15 @@ public class MatchPhraseQueryBuilder extends AbstractQueryBuilder<MatchPhraseQue
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.QUERY_FIELD)) {
+                        if (MatchQueryBuilder.QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MatchQueryBuilder.ANALYZER_FIELD)) {
+                        } else if (MatchQueryBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                        } else if (SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index 6c9a2bb9380..e7936b43b1c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -455,41 +455,13 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
         matchQuery.setFuzzyPrefixLength(prefixLength);
         matchQuery.setMaxExpansions(maxExpansions);
         matchQuery.setTranspositions(fuzzyTranspositions);
-        matchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), fuzzyRewrite, null));
+        matchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(fuzzyRewrite, null));
         matchQuery.setLenient(lenient);
         matchQuery.setCommonTermsCutoff(cutoffFrequency);
         matchQuery.setZeroTermsQuery(zeroTermsQuery);
 
         Query query = matchQuery.parse(type, fieldName, value);
-        if (query == null) {
-            return null;
-        }
-
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        } else if (query instanceof GraphQuery && ((GraphQuery) query).hasBoolean()) {
-            // we have a graph query that has at least one boolean sub-query
-            // re-build and set minimum should match value on all boolean queries
-            List<Query> oldQueries = ((GraphQuery) query).getQueries();
-            Query[] queries = new Query[oldQueries.size()];
-            for (int i = 0; i < queries.length; i++) {
-                Query oldQuery = oldQueries.get(i);
-                if (oldQuery instanceof BooleanQuery) {
-                    queries[i] = Queries.applyMinimumShouldMatch((BooleanQuery) oldQuery, minimumShouldMatch);
-                } else {
-                    queries[i] = oldQuery;
-                }
-            }
-
-            query = new GraphQuery(queries);
-        } else if (query instanceof ExtendedCommonTermsQuery) {
-            ((ExtendedCommonTermsQuery)query).setLowFreqMinimumNumberShouldMatch(minimumShouldMatch);
-        }
-
-        return query;
+        return Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
     }
 
     @Override
@@ -556,9 +528,9 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                        if (QUERY_FIELD.match(currentFieldName)) {
                             value = parser.objectText();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                        } else if (TYPE_FIELD.match(currentFieldName)) {
                             String tStr = parser.text();
                             if ("boolean".equals(tStr)) {
                                 type = MatchQuery.Type.BOOLEAN;
@@ -569,31 +541,31 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
                             } else {
                                 throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] query does not support type " + tStr);
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                        } else if (ANALYZER_FIELD.match(currentFieldName)) {
                             analyzer = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                        } else if (SLOP_FIELD.match(currentFieldName)) {
                             slop = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                        } else if (Fuzziness.FIELD.match(currentFieldName)) {
                             fuzziness = Fuzziness.parse(parser);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                        } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                             prefixLength = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                        } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                             maxExpansion = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) {
+                        } else if (OPERATOR_FIELD.match(currentFieldName)) {
                             operator = Operator.fromString(parser.text());
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                        } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                             minimumShouldMatch = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                        } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                             fuzzyRewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_TRANSPOSITIONS_FIELD)) {
+                        } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName)) {
                             fuzzyTranspositions = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                        } else if (LENIENT_FIELD.match(currentFieldName)) {
                             lenient = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                        } else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                             cutOffFrequency = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) {
+                        } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName)) {
                             String zeroTermsDocs = parser.text();
                             if ("none".equalsIgnoreCase(zeroTermsDocs)) {
                                 zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE;
@@ -603,7 +575,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
                                 throw new ParsingException(parser.getTokenLocation(),
                                         "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]");
                             }
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index cad4f11214e..ee0eff2a451 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -28,6 +28,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
@@ -36,7 +37,6 @@ import org.elasticsearch.action.termvectors.TermVectorsResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -52,10 +52,10 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -162,6 +162,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         private String type;
         private String id;
         private BytesReference doc;
+        private XContentType xContentType;
         private String[] fields;
         private Map<String, String> perFieldAnalyzer;
         private String routing;
@@ -179,6 +180,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             this.type = copy.type;
             this.id = copy.id;
             this.doc = copy.doc;
+            this.xContentType = copy.xContentType;
             this.fields = copy.fields;
             this.perFieldAnalyzer = copy.perFieldAnalyzer;
             this.version = copy.version;
@@ -215,6 +217,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             this.index = index;
             this.type = type;
             this.doc = doc.bytes();
+            this.xContentType = doc.contentType();
         }
 
         /**
@@ -225,6 +228,11 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             type = in.readOptionalString();
             if (in.readBoolean()) {
                 doc = (BytesReference) in.readGenericValue();
+                if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+                    xContentType = XContentType.readFrom(in);
+                } else {
+                    xContentType = XContentFactory.xContentType(doc);
+                }
             } else {
                 id = in.readString();
             }
@@ -242,6 +250,9 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             out.writeBoolean(doc != null);
             if (doc != null) {
                 out.writeGenericValue(doc);
+                if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+                    xContentType.writeTo(out);
+                }
             } else {
                 out.writeString(id);
             }
@@ -326,6 +337,10 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             return this;
         }
 
+        XContentType xContentType() {
+            return xContentType;
+        }
+
         /**
          * Convert this to a {@link TermVectorsRequest} for fetching the terms of the document.
          */
@@ -343,7 +358,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                     .termStatistics(false);
             // for artificial docs to make sure that the id has changed in the item too
             if (doc != null) {
-                termVectorsRequest.doc(doc, true);
+                termVectorsRequest.doc(doc, true, xContentType);
                 this.id = termVectorsRequest.id();
             }
             return termVectorsRequest;
@@ -352,22 +367,23 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         /**
          * Parses and returns the given item.
          */
-        public static Item parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, Item item) throws IOException {
+        public static Item parse(XContentParser parser, Item item) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (currentFieldName != null) {
-                    if (parseFieldMatcher.match(currentFieldName, Field.INDEX)) {
+                    if (Field.INDEX.match(currentFieldName)) {
                         item.index = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.TYPE)) {
+                    } else if (Field.TYPE.match(currentFieldName)) {
                         item.type = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.ID)) {
+                    } else if (Field.ID.match(currentFieldName)) {
                         item.id = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.DOC)) {
+                    } else if (Field.DOC.match(currentFieldName)) {
                         item.doc = jsonBuilder().copyCurrentStructure(parser).bytes();
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.FIELDS)) {
+                        item.xContentType = XContentType.JSON;
+                    } else if (Field.FIELDS.match(currentFieldName)) {
                         if (token == XContentParser.Token.START_ARRAY) {
                             List<String> fields = new ArrayList<>();
                             while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
@@ -378,7 +394,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                             throw new ElasticsearchParseException(
                                     "failed to parse More Like This item. field [fields] must be an array");
                         }
-                    } else if (parseFieldMatcher.match(currentFieldName, Field.PER_FIELD_ANALYZER)) {
+                    } else if (Field.PER_FIELD_ANALYZER.match(currentFieldName)) {
                         item.perFieldAnalyzer(TermVectorsRequest.readPerFieldAnalyzer(parser.map()));
                     } else if ("_routing".equals(currentFieldName) || "routing".equals(currentFieldName)) {
                         item.routing = parser.text();
@@ -417,16 +433,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                 builder.field(Field.ID.getPreferredName(), this.id);
             }
             if (this.doc != null) {
-                XContentType contentType = XContentFactory.xContentType(this.doc);
-                if (contentType == builder.contentType()) {
-                    builder.rawField(Field.DOC.getPreferredName(), this.doc);
-                } else {
-                    try (XContentParser parser = XContentFactory.xContent(contentType).createParser(this.doc)) {
-                        parser.nextToken();
-                        builder.field(Field.DOC.getPreferredName());
-                        builder.copyCurrentStructure(parser);
-                    }
-                }
+                builder.rawField(Field.DOC.getPreferredName(), this.doc, xContentType);
             }
             if (this.fields != null) {
                 builder.array(Field.FIELDS.getPreferredName(), this.fields);
@@ -838,33 +845,33 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                if (Field.LIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, likeTexts, likeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, unlikeTexts, unlikeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE_TEXT)) {
+                } else if (Field.LIKE_TEXT.match(currentFieldName)) {
                     likeTexts.add(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_QUERY_TERMS)) {
+                } else if (Field.MAX_QUERY_TERMS.match(currentFieldName)) {
                     maxQueryTerms = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_TERM_FREQ)) {
+                } else if (Field.MIN_TERM_FREQ.match(currentFieldName)) {
                     minTermFreq =parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_DOC_FREQ)) {
+                } else if (Field.MIN_DOC_FREQ.match(currentFieldName)) {
                     minDocFreq = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_DOC_FREQ)) {
+                } else if (Field.MAX_DOC_FREQ.match(currentFieldName)) {
                     maxDocFreq = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MIN_WORD_LENGTH)) {
+                } else if (Field.MIN_WORD_LENGTH.match(currentFieldName)) {
                     minWordLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MAX_WORD_LENGTH)) {
+                } else if (Field.MAX_WORD_LENGTH.match(currentFieldName)) {
                     maxWordLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.ANALYZER)) {
+                } else if (Field.ANALYZER.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.MINIMUM_SHOULD_MATCH)) {
+                } else if (Field.MINIMUM_SHOULD_MATCH.match(currentFieldName)) {
                     minimumShouldMatch = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.BOOST_TERMS)) {
+                } else if (Field.BOOST_TERMS.match(currentFieldName)) {
                     boostTerms = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.INCLUDE)) {
+                } else if (Field.INCLUDE.match(currentFieldName)) {
                     include = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.FAIL_ON_UNSUPPORTED_FIELD)) {
+                } else if (Field.FAIL_ON_UNSUPPORTED_FIELD.match(currentFieldName)) {
                     failOnUnsupportedField = parser.booleanValue();
                 } else if ("boost".equals(currentFieldName)) {
                     boost = parser.floatValue();
@@ -874,34 +881,34 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.FIELDS)) {
+                if (Field.FIELDS.match(currentFieldName)) {
                     fields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         fields.add(parser.text());
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                } else if (Field.LIKE.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseLikeField(parseContext, likeTexts, likeItems);
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseLikeField(parseContext, unlikeTexts, unlikeItems);
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.IDS)) {
+                } else if (Field.IDS.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (!token.isValue()) {
                             throw new IllegalArgumentException("ids array element should only contain ids");
                         }
                         likeItems.add(new Item(null, null, parser.text()));
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.DOCS)) {
+                } else if (Field.DOCS.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token != XContentParser.Token.START_OBJECT) {
                             throw new IllegalArgumentException("docs array element should include an object");
                         }
-                        likeItems.add(Item.parse(parser, parseContext.getParseFieldMatcher(), new Item()));
+                        likeItems.add(Item.parse(parser, new Item()));
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.STOP_WORDS)) {
+                } else if (Field.STOP_WORDS.match(currentFieldName)) {
                     stopWords = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         stopWords.add(parser.text());
@@ -910,9 +917,9 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.LIKE)) {
+                if (Field.LIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, likeTexts, likeItems);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Field.UNLIKE)) {
+                } else if (Field.UNLIKE.match(currentFieldName)) {
                     parseLikeField(parseContext, unlikeTexts, unlikeItems);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[mlt] query does not support [" + currentFieldName + "]");
@@ -960,7 +967,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         if (parser.currentToken().isValue()) {
             texts.add(parser.text());
         } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-            items.add(Item.parse(parser, parseContext.getParseFieldMatcher(), new Item()));
+            items.add(Item.parse(parser, new Item()));
         } else {
             throw new IllegalArgumentException("Content of 'like' parameter should either be a string or an object");
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index b6ead4a551e..b668b8d9831 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -43,7 +42,6 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.TreeMap;
 
 /**
@@ -148,11 +146,11 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
             return parseField;
         }
 
-        public static Type parse(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static Type parse(String value) {
             MultiMatchQueryBuilder.Type[] values = MultiMatchQueryBuilder.Type.values();
             Type type = null;
             for (MultiMatchQueryBuilder.Type t : values) {
-                if (parseFieldMatcher.match(value, t.parseField())) {
+                if (t.parseField().match(value)) {
                     type = t;
                     break;
                 }
@@ -304,7 +302,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
         if (type == null) {
             throw new IllegalArgumentException("[" + NAME + "] requires type to be non-null");
         }
-        this.type = Type.parse(type.toString().toLowerCase(Locale.ROOT), ParseFieldMatcher.EMPTY);
+        this.type = Type.parse(type.toString().toLowerCase(Locale.ROOT));
         return this;
     }
 
@@ -584,7 +582,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+            } else if (FIELDS_FIELD.match(currentFieldName)) {
                 if (token == XContentParser.Token.START_ARRAY) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         parseFieldAndBoost(parser, fieldsBoosts);
@@ -596,37 +594,37 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
                             "[" + NAME + "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     value = parser.objectText();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
-                    type = MultiMatchQueryBuilder.Type.parse(parser.text(), parseContext.getParseFieldMatcher());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (TYPE_FIELD.match(currentFieldName)) {
+                    type = MultiMatchQueryBuilder.Type.parse(parser.text());
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                } else if (SLOP_FIELD.match(currentFieldName)) {
                     slop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                } else if (Fuzziness.FIELD.match(currentFieldName)) {
                     fuzziness = Fuzziness.parse(parser);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     prefixLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) {
+                } else if (MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                     maxExpansions = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) {
+                } else if (OPERATOR_FIELD.match(currentFieldName)) {
                     operator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                     fuzzyRewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) {
+                } else if (USE_DIS_MAX_FIELD.match(currentFieldName)) {
                     useDisMax = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                }  else if (parseContext.getParseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) {
+                }  else if (CUTOFF_FREQUENCY_FIELD.match(currentFieldName)) {
                     cutoffFrequency = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) {
+                } else if (ZERO_TERMS_QUERY_FIELD.match(currentFieldName)) {
                     String zeroTermsDocs = parser.text();
                     if ("none".equalsIgnoreCase(zeroTermsDocs)) {
                         zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE;
@@ -635,7 +633,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
                     } else {
                         throw new ParsingException(parser.getTokenLocation(), "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]");
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -721,7 +719,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
         multiMatchQuery.setMaxExpansions(maxExpansions);
         multiMatchQuery.setOccur(operator.toBooleanClauseOccur());
         if (fuzzyRewrite != null) {
-            multiMatchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), fuzzyRewrite, null));
+            multiMatchQuery.setFuzzyRewriteMethod(QueryParsers.parseRewriteMethod(fuzzyRewrite, null));
         }
         if (tieBreaker != null) {
             multiMatchQuery.setTieBreaker(tieBreaker);
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 6c1a8b27f6b..322db3815f9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -103,8 +103,8 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         return innerHitBuilder;
     }
 
-    public NestedQueryBuilder innerHit(InnerHitBuilder innerHit) {
-        this.innerHitBuilder = new InnerHitBuilder(innerHit, path, query);
+    public NestedQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) {
+        this.innerHitBuilder = new InnerHitBuilder(innerHit, path, query, ignoreUnmapped);
         return this;
     }
 
@@ -166,23 +166,23 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     query = parseContext.parseInnerQueryBuilder();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) {
+                } else if (INNER_HITS_FIELD.match(currentFieldName)) {
                     innerHitBuilder = InnerHitBuilder.fromXContent(parseContext);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, PATH_FIELD)) {
+                if (PATH_FIELD.match(currentFieldName)) {
                     path = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
+                } else if (SCORE_MODE_FIELD.match(currentFieldName)) {
                     scoreMode = HasChildQueryBuilder.parseScoreMode(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]");
@@ -194,7 +194,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
                 .queryName(queryName)
                 .boost(boost);
         if (innerHitBuilder != null) {
-            queryBuilder.innerHit(innerHitBuilder);
+            queryBuilder.innerHit(innerHitBuilder, ignoreUnmapped);
         }
         return queryBuilder;
     }
@@ -255,7 +255,9 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         QueryBuilder rewrittenQuery = query.rewrite(queryRewriteContext);
         if (rewrittenQuery != query) {
             InnerHitBuilder rewrittenInnerHit = InnerHitBuilder.rewrite(innerHitBuilder, rewrittenQuery);
-            return new NestedQueryBuilder(path, rewrittenQuery, scoreMode, rewrittenInnerHit);
+            NestedQueryBuilder nestedQuery = new NestedQueryBuilder(path, rewrittenQuery, scoreMode, rewrittenInnerHit);
+            nestedQuery.ignoreUnmapped(ignoreUnmapped);
+            return nestedQuery;
         }
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
index f85e232032a..f5608df0703 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
@@ -128,15 +128,15 @@ public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQue
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, TYPE_FIELD)) {
+                if (TYPE_FIELD.match(currentFieldName)) {
                     type = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ID_FIELD)) {
+                } else if (ID_FIELD.match(currentFieldName)) {
                     id = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, IGNORE_UNMAPPED_FIELD)) {
+                } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName)) {
                     ignoreUnmapped = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[parent_id] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 678175916b2..7410ca68afc 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -139,13 +139,13 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder>
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PREFIX_FIELD)) {
+                        } else if (PREFIX_FIELD.match(currentFieldName)) {
                             value = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -173,7 +173,7 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder>
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null);
 
         Query query = null;
         MappedFieldType fieldType = context.fieldMapper(fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index b944bd3a42c..74e10dcebff 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -20,35 +20,30 @@
 package org.elasticsearch.index.query;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.Script;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class QueryParseContext implements ParseFieldMatcherSupplier {
+public class QueryParseContext {
 
     private static final ParseField CACHE = new ParseField("_cache").withAllDeprecated("Elasticsearch makes its own caching decisions");
     private static final ParseField CACHE_KEY = new ParseField("_cache_key").withAllDeprecated("Filters are always used as cache keys");
 
     private final XContentParser parser;
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-    private final ParseFieldMatcher parseFieldMatcher;
     private final String defaultScriptLanguage;
 
-    public QueryParseContext(IndicesQueriesRegistry registry, XContentParser parser, ParseFieldMatcher parseFieldMatcher) {
-        this(Script.DEFAULT_SCRIPT_LANG, registry, parser, parseFieldMatcher);
+    public QueryParseContext(XContentParser parser) {
+        this(Script.DEFAULT_SCRIPT_LANG, parser);
     }
 
-    public QueryParseContext(String defaultScriptLanguage, IndicesQueriesRegistry registry, XContentParser parser,
-                             ParseFieldMatcher parseFieldMatcher) {
-        this.indicesQueriesRegistry = Objects.requireNonNull(registry, "indices queries registry cannot be null");
+    //TODO this constructor can be removed from master branch
+    public QueryParseContext(String defaultScriptLanguage, XContentParser parser) {
         this.parser = Objects.requireNonNull(parser, "parser cannot be null");
-        this.parseFieldMatcher = Objects.requireNonNull(parseFieldMatcher, "parse field matcher cannot be null");
         this.defaultScriptLanguage = defaultScriptLanguage;
     }
 
@@ -57,7 +52,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
     }
 
     public boolean isDeprecatedSetting(String setting) {
-        return this.parseFieldMatcher.match(setting, CACHE) || this.parseFieldMatcher.match(setting, CACHE_KEY);
+        return CACHE.match(setting) || CACHE_KEY.match(setting);
     }
 
     /**
@@ -105,7 +100,15 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new ParsingException(parser.getTokenLocation(), "[" + queryName + "] query malformed, no start_object after query name");
         }
-        QueryBuilder result = indicesQueriesRegistry.lookup(queryName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(this);
+        QueryBuilder result;
+        try {
+            result = parser.namedObject(QueryBuilder.class, queryName, this);
+        } catch (UnknownNamedObjectException e) {
+            // Preserve the error message from 5.0 until we have a compellingly better message so we don't break BWC.
+            // This intentionally doesn't include the causing exception because that'd change the "root_cause" of any unknown query errors
+            throw new ParsingException(new XContentLocation(e.getLineNumber(), e.getColumnNumber()),
+                    "no [query] registered for [" + e.getName() + "]");
+        }
         //end_object of the specific query (e.g. match, multi_match etc.) element
         if (parser.currentToken() != XContentParser.Token.END_OBJECT) {
             throw new ParsingException(parser.getTokenLocation(),
@@ -119,11 +122,6 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
         return result;
     }
 
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return parseFieldMatcher;
-    }
-
     /**
      * Returns the default scripting language, that should be used if scripts don't specify the script language
      * explicitly.
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index 6521369453c..80726496a73 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -20,13 +20,11 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.index.IndexReader;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -37,22 +35,22 @@ import java.util.function.LongSupplier;
 /**
  * Context object used to rewrite {@link QueryBuilder} instances into simplified version.
  */
-public class QueryRewriteContext implements ParseFieldMatcherSupplier {
+public class QueryRewriteContext {
     protected final MapperService mapperService;
     protected final ScriptService scriptService;
     protected final IndexSettings indexSettings;
-    protected final IndicesQueriesRegistry indicesQueriesRegistry;
+    private final NamedXContentRegistry xContentRegistry;
     protected final Client client;
     protected final IndexReader reader;
     protected final LongSupplier nowInMillis;
 
     public QueryRewriteContext(IndexSettings indexSettings, MapperService mapperService, ScriptService scriptService,
-                               IndicesQueriesRegistry indicesQueriesRegistry, Client client, IndexReader reader,
-                               LongSupplier nowInMillis) {
+            NamedXContentRegistry xContentRegistry, Client client, IndexReader reader,
+            LongSupplier nowInMillis) {
         this.mapperService = mapperService;
         this.scriptService = scriptService;
         this.indexSettings = indexSettings;
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
+        this.xContentRegistry = xContentRegistry;
         this.client = client;
         this.reader = reader;
         this.nowInMillis = nowInMillis;
@@ -87,17 +85,18 @@ public class QueryRewriteContext implements ParseFieldMatcherSupplier {
         return reader;
     }
 
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return this.indexSettings.getParseFieldMatcher();
+    /**
+     * The registry used to build new {@link XContentParser}s. Contains registered named parsers needed to parse the query.
+     */
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
     }
 
     /**
-     * Returns a new {@link QueryParseContext} that wraps the provided parser, using the ParseFieldMatcher settings that
-     * are configured in the index settings. The default script language will always default to Painless.
+     * Returns a new {@link QueryParseContext} that wraps the provided parser.
      */
     public QueryParseContext newParseContext(XContentParser parser) {
-        return new QueryParseContext(indicesQueriesRegistry, parser, indexSettings.getParseFieldMatcher());
+        return new QueryParseContext(parser);
     }
 
     public long nowInMillis() {
@@ -108,6 +107,4 @@ public class QueryRewriteContext implements ParseFieldMatcherSupplier {
         ExecutableScript executable = scriptService.executable(template, ScriptContext.Standard.SEARCH);
         return (BytesReference) executable.run();
     }
-
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 1d5fc1bc9a1..2b5e69947f3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -29,10 +29,12 @@ import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -40,6 +42,7 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -47,7 +50,6 @@ import org.elasticsearch.index.mapper.ObjectMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -91,17 +93,16 @@ public class QueryShardContext extends QueryRewriteContext {
 
     private final Map<String, Query> namedQueries = new HashMap<>();
     private final MapperQueryParser queryParser = new MapperQueryParser(this);
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
     private boolean isFilter;
 
     public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache,
-                             IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService,
-                             ScriptService scriptService, final IndicesQueriesRegistry indicesQueriesRegistry, Client client,
-                             IndexReader reader, LongSupplier nowInMillis) {
-        super(indexSettings, mapperService, scriptService, indicesQueriesRegistry, client, reader, nowInMillis);
+            IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService,
+            ScriptService scriptService, NamedXContentRegistry xContentRegistry,
+            Client client, IndexReader reader, LongSupplier nowInMillis) {
+        super(indexSettings, mapperService, scriptService, xContentRegistry, client, reader, nowInMillis);
         this.shardId = shardId;
         this.indexSettings = indexSettings;
         this.similarityService = similarityService;
@@ -109,14 +110,13 @@ public class QueryShardContext extends QueryRewriteContext {
         this.bitsetFilterCache = bitsetFilterCache;
         this.indexFieldDataService = indexFieldDataService;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.nestedScope = new NestedScope();
 
     }
 
     public QueryShardContext(QueryShardContext source) {
         this(source.shardId, source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService,
-                source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.client,
+                source.similarityService, source.scriptService, source.getXContentRegistry(), source.client,
                 source.reader, source.nowInMillis);
         this.types = source.getTypes();
     }
@@ -209,6 +209,14 @@ public class QueryShardContext extends QueryRewriteContext {
         return mapperService.getObjectMapper(name);
     }
 
+    /**
+     * Returns s {@link DocumentMapper} instance for the given type.
+     * Delegates to {@link MapperService#documentMapper(String)}
+     */
+    public DocumentMapper documentMapper(String type) {
+        return mapperService.documentMapper(type);
+    }
+
     /**
      * Gets the search analyzer for the given field, or the default if there is none present for the field
      * TODO: remove this by moving defaults into mappers themselves
@@ -301,12 +309,7 @@ public class QueryShardContext extends QueryRewriteContext {
         });
     }
 
-    @FunctionalInterface
-    private interface CheckedFunction<T, R> {
-       R apply(T t) throws IOException;
-    }
-
-    private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuilder, Query> filterOrQuery) {
+    private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuilder, Query, IOException> filterOrQuery) {
         reset();
         try {
             QueryBuilder rewriteQuery = QueryBuilder.rewriteQuery(queryBuilder, this);
@@ -338,7 +341,7 @@ public class QueryShardContext extends QueryRewriteContext {
      */
     public final Function<Map<String, Object>, SearchScript> getLazySearchScript(Script script, ScriptContext context) {
         failIfFrozen();
-        CompiledScript compile = scriptService.compile(script, context, script.getOptions());
+        CompiledScript compile = scriptService.compile(script, context);
         return (p) -> scriptService.search(lookup(), compile, p);
     }
 
@@ -357,7 +360,7 @@ public class QueryShardContext extends QueryRewriteContext {
      */
     public final Function<Map<String, Object>, ExecutableScript> getLazyExecutableScript(Script script, ScriptContext context) {
         failIfFrozen();
-        CompiledScript executable = scriptService.compile(script, context, script.getOptions());
+        CompiledScript executable = scriptService.compile(script, context);
         return (p) ->  scriptService.executable(executable, p);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
index 1e31c7c50e1..9b6ce3a6e4b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardException.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.query;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestStatus;
 
@@ -60,11 +59,6 @@ public class QueryShardException extends ElasticsearchException {
         return RestStatus.BAD_REQUEST;
     }
 
-    @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        super.innerToXContent(builder, params);
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
index e9e194c6a28..1947e9cef40 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.Query;
@@ -92,7 +91,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     private static final ParseField QUOTE_ANALYZER_FIELD = new ParseField("quote_analyzer");
     private static final ParseField ALLOW_LEADING_WILDCARD_FIELD = new ParseField("allow_leading_wildcard");
     private static final ParseField AUTO_GENERATE_PHRASE_QUERIES_FIELD = new ParseField("auto_generate_phrase_queries");
-    private static final ParseField MAX_DETERMINED_STATES_FIELD = new ParseField("max_determined_states");
+    private static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states");
     private static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms")
             .withAllDeprecated("Decision is now made by the analyzer");
     private static final ParseField ENABLE_POSITION_INCREMENTS_FIELD = new ParseField("enable_position_increment");
@@ -645,7 +644,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
             builder.field(QUOTE_ANALYZER_FIELD.getPreferredName(), this.quoteAnalyzer);
         }
         builder.field(AUTO_GENERATE_PHRASE_QUERIES_FIELD.getPreferredName(), this.autoGeneratePhraseQueries);
-        builder.field(MAX_DETERMINED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates);
+        builder.field(MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates);
         if (this.allowLeadingWildcard != null) {
             builder.field(ALLOW_LEADING_WILDCARD_FIELD.getPreferredName(), this.allowLeadingWildcard);
         }
@@ -720,7 +719,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                if (FIELDS_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String fField = null;
                         float fBoost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -744,68 +743,70 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
                             "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryString = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_FIELD_FIELD)) {
+                } else if (DEFAULT_FIELD_FIELD.match(currentFieldName)) {
                     defaultField = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) {
+                } else if (DEFAULT_OPERATOR_FIELD.match(currentFieldName)) {
                     defaultOperator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_ANALYZER_FIELD)) {
+                } else if (QUOTE_ANALYZER_FIELD.match(currentFieldName)) {
                     quoteAnalyzer = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALLOW_LEADING_WILDCARD_FIELD)) {
+                } else if (ALLOW_LEADING_WILDCARD_FIELD.match(currentFieldName)) {
                     allowLeadingWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AUTO_GENERATE_PHRASE_QUERIES_FIELD)) {
+                } else if (AUTO_GENERATE_PHRASE_QUERIES_FIELD.match(currentFieldName)) {
                     autoGeneratePhraseQueries = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_DETERMINED_STATES_FIELD)) {
+                } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
                     maxDeterminizedStates = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) {
+                } else if (LOWERCASE_EXPANDED_TERMS_FIELD.match(currentFieldName)) {
                     // ignore, deprecated setting
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ENABLE_POSITION_INCREMENTS_FIELD)) {
+                } else if (ENABLE_POSITION_INCREMENTS_FIELD.match(currentFieldName)) {
                     enablePositionIncrements = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ESCAPE_FIELD)) {
+                } else if (ESCAPE_FIELD.match(currentFieldName)) {
                     escape = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) {
+                } else if (USE_DIS_MAX_FIELD.match(currentFieldName)) {
                     useDisMax = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_PREFIX_LENGTH_FIELD)) {
+                } else if (FUZZY_PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     fuzzyPrefixLength = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_MAX_EXPANSIONS_FIELD)) {
+                } else if (FUZZY_MAX_EXPANSIONS_FIELD.match(currentFieldName)) {
                     fuzzyMaxExpansions = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) {
+                } else if (FUZZY_REWRITE_FIELD.match(currentFieldName)) {
                     fuzzyRewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PHRASE_SLOP_FIELD)) {
+                } else if (PHRASE_SLOP_FIELD.match(currentFieldName)) {
                     phraseSlop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) {
+                } else if (Fuzziness.FIELD.match(currentFieldName)) {
                     fuzziness = Fuzziness.parse(parser);
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) {
+                } else if (TIE_BREAKER_FIELD.match(currentFieldName)) {
                     tieBreaker = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) {
+                } else if (ANALYZE_WILDCARD_FIELD.match(currentFieldName)) {
                     analyzeWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                } else if (REWRITE_FIELD.match(currentFieldName)) {
                     rewrite = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) {
+                } else if (QUOTE_FIELD_SUFFIX_FIELD.match(currentFieldName)) {
                     quoteFieldSuffix = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) {
+                } else if (LOCALE_FIELD.match(currentFieldName)) {
                     // ignore, deprecated setting
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALL_FIELDS_FIELD)) {
+                } else if (ALL_FIELDS_FIELD.match(currentFieldName)) {
                     useAllFields = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) {
+                } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
+                    maxDeterminizedStates = parser.intValue();
+                } else if (TIME_ZONE_FIELD.match(currentFieldName)) {
                     try {
                         timeZone = parser.text();
                     } catch (IllegalArgumentException e) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME +
                                 "] time_zone [" + parser.text() + "] is unknown");
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SPLIT_ON_WHITESPACE)) {
+                } else if (SPLIT_ON_WHITESPACE.match(currentFieldName)) {
                     splitOnWhitespace = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME +
@@ -1012,11 +1013,11 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
         qpSettings.fuzziness(fuzziness);
         qpSettings.fuzzyPrefixLength(fuzzyPrefixLength);
         qpSettings.fuzzyMaxExpansions(fuzzyMaxExpansions);
-        qpSettings.fuzzyRewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), this.fuzzyRewrite));
+        qpSettings.fuzzyRewriteMethod(QueryParsers.parseRewriteMethod(this.fuzzyRewrite));
         qpSettings.phraseSlop(phraseSlop);
         qpSettings.useDisMax(useDisMax);
         qpSettings.tieBreaker(tieBreaker);
-        qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), this.rewrite));
+        qpSettings.rewriteMethod(QueryParsers.parseRewriteMethod(this.rewrite));
         qpSettings.timeZone(timeZone);
         qpSettings.maxDeterminizedStates(maxDeterminizedStates);
         qpSettings.splitOnWhitespace(splitOnWhitespace);
@@ -1042,12 +1043,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
         }
 
         query = Queries.fixNegativeQueryIfNeeded(query);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, this.minimumShouldMatch());
-        }
+        query = Queries.maybeApplyMinimumShouldMatch(query, this.minimumShouldMatch);
 
         //restore the previous BoostQuery wrapping
         for (int i = boosts.size() - 1; i >= 0; i--) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index 5cc9b54a758..bc1bd4bea93 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -371,35 +371,35 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, FROM_FIELD)) {
+                        if (FROM_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TO_FIELD)) {
+                        } else if (TO_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_LOWER_FIELD)) {
+                        } else if (INCLUDE_LOWER_FIELD.match(currentFieldName)) {
                             includeLower = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_UPPER_FIELD)) {
+                        } else if (INCLUDE_UPPER_FIELD.match(currentFieldName)) {
                             includeUpper = parser.booleanValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GT_FIELD)) {
+                        } else if (GT_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
                             includeLower = false;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, GTE_FIELD)) {
+                        } else if (GTE_FIELD.match(currentFieldName)) {
                             from = parser.objectBytes();
                             includeLower = true;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LT_FIELD)) {
+                        } else if (LT_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
                             includeUpper = false;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LTE_FIELD)) {
+                        } else if (LTE_FIELD.match(currentFieldName)) {
                             to = parser.objectBytes();
                             includeUpper = true;
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) {
+                        } else if (TIME_ZONE_FIELD.match(currentFieldName)) {
                             timeZone = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) {
+                        } else if (FORMAT_FIELD.match(currentFieldName)) {
                             format = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, RELATION_FIELD)) {
+                        } else if (RELATION_FIELD.match(currentFieldName)) {
                             relation = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -408,9 +408,9 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
                     }
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
+                if (NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDDATA_FIELD)) {
+                } else if (FIELDDATA_FIELD.match(currentFieldName)) {
                     // ignore
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[range] query does not support [" + currentFieldName + "]");
@@ -469,12 +469,12 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
         case DISJOINT:
             return new MatchNoneQueryBuilder();
         case WITHIN:
-            if (from != null || to != null) {
+            if (from != null || to != null || format != null || timeZone != null) {
                 RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName);
                 newRangeQuery.from(null);
                 newRangeQuery.to(null);
-                newRangeQuery.format = format;
-                newRangeQuery.timeZone = timeZone;
+                newRangeQuery.format = null;
+                newRangeQuery.timeZone = null;
                 return newRangeQuery;
             } else {
                 return this;
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
index c0fc8b80928..e00c19b68b5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
@@ -78,7 +78,7 @@ public enum RegexpFlag {
 
     final int value;
 
-    private RegexpFlag(int value) {
+    RegexpFlag(int value) {
         this.value = value;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index e5632e96c81..7957f0de0a1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -200,20 +200,20 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) {
+                        } else if (FLAGS_FIELD.match(currentFieldName)) {
                             String flags = parser.textOrNull();
                             flagsValue = RegexpFlag.resolveValue(flags);
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_DETERMINIZED_STATES_FIELD)) {
+                        } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
                             maxDeterminizedStates = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_VALUE_FIELD)) {
+                        } else if (FLAGS_VALUE_FIELD.match(currentFieldName)) {
                             flagsValue = parser.intValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -222,7 +222,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
                     }
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, NAME_FIELD)) {
+                if (NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, parser.currentName());
@@ -247,7 +247,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws QueryShardException, IOException {
-        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null);
 
         Query query = null;
         MappedFieldType fieldType = context.fieldMapper(fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 377aeb103d6..5ca91a3d5a6 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -99,18 +99,18 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
             } else if (parseContext.isDeprecatedSetting(currentFieldName)) {
                 // skip
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]");
                 }
@@ -136,7 +136,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
         final Script script;
         final SearchScript searchScript;
 
-        public ScriptQuery(Script script, SearchScript searchScript) {
+        ScriptQuery(Script script, SearchScript searchScript) {
             this.script = script;
             this.searchScript = searchScript;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
index 4a49405ec2f..ddc1c620e3d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryParser.java
@@ -267,10 +267,10 @@ public class SimpleQueryParser extends org.apache.lucene.queryparser.simple.Simp
          * Generates default {@link Settings} object (uses ROOT locale, does
          * lowercase terms, no lenient parsing, no wildcard analysis).
          * */
-        public Settings() {
+        Settings() {
         }
 
-        public Settings(Settings other) {
+        Settings(Settings other) {
             this.lenient = other.lenient;
             this.analyzeWildcard = other.analyzeWildcard;
             this.quoteFieldSuffix = other.quoteFieldSuffix;
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
index 8025b25c30c..0ea2097e0c2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
@@ -37,6 +38,7 @@ import org.elasticsearch.index.query.SimpleQueryParser.Settings;
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -412,15 +414,8 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
 
         SimpleQueryParser sqp = new SimpleQueryParser(luceneAnalyzer, resolvedFieldsAndWeights, flags, newSettings, context);
         sqp.setDefaultOperator(defaultOperator.toBooleanClauseOccur());
-
         Query query = sqp.parse(queryText);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (minimumShouldMatch != null && query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        }
-        return query;
+        return Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
     }
 
     private static String resolveIndexName(String fieldName, QueryShardContext context) {
@@ -492,7 +487,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                if (FIELDS_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String fField = null;
                         float fBoost = 1;
@@ -516,15 +511,15 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
                             "] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryBody = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) {
+                } else if (ANALYZER_FIELD.match(currentFieldName)) {
                     analyzerName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) {
+                } else if (DEFAULT_OPERATOR_FIELD.match(currentFieldName)) {
                     defaultOperator = Operator.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) {
+                } else if (FLAGS_FIELD.match(currentFieldName)) {
                     if (parser.currentToken() != XContentParser.Token.VALUE_NUMBER) {
                         // Possible options are:
                         // ALL, NONE, AND, OR, PREFIX, PHRASE, PRECEDENCE, ESCAPE, WHITESPACE, FUZZY, NEAR, SLOP
@@ -535,21 +530,21 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
                             flags = SimpleQueryStringFlag.ALL.value();
                         }
                     }
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) {
+                } else if (LOCALE_FIELD.match(currentFieldName)) {
                     // ignore, deprecated setting
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) {
+                } else if (LOWERCASE_EXPANDED_TERMS_FIELD.match(currentFieldName)) {
                     // ignore, deprecated setting
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) {
+                } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) {
+                } else if (ANALYZE_WILDCARD_FIELD.match(currentFieldName)) {
                     analyzeWildcard = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) {
+                } else if (MINIMUM_SHOULD_MATCH_FIELD.match(currentFieldName)) {
                     minimumShouldMatch = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) {
+                } else if (QUOTE_FIELD_SUFFIX_FIELD.match(currentFieldName)) {
                     quoteFieldSuffix = parser.textOrNull();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, ALL_FIELDS_FIELD)) {
+                } else if (ALL_FIELDS_FIELD.match(currentFieldName)) {
                     useAllFields = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + SimpleQueryStringBuilder.NAME +
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
index 68d19db7cc6..e8cbe035c90 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringFlag.java
@@ -43,7 +43,7 @@ public enum SimpleQueryStringFlag {
 
     final int value;
 
-    private SimpleQueryStringFlag(int value) {
+    SimpleQueryStringFlag(int value) {
         this.value = value;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
index a6687a22019..08a3e62fc61 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
@@ -112,13 +112,13 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
+                if (BIG_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query");
                     }
                     big = (SpanQueryBuilder) query;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
+                } else if (LITTLE_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query");
@@ -128,9 +128,9 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_containing] query does not support [" + currentFieldName + "]");
                 }
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+            } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                 boost = parser.floatValue();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+            } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                 queryName = parser.text();
             } else {
                 throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
index b84c50b08ff..d1d27924cc6 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
@@ -114,7 +114,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
+                if (MATCH_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanFirst [match] must be of type span query");
@@ -124,11 +124,11 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
                     throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, END_FIELD)) {
+                } else if (END_FIELD.match(currentFieldName)) {
                     end = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index 1b276c292d0..c703996d168 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -92,7 +92,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
+                if (MATCH_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof MultiTermQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(),
@@ -103,9 +103,9 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
                     throw new ParsingException(parser.getTokenLocation(), "[span_multi] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_multi] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index ee4eacd31fc..bc022e81199 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -159,7 +159,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
+                if (CLAUSES_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         QueryBuilder query = parseContext.parseInnerQueryBuilder();
                         if (query instanceof SpanQueryBuilder == false) {
@@ -171,13 +171,13 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
                     throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, IN_ORDER_FIELD)) {
+                if (IN_ORDER_FIELD.match(currentFieldName)) {
                     inOrder = parser.booleanValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, SLOP_FIELD)) {
+                } else if (SLOP_FIELD.match(currentFieldName)) {
                     slop = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
index bc51f78b852..865dbc8ae5c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
@@ -180,13 +180,13 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_FIELD)) {
+                if (INCLUDE_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [include] must be of type span query");
                     }
                     include = (SpanQueryBuilder) query;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, EXCLUDE_FIELD)) {
+                } else if (EXCLUDE_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [exclude] must be of type span query");
@@ -196,15 +196,15 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
                     throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DIST_FIELD)) {
+                if (DIST_FIELD.match(currentFieldName)) {
                     dist = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, PRE_FIELD)) {
+                } else if (PRE_FIELD.match(currentFieldName)) {
                     pre = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, POST_FIELD)) {
+                } else if (POST_FIELD.match(currentFieldName)) {
                     post = parser.intValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index e1e3794f7cd..afe315c61f7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -111,7 +111,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
+                if (CLAUSES_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         QueryBuilder query = parseContext.parseInnerQueryBuilder();
                         if (query instanceof SpanQueryBuilder == false) {
@@ -123,9 +123,9 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
                     throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]");
                 }
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
index fae55b35e66..67319dfe6e4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
@@ -109,13 +109,13 @@ public class SpanTermQueryBuilder extends BaseTermQueryBuilder<SpanTermQueryBuil
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BaseTermQueryBuilder.VALUE_FIELD)) {
+                        } else if (BaseTermQueryBuilder.VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
index d44585e3f9f..7a7cf7f0403 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
@@ -118,13 +118,13 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
+                if (BIG_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [big] must be of type span query");
                     }
                     big = (SpanQueryBuilder) query;
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
+                } else if (LITTLE_FIELD.match(currentFieldName)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     if (query instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [little] must be of type span query");
@@ -134,9 +134,9 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_within] query does not support [" + currentFieldName + "]");
                 }
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+            } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                 boost = parser.floatValue();
-            } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+            } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                 queryName = parser.text();
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "[span_within] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
index b3062525638..fe6282a257e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
@@ -103,13 +103,13 @@ public class TermQueryBuilder extends BaseTermQueryBuilder<TermQueryBuilder> {
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, TERM_FIELD)) {
+                        if (TERM_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.objectBytes();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index c301aedcd89..198537f0060 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -245,7 +245,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
         final boolean allStrings = list.stream().allMatch(o -> o != null && STRING_TYPES.contains(o.getClass()));
         if (allStrings) {
             final BytesRefBuilder builder = new BytesRefBuilder();
-            try (final BytesStreamOutput bytesOut = new BytesStreamOutput()) {
+            try (BytesStreamOutput bytesOut = new BytesStreamOutput()) {
                 final int[] endOffsets = new int[list.size()];
                 int i = 0;
                 for (Object o : list) {
@@ -353,9 +353,9 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
                 fieldName = currentFieldName;
                 termsLookup = TermsLookup.parseTermsLookup(parser);
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
index e7fb520faf1..88e655555f8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
@@ -93,11 +93,11 @@ public class TypeQueryBuilder extends AbstractQueryBuilder<TypeQueryBuilder> {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                } else if (VALUE_FIELD.match(currentFieldName)) {
                     type = parser.utf8Bytes();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index bfe173664f4..006f55c8710 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -153,15 +153,15 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder<WildcardQueryBuil
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, WILDCARD_FIELD)) {
+                        if (WILDCARD_FIELD.match(currentFieldName)) {
                             value = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, VALUE_FIELD)) {
+                        } else if (VALUE_FIELD.match(currentFieldName)) {
                             value = parser.text();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                        } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                             boost = parser.floatValue();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) {
+                        } else if (REWRITE_FIELD.match(currentFieldName)) {
                             rewrite = parser.textOrNull();
-                        } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                        } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                             queryName = parser.text();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
@@ -194,7 +194,7 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder<WildcardQueryBuil
         }
 
         WildcardQuery query = new WildcardQuery(term);
-        MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(context.getParseFieldMatcher(), rewrite, null);
+        MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null);
         QueryParsers.setRewriteMethod(query, rewriteMethod);
         return query;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 1702679e933..779efaa4410 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -123,7 +123,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
             throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed");
         }
         String fieldName = parser.currentName();
-        if (! parseContext.getParseFieldMatcher().match(fieldName, QUERY_FIELD)) {
+        if (! QUERY_FIELD.match(fieldName)) {
             throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed, expected `query` but was " + fieldName);
         }
         parser.nextToken();
@@ -160,7 +160,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException {
-        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
+        try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(context.getXContentRegistry(), source)) {
             QueryParseContext parseContext = context.newParseContext(qSourceParser);
 
             final QueryBuilder queryBuilder = parseContext.parseInnerQueryBuilder();
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
index cf36ea9dffd..6235f654884 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.lucene.search.function.LeafScoreFunction;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -145,10 +146,7 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB>
     @Override
     public void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(getName());
-        builder.field(fieldName);
-        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(functionBytes)) {
-            builder.copyCurrentStructure(parser);
-        }
+        builder.rawField(fieldName, functionBytes);
         builder.field(DecayFunctionParser.MULTI_VALUE_MODE.getPreferredName(), multiValueMode.name());
         builder.endObject();
     }
@@ -181,7 +179,8 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB>
     @Override
     protected ScoreFunction doToFunction(QueryShardContext context) throws IOException {
         AbstractDistanceScoreFunction scoreFunction;
-        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(functionBytes)) {
+        // EMPTY is safe because parseVariable doesn't use namedObject
+        try (XContentParser parser = XContentFactory.xContent(functionBytes).createParser(NamedXContentRegistry.EMPTY, functionBytes)) {
             scoreFunction = parseVariable(fieldName, parser, context, multiValueMode);
         }
         return scoreFunction;
@@ -330,9 +329,9 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB>
         private final GeoPoint origin;
         private final IndexGeoPointFieldData fieldData;
 
-        private static final GeoDistance distFunction = GeoDistance.DEFAULT;
+        private static final GeoDistance distFunction = GeoDistance.ARC;
 
-        public GeoFieldDataScoreFunction(GeoPoint origin, double scale, double decay, double offset, DecayFunction func,
+        GeoFieldDataScoreFunction(GeoPoint origin, double scale, double decay, double offset, DecayFunction func,
                                          IndexGeoPointFieldData fieldData, MultiValueMode mode) {
             super(scale, decay, offset, func, mode);
             this.origin = origin;
@@ -414,7 +413,7 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB>
         private final IndexNumericFieldData fieldData;
         private final double origin;
 
-        public NumericFieldDataScoreFunction(double origin, double scale, double decay, double offset, DecayFunction func,
+        NumericFieldDataScoreFunction(double origin, double scale, double decay, double offset, DecayFunction func,
                                              IndexNumericFieldData fieldData, MultiValueMode mode) {
             super(scale, decay, offset, func, mode);
             this.fieldData = fieldData;
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
index 2c8b9af28d6..5a767ecc3a0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionParser.java
@@ -112,7 +112,7 @@ public final class DecayFunctionParser<DFB extends DecayFunctionBuilder<DFB>> im
                 XContentBuilder builder = XContentFactory.jsonBuilder();
                 builder.copyCurrentStructure(parser);
                 functionBytes = builder.bytes();
-            } else if (context.getParseFieldMatcher().match(currentFieldName, MULTI_VALUE_MODE)) {
+            } else if (MULTI_VALUE_MODE.match(currentFieldName)) {
                 multiValueMode = MultiValueMode.fromString(parser.text());
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "malformed score function score parameters.");
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index b037261d1f0..fcb4ab4ddc6 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery.FilterFunction;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
@@ -436,8 +435,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         InnerHitBuilder.extractInnerHits(query(), innerHits);
     }
 
-    public static FunctionScoreQueryBuilder fromXContent(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
-            QueryParseContext parseContext) throws IOException {
+    public static FunctionScoreQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         QueryBuilder query = null;
@@ -461,7 +459,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     if (query != null) {
                         throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. [query] is already defined.",
                                 NAME);
@@ -481,38 +479,35 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                     singleFunctionFound = true;
                     singleFunctionName = currentFieldName;
 
-                    // we try to parse a score function. If there is no score function for the current field name,
-                    // getScoreFunction will throw.
-                    ScoreFunctionBuilder<?> scoreFunction = scoreFunctionsRegistry
-                            .lookup(currentFieldName, parseContext.getParseFieldMatcher(), parser.getTokenLocation())
-                            .fromXContent(parseContext);
+                        ScoreFunctionBuilder<?> scoreFunction = parser.namedObject(ScoreFunctionBuilder.class, currentFieldName,
+                                parseContext);
                     filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(scoreFunction));
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, FUNCTIONS_FIELD)) {
+                if (FUNCTIONS_FIELD.match(currentFieldName)) {
                     if (singleFunctionFound) {
                         String errorString = "already found [" + singleFunctionName + "], now encountering [functions].";
                         handleMisplacedFunctionsDeclaration(parser.getTokenLocation(), errorString);
                     }
                     functionArrayFound = true;
-                    currentFieldName = parseFiltersAndFunctions(scoreFunctionsRegistry, parseContext, filterFunctionBuilders);
+                    currentFieldName = parseFiltersAndFunctions(parseContext, filterFunctionBuilders);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. array [{}] is not supported",
                             NAME, currentFieldName);
                 }
 
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
+                if (SCORE_MODE_FIELD.match(currentFieldName)) {
                     scoreMode = FiltersFunctionScoreQuery.ScoreMode.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, BOOST_MODE_FIELD)) {
+                } else if (BOOST_MODE_FIELD.match(currentFieldName)) {
                     combineFunction = CombineFunction.fromString(parser.text());
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MAX_BOOST_FIELD)) {
+                } else if (MAX_BOOST_FIELD.match(currentFieldName)) {
                     maxBoost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
+                } else if (MIN_SCORE_FIELD.match(currentFieldName)) {
                     minScore = parser.floatValue();
                 } else {
                     if (singleFunctionFound) {
@@ -525,7 +520,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                         String errorString = "already found [functions] array, now encountering [" + currentFieldName + "].";
                         handleMisplacedFunctionsDeclaration(parser.getTokenLocation(), errorString);
                     }
-                    if (parseContext.getParseFieldMatcher().match(currentFieldName, WEIGHT_FIELD)) {
+                    if (WEIGHT_FIELD.match(currentFieldName)) {
                         filterFunctionBuilders.add(
                                 new FunctionScoreQueryBuilder.FilterFunctionBuilder(new WeightBuilder().setWeight(parser.floatValue())));
                         singleFunctionFound = true;
@@ -562,9 +557,8 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                 MISPLACED_FUNCTION_MESSAGE_PREFIX + errorString);
     }
 
-    private static String parseFiltersAndFunctions(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
-            QueryParseContext parseContext, List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders)
-                    throws IOException {
+    private static String parseFiltersAndFunctions(QueryParseContext parseContext,
+            List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders) throws IOException {
         String currentFieldName = null;
         XContentParser.Token token;
         XContentParser parser = parseContext.parser();
@@ -581,7 +575,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, FILTER_FIELD)) {
+                        if (FILTER_FIELD.match(currentFieldName)) {
                             filter = parseContext.parseInnerQueryBuilder();
                         } else {
                             if (scoreFunction != null) {
@@ -589,11 +583,10 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                                         "failed to parse function_score functions. already found [{}], now encountering [{}].",
                                         scoreFunction.getName(), currentFieldName);
                             }
-                            scoreFunction = scoreFunctionsRegistry.lookup(currentFieldName, parseContext.getParseFieldMatcher(),
-                                    parser.getTokenLocation()).fromXContent(parseContext);
+                            scoreFunction = parser.namedObject(ScoreFunctionBuilder.class, currentFieldName, parseContext);
                         }
                     } else if (token.isValue()) {
-                        if (parseContext.getParseFieldMatcher().match(currentFieldName, WEIGHT_FIELD)) {
+                        if (WEIGHT_FIELD.match(currentFieldName)) {
                             functionWeight = parser.floatValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. field [{}] is not supported",
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
index 3c01c2d92f3..1a2fad90c46 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScoreFunctionParser.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.query.functionscore;
 
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
@@ -29,5 +28,5 @@ import java.io.IOException;
  */
 @FunctionalInterface
 public interface ScoreFunctionParser<FB extends ScoreFunctionBuilder<FB>> {
-    FB fromXContent(QueryParseContext context) throws IOException, ParsingException;
+    FB fromXContent(QueryParseContext context) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
index 8a5e950432b..68913cd9e21 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
@@ -111,8 +111,8 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder<ScriptScore
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, parseContext.getParseFieldMatcher(), parseContext.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, parseContext.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), NAME + " query does not support [" + currentFieldName + "]");
                 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java b/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
index 56cea478a2c..62750d275b1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.query.support;
 import org.apache.lucene.search.MultiTermQuery;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 
 public final class QueryParsers {
 
@@ -44,28 +43,22 @@ public final class QueryParsers {
         query.setRewriteMethod(rewriteMethod);
     }
 
-    public static void setRewriteMethod(MultiTermQuery query, ParseFieldMatcher matcher, @Nullable String rewriteMethod) {
-        if (rewriteMethod == null) {
-            return;
-        }
-        query.setRewriteMethod(parseRewriteMethod(matcher, rewriteMethod));
+    public static MultiTermQuery.RewriteMethod parseRewriteMethod(@Nullable String rewriteMethod) {
+        return parseRewriteMethod(rewriteMethod, MultiTermQuery.CONSTANT_SCORE_REWRITE);
     }
 
-    public static MultiTermQuery.RewriteMethod parseRewriteMethod(ParseFieldMatcher matcher, @Nullable String rewriteMethod) {
-        return parseRewriteMethod(matcher, rewriteMethod, MultiTermQuery.CONSTANT_SCORE_REWRITE);
-    }
-
-    public static MultiTermQuery.RewriteMethod parseRewriteMethod(ParseFieldMatcher matcher, @Nullable String rewriteMethod, @Nullable MultiTermQuery.RewriteMethod defaultRewriteMethod) {
+    public static MultiTermQuery.RewriteMethod parseRewriteMethod(@Nullable String rewriteMethod,
+                                                                  @Nullable MultiTermQuery.RewriteMethod defaultRewriteMethod) {
         if (rewriteMethod == null) {
             return defaultRewriteMethod;
         }
-        if (matcher.match(rewriteMethod, CONSTANT_SCORE)) {
+        if (CONSTANT_SCORE.match(rewriteMethod)) {
             return MultiTermQuery.CONSTANT_SCORE_REWRITE;
         }
-        if (matcher.match(rewriteMethod, SCORING_BOOLEAN)) {
+        if (SCORING_BOOLEAN.match(rewriteMethod)) {
             return MultiTermQuery.SCORING_BOOLEAN_REWRITE;
         }
-        if (matcher.match(rewriteMethod, CONSTANT_SCORE_BOOLEAN)) {
+        if (CONSTANT_SCORE_BOOLEAN.match(rewriteMethod)) {
             return MultiTermQuery.CONSTANT_SCORE_BOOLEAN_REWRITE;
         }
 
@@ -81,18 +74,17 @@ public final class QueryParsers {
             final int size = Integer.parseInt(rewriteMethod.substring(firstDigit));
             String rewriteMethodName = rewriteMethod.substring(0, firstDigit);
 
-            if (matcher.match(rewriteMethodName, TOP_TERMS)) {
+            if (TOP_TERMS.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsScoringBooleanQueryRewrite(size);
             }
-            if (matcher.match(rewriteMethodName, TOP_TERMS_BOOST)) {
+            if (TOP_TERMS_BOOST.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsBoostOnlyBooleanQueryRewrite(size);
             }
-            if (matcher.match(rewriteMethodName, TOP_TERMS_BLENDED_FREQS)) {
+            if (TOP_TERMS_BLENDED_FREQS.match(rewriteMethodName)) {
                 return new MultiTermQuery.TopTermsBlendedFreqScoringRewrite(size);
             }
         }
 
         throw new IllegalArgumentException("Failed to parse rewrite_method [" + rewriteMethod + "]");
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java b/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
index 3a3edd10dcc..8fc16ae1b1d 100644
--- a/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
+++ b/core/src/main/java/org/elasticsearch/index/refresh/RefreshStats.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.refresh;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -27,6 +28,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Objects;
 
 public class RefreshStats implements Streamable, ToXContent {
 
@@ -34,18 +36,19 @@ public class RefreshStats implements Streamable, ToXContent {
 
     private long totalTimeInMillis;
 
+    /**
+     * Number of waiting refresh listeners.
+     */
+    private int listeners;
+
     public RefreshStats() {
 
     }
 
-    public RefreshStats(long total, long totalTimeInMillis) {
+    public RefreshStats(long total, long totalTimeInMillis, int listeners) {
         this.total = total;
         this.totalTimeInMillis = totalTimeInMillis;
-    }
-
-    public void add(long total, long totalTimeInMillis) {
-        this.total += total;
-        this.totalTimeInMillis += totalTimeInMillis;
+        this.listeners = listeners;
     }
 
     public void add(RefreshStats refreshStats) {
@@ -58,6 +61,7 @@ public class RefreshStats implements Streamable, ToXContent {
         }
         this.total += refreshStats.total;
         this.totalTimeInMillis += refreshStats.totalTimeInMillis;
+        this.listeners += refreshStats.listeners;
     }
 
     /**
@@ -81,31 +85,56 @@ public class RefreshStats implements Streamable, ToXContent {
         return new TimeValue(totalTimeInMillis);
     }
 
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.REFRESH);
-        builder.field(Fields.TOTAL, total);
-        builder.timeValueField(Fields.TOTAL_TIME_IN_MILLIS, Fields.TOTAL_TIME, totalTimeInMillis);
-        builder.endObject();
-        return builder;
+    /**
+     * The number of waiting refresh listeners.
+     */
+    public int getListeners() {
+        return listeners;
     }
 
-    static final class Fields {
-        static final String REFRESH = "refresh";
-        static final String TOTAL = "total";
-        static final String TOTAL_TIME = "total_time";
-        static final String TOTAL_TIME_IN_MILLIS = "total_time_in_millis";
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("refresh");
+        builder.field("total", total);
+        builder.timeValueField("total_time_in_millis", "total_time", totalTimeInMillis);
+        builder.field("listeners", listeners);
+        builder.endObject();
+        return builder;
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
         total = in.readVLong();
         totalTimeInMillis = in.readVLong();
+        if (in.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            listeners = in.readVInt();
+        } else {
+            listeners = 0;
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(total);
         out.writeVLong(totalTimeInMillis);
+        if (out.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            out.writeVInt(listeners);
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != RefreshStats.class) {
+            return false;
+        }
+        RefreshStats rhs = (RefreshStats) obj;
+        return total == rhs.total
+                && totalTimeInMillis == rhs.totalTimeInMillis
+                && listeners == rhs.listeners;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(total, totalTimeInMillis, listeners);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
index c9664265d3a..be56463ee1b 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -19,16 +19,7 @@
 
 package org.elasticsearch.index.search;
 
-import static org.apache.lucene.analysis.synonym.SynonymGraphFilter.GRAPH_FLAG;
-
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.synonym.GraphTokenStreamFiniteStrings;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
@@ -58,12 +49,11 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
 public class MatchQuery {
 
-    public static enum Type implements Writeable {
+    public enum Type implements Writeable {
         /**
          * The text is analyzed and terms are added to a boolean query.
          */
@@ -79,7 +69,7 @@ public class MatchQuery {
 
         private final int ordinal;
 
-        private Type(int ordinal) {
+        Type(int ordinal) {
             this.ordinal = ordinal;
         }
 
@@ -99,13 +89,13 @@ public class MatchQuery {
         }
     }
 
-    public static enum ZeroTermsQuery implements Writeable {
+    public enum ZeroTermsQuery implements Writeable {
         NONE(0),
         ALL(1);
 
         private final int ordinal;
 
-        private ZeroTermsQuery(int ordinal) {
+        ZeroTermsQuery(int ordinal) {
             this.ordinal = ordinal;
         }
 
@@ -311,121 +301,11 @@ public class MatchQuery {
         /**
          * Creates a new QueryBuilder using the given analyzer.
          */
-        public MatchQueryBuilder(Analyzer analyzer, @Nullable MappedFieldType mapper) {
+        MatchQueryBuilder(Analyzer analyzer, @Nullable MappedFieldType mapper) {
             super(analyzer);
             this.mapper = mapper;
         }
 
-        /**
-         * Creates a query from the analysis chain.  Overrides original so all it does is create the token stream and pass that into the
-         * new {@link #createFieldQuery(TokenStream, Occur, String, boolean, int)} method which has all the original query generation logic.
-         *
-         * @param analyzer   analyzer used for this query
-         * @param operator   default boolean operator used for this query
-         * @param field      field to create queries against
-         * @param queryText  text to be passed to the analysis chain
-         * @param quoted     true if phrases should be generated when terms occur at more than one position
-         * @param phraseSlop slop factor for phrase/multiphrase queries
-         */
-        @Override
-        protected final Query createFieldQuery(Analyzer analyzer, BooleanClause.Occur operator, String field, String queryText,
-                                               boolean quoted, int phraseSlop) {
-            assert operator == BooleanClause.Occur.SHOULD || operator == BooleanClause.Occur.MUST;
-
-            // Use the analyzer to get all the tokens, and then build an appropriate
-            // query based on the analysis chain.
-            try (TokenStream source = analyzer.tokenStream(field, queryText)) {
-                return createFieldQuery(source, operator, field, quoted, phraseSlop);
-            } catch (IOException e) {
-                throw new RuntimeException("Error analyzing query text", e);
-            }
-        }
-
-        /**
-         * Creates a query from a token stream.  Same logic as {@link #createFieldQuery(Analyzer, Occur, String, String, boolean, int)}
-         * with additional graph token stream detection.
-         *
-         * @param source     the token stream to create the query from
-         * @param operator   default boolean operator used for this query
-         * @param field      field to create queries against
-         * @param quoted     true if phrases should be generated when terms occur at more than one position
-         * @param phraseSlop slop factor for phrase/multiphrase queries
-         */
-        protected final Query createFieldQuery(TokenStream source, BooleanClause.Occur operator, String field, boolean quoted,
-                                               int phraseSlop) {
-            assert operator == BooleanClause.Occur.SHOULD || operator == BooleanClause.Occur.MUST;
-
-            // Build an appropriate query based on the analysis chain.
-            try (CachingTokenFilter stream = new CachingTokenFilter(source)) {
-
-                TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
-                PositionIncrementAttribute posIncAtt = stream.addAttribute(PositionIncrementAttribute.class);
-                PositionLengthAttribute posLenAtt = stream.addAttribute(PositionLengthAttribute.class);
-                FlagsAttribute flagsAtt = stream.addAttribute(FlagsAttribute.class);
-
-                if (termAtt == null) {
-                    return null;
-                }
-
-                // phase 1: read through the stream and assess the situation:
-                // counting the number of tokens/positions and marking if we have any synonyms.
-
-                int numTokens = 0;
-                int positionCount = 0;
-                boolean hasSynonyms = false;
-                boolean isGraph = false;
-
-                stream.reset();
-                while (stream.incrementToken()) {
-                    numTokens++;
-                    int positionIncrement = posIncAtt.getPositionIncrement();
-                    if (positionIncrement != 0) {
-                        positionCount += positionIncrement;
-                    } else {
-                        hasSynonyms = true;
-                    }
-
-                    int positionLength = posLenAtt.getPositionLength();
-                    if (!isGraph && positionLength > 1 && ((flagsAtt.getFlags() & GRAPH_FLAG) == GRAPH_FLAG)) {
-                        isGraph = true;
-                    }
-                }
-
-                // phase 2: based on token count, presence of synonyms, and options
-                // formulate a single term, boolean, or phrase.
-
-                if (numTokens == 0) {
-                    return null;
-                } else if (numTokens == 1) {
-                    // single term
-                    return analyzeTerm(field, stream);
-                } else if (isGraph) {
-                    // graph
-                    return analyzeGraph(stream, operator, field, quoted, phraseSlop);
-                } else if (quoted && positionCount > 1) {
-                    // phrase
-                    if (hasSynonyms) {
-                        // complex phrase with synonyms
-                        return analyzeMultiPhrase(field, stream, phraseSlop);
-                    } else {
-                        // simple phrase
-                        return analyzePhrase(field, stream, phraseSlop);
-                    }
-                } else {
-                    // boolean
-                    if (positionCount == 1) {
-                        // only one position, with synonyms
-                        return analyzeBoolean(field, stream);
-                    } else {
-                        // complex case: multiple positions
-                        return analyzeMultiBoolean(field, stream, operator);
-                    }
-                }
-            } catch (IOException e) {
-                throw new RuntimeException("Error analyzing query text", e);
-            }
-        }
-
         @Override
         protected Query newTermQuery(Term term) {
             return blendTermQuery(term, mapper);
@@ -438,6 +318,21 @@ public class MatchQuery {
 
         public Query createPhrasePrefixQuery(String field, String queryText, int phraseSlop, int maxExpansions) {
             final Query query = createFieldQuery(getAnalyzer(), Occur.MUST, field, queryText, true, phraseSlop);
+            if (query instanceof GraphQuery) {
+                // we have a graph query, convert inner queries to multi phrase prefix queries
+                List<Query> oldQueries = ((GraphQuery) query).getQueries();
+                Query[] queries = new Query[oldQueries.size()];
+                for (int i = 0; i < queries.length; i++) {
+                    queries[i] = toMultiPhrasePrefix(oldQueries.get(i), phraseSlop, maxExpansions);
+                }
+
+                return new GraphQuery(queries);
+            }
+
+            return toMultiPhrasePrefix(query, phraseSlop, maxExpansions);
+        }
+
+        private Query toMultiPhrasePrefix(final Query query, int phraseSlop, int maxExpansions) {
             float boost = 1;
             Query innerQuery = query;
             while (innerQuery instanceof BoostQuery) {
@@ -479,42 +374,38 @@ public class MatchQuery {
             Query booleanQuery = createBooleanQuery(field, queryText, lowFreqOccur);
             if (booleanQuery != null && booleanQuery instanceof BooleanQuery) {
                 BooleanQuery bq = (BooleanQuery) booleanQuery;
-                ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency, (
-                    (BooleanQuery) booleanQuery).isCoordDisabled(), fieldType);
-                for (BooleanClause clause : bq.clauses()) {
-                    if (!(clause.getQuery() instanceof TermQuery)) {
-                        return booleanQuery;
+                return boolToExtendedCommonTermsQuery(bq, highFreqOccur, lowFreqOccur, maxTermFrequency, fieldType);
+            } else if (booleanQuery != null && booleanQuery instanceof GraphQuery && ((GraphQuery) booleanQuery).hasBoolean()) {
+                // we have a graph query that has at least one boolean sub-query
+                // re-build and use extended common terms
+                List<Query> oldQueries = ((GraphQuery) booleanQuery).getQueries();
+                Query[] queries = new Query[oldQueries.size()];
+                for (int i = 0; i < queries.length; i++) {
+                    Query oldQuery = oldQueries.get(i);
+                    if (oldQuery instanceof BooleanQuery) {
+                        queries[i] = boolToExtendedCommonTermsQuery((BooleanQuery) oldQuery, highFreqOccur, lowFreqOccur, maxTermFrequency, fieldType);
+                    } else {
+                        queries[i] = oldQuery;
                     }
-                    query.add(((TermQuery) clause.getQuery()).getTerm());
                 }
-                return query;
-            }
-            return booleanQuery;
 
+                return new GraphQuery(queries);
+            }
+
+            return booleanQuery;
         }
 
-        /**
-         * Creates a query from a graph token stream by extracting all the finite strings from the graph and using them to create the query.
-         */
-        protected Query analyzeGraph(TokenStream source, BooleanClause.Occur operator, String field, boolean quoted, int phraseSlop)
-            throws IOException {
-            source.reset();
-            GraphTokenStreamFiniteStrings graphTokenStreams = new GraphTokenStreamFiniteStrings();
-            List<TokenStream> tokenStreams = graphTokenStreams.getTokenStreams(source);
-
-            if (tokenStreams.isEmpty()) {
-                return null;
-            }
-
-            List<Query> queries = new ArrayList<>(tokenStreams.size());
-            for (TokenStream ts : tokenStreams) {
-                Query query = createFieldQuery(ts, operator, field, quoted, phraseSlop);
-                if (query != null) {
-                    queries.add(query);
+        private Query boolToExtendedCommonTermsQuery(BooleanQuery bq, Occur highFreqOccur, Occur lowFreqOccur, float
+            maxTermFrequency, MappedFieldType fieldType) {
+            ExtendedCommonTermsQuery query = new ExtendedCommonTermsQuery(highFreqOccur, lowFreqOccur, maxTermFrequency,
+                bq.isCoordDisabled(), fieldType);
+            for (BooleanClause clause : bq.clauses()) {
+                if (!(clause.getQuery() instanceof TermQuery)) {
+                    return bq;
                 }
+                query.add(((TermQuery) clause.getQuery()).getTerm());
             }
-
-            return new GraphQuery(queries.toArray(new Query[0]));
+            return query;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
index d08d4aaddc1..15e8561c8fd 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
@@ -61,12 +61,7 @@ public class MultiMatchQuery extends MatchQuery {
 
     private Query parseAndApply(Type type, String fieldName, Object value, String minimumShouldMatch, Float boostValue) throws IOException {
         Query query = parse(type, fieldName, value);
-        // If the coordination factor is disabled on a boolean query we don't apply the minimum should match.
-        // This is done to make sure that the minimum_should_match doesn't get applied when there is only one word
-        // and multiple variations of the same word in the query (synonyms for instance).
-        if (query instanceof BooleanQuery && !((BooleanQuery) query).isCoordDisabled()) {
-            query = Queries.applyMinimumShouldMatch((BooleanQuery) query, minimumShouldMatch);
-        }
+        query = Queries.maybeApplyMinimumShouldMatch(query, minimumShouldMatch);
         if (query != null && boostValue != null && boostValue != AbstractQueryBuilder.DEFAULT_BOOST) {
             query = new BoostQuery(query, boostValue);
         }
@@ -170,7 +165,7 @@ public class MultiMatchQuery extends MatchQuery {
     final class CrossFieldsQueryBuilder extends QueryBuilder {
         private FieldAndFieldType[] blendedFields;
 
-        public CrossFieldsQueryBuilder(float tieBreaker) {
+        CrossFieldsQueryBuilder(float tieBreaker) {
             super(false, tieBreaker);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
index 8e877298313..6e13573794d 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -20,20 +20,21 @@ package org.elasticsearch.index.seqno;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -41,8 +42,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.io.UnsupportedEncodingException;
 
 public class GlobalCheckpointSyncAction extends TransportReplicationAction<GlobalCheckpointSyncAction.PrimaryRequest,
     GlobalCheckpointSyncAction.ReplicaRequest, ReplicationResponse> {
@@ -65,6 +64,17 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<Globa
         return new ReplicationResponse();
     }
 
+    @Override
+    protected void sendReplicaRequest(ConcreteShardRequest<ReplicaRequest> concreteShardRequest, DiscoveryNode node,
+                                      ActionListener<ReplicationOperation.ReplicaResponse> listener) {
+        if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            super.sendReplicaRequest(concreteShardRequest, node, listener);
+        } else {
+            listener.onResponse(
+                new ReplicaResponse(concreteShardRequest.getTargetAllocationID(), SequenceNumbersService.UNASSIGNED_SEQ_NO));
+        }
+    }
+
     @Override
     protected PrimaryResult shardOperationOnPrimary(PrimaryRequest request, IndexShard indexShard) throws Exception {
         long checkpoint = indexShard.getGlobalCheckpoint();
@@ -105,6 +115,11 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<Globa
         public PrimaryRequest(ShardId shardId) {
             super(shardId);
         }
+
+        @Override
+        public String toString() {
+            return "GlobalCkpSyncPrimary{" + shardId + "}";
+        }
     }
 
     public static final class ReplicaRequest extends ReplicationRequest<GlobalCheckpointSyncAction.ReplicaRequest> {
@@ -134,6 +149,14 @@ public class GlobalCheckpointSyncAction extends TransportReplicationAction<Globa
         public long getCheckpoint() {
             return checkpoint;
         }
+
+        @Override
+        public String toString() {
+            return "GlobalCkpSyncReplica{" +
+                "checkpoint=" + checkpoint +
+                ", shardId=" + shardId +
+                '}';
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointService.java b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointTracker.java
similarity index 50%
rename from core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointService.java
rename to core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointTracker.java
index e8aa0cdeb89..fd9b2f4687f 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointService.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/GlobalCheckpointTracker.java
@@ -16,110 +16,112 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.seqno;
 
 import com.carrotsearch.hppc.ObjectLongHashMap;
 import com.carrotsearch.hppc.ObjectLongMap;
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Set;
 
 import static org.elasticsearch.index.seqno.SequenceNumbersService.UNASSIGNED_SEQ_NO;
 
 /**
- * A shard component that is responsible of tracking the global checkpoint. The global checkpoint
- * is the highest seq_no for which all lower (or equal) seq_no have been processed on all shards that
- * are currently active. Since shards count as "active" when the master starts them, and before this primary shard
- * has been notified of this fact, we also include shards that have completed recovery. These shards have received
- * all old operations via the recovery mechanism and are kept up to date by the various replications actions. The set
- * of shards that are taken into account for the global checkpoint calculation are called the "in sync" shards.
- *
+ * This class is responsible of tracking the global checkpoint. The global checkpoint is the highest sequence number for which all lower (or
+ * equal) sequence number have been processed on all shards that are currently active. Since shards count as "active" when the master starts
+ * them, and before this primary shard has been notified of this fact, we also include shards that have completed recovery. These shards
+ * have received all old operations via the recovery mechanism and are kept up to date by the various replications actions. The set of
+ * shards that are taken into account for the global checkpoint calculation are called the "in-sync shards".
  * <p>
- * The global checkpoint is maintained by the primary shard and is replicated to all the replicas
- * (via {@link GlobalCheckpointSyncAction}).
+ * The global checkpoint is maintained by the primary shard and is replicated to all the replicas (via {@link GlobalCheckpointSyncAction}).
  */
-public class GlobalCheckpointService extends AbstractIndexShardComponent {
+public class GlobalCheckpointTracker extends AbstractIndexShardComponent {
 
-
-    /**
-     * This map holds the last known local checkpoint for every active shard and initializing shard copies that has been brought up
-     * to speed through recovery. These shards are treated as valid copies and participate in determining the global
-     * checkpoint.
-     * <p>
-     * Keyed by allocation ids.
+    /*
+     * This map holds the last known local checkpoint for every active shard and initializing shard copies that has been brought up to speed
+     * through recovery. These shards are treated as valid copies and participate in determining the global checkpoint. This map is keyed by
+     * allocation IDs. All accesses to this set are guarded by a lock on this.
      */
-    private final ObjectLongMap<String> inSyncLocalCheckpoints; // keyed by allocation ids
+    private final ObjectLongMap<String> inSyncLocalCheckpoints;
 
-    /**
-     * This set holds the last set of known valid allocation ids as received by the master. This is important to make sure
-     * shard that are failed or relocated are cleaned up from {@link #inSyncLocalCheckpoints} and do not hold the global
-     * checkpoint back
+    /*
+     * This set holds the last set of known valid allocation ids as received by the master. This is important to make sure shard that are
+     * failed or relocated are cleaned up from {@link #inSyncLocalCheckpoints} and do not hold the global checkpoint back. All accesses to
+     * this set are guarded by a lock on this.
      */
     private final Set<String> assignedAllocationIds;
 
+    /*
+     * The current global checkpoint for this shard. Note that this field is guarded by a lock on this and thus this field does not need to
+     * be volatile.
+     */
     private long globalCheckpoint;
 
     /**
-     * Initialize the global checkpoint service. The {@code globalCheckpoint}
-     * should be set to the last known global checkpoint for this shard, or
+     * Initialize the global checkpoint service. The specified global checkpoint should be set to the last known global checkpoint, or
      * {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}.
      *
-     * @param shardId          the shard this service is providing tracking
-     *                         local checkpoints for
+     * @param shardId          the shard ID
      * @param indexSettings    the index settings
-     * @param globalCheckpoint the last known global checkpoint for this shard,
-     *                         or
-     *                         {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}
+     * @param globalCheckpoint the last known global checkpoint for this shard, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}
      */
-    GlobalCheckpointService(final ShardId shardId, final IndexSettings indexSettings, final long globalCheckpoint) {
+    GlobalCheckpointTracker(final ShardId shardId, final IndexSettings indexSettings, final long globalCheckpoint) {
         super(shardId, indexSettings);
-        assert globalCheckpoint >= UNASSIGNED_SEQ_NO : "illegal initial global checkpoint:" + globalCheckpoint;
+        assert globalCheckpoint >= UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint;
         inSyncLocalCheckpoints = new ObjectLongHashMap<>(1 + indexSettings.getNumberOfReplicas());
         assignedAllocationIds = new HashSet<>(1 + indexSettings.getNumberOfReplicas());
         this.globalCheckpoint = globalCheckpoint;
     }
 
     /**
-     * Notifies the service of a local checkpoint. If the checkpoint is lower than the currently known one,
-     * this is a noop. Last, if the allocation id is not in sync, it is ignored. This to prevent late
-     * arrivals from shards that are removed to be re-added.
+     * Notifies the service to update the local checkpoint for the shard with the provided allocation ID. If the checkpoint is lower than
+     * the currently known one, this is a no-op. If the allocation ID is not in sync, it is ignored. This is to prevent late arrivals from
+     * shards that are removed to be re-added.
+     *
+     * @param allocationId the allocation ID of the shard to update the local checkpoint for
+     * @param checkpoint   the local checkpoint for the shard
      */
-    public synchronized void updateLocalCheckpoint(String allocationId, long localCheckpoint) {
+    public synchronized void updateLocalCheckpoint(final String allocationId, final long checkpoint) {
         final int indexOfKey = inSyncLocalCheckpoints.indexOf(allocationId);
         if (indexOfKey >= 0) {
             final long current = inSyncLocalCheckpoints.indexGet(indexOfKey);
-
-            if (current < localCheckpoint) {
-                inSyncLocalCheckpoints.indexReplace(indexOfKey, localCheckpoint);
+            if (current < checkpoint) {
+                inSyncLocalCheckpoints.indexReplace(indexOfKey, checkpoint);
                 if (logger.isTraceEnabled()) {
-                    logger.trace("updated local checkpoint of [{}] to [{}] (was [{}])", allocationId, localCheckpoint, current);
+                    logger.trace("updated local checkpoint of [{}] to [{}] (was [{}])", allocationId, checkpoint, current);
                 }
             } else {
-                logger.trace("skipping update of local checkpoint [{}], current checkpoint is higher " +
-                        "(current [{}], incoming [{}], type [{}])",
-                    allocationId, current, localCheckpoint, allocationId);
+                logger.trace(
+                    "skipping update of local checkpoint [{}], current checkpoint is higher (current [{}], incoming [{}], type [{}])",
+                    allocationId,
+                    current,
+                    checkpoint,
+                    allocationId);
             }
         } else {
-            logger.trace("[{}] isn't marked as in sync. ignoring local checkpoint of [{}].", allocationId, localCheckpoint);
+            logger.trace("[{}] isn't marked as in sync. ignoring local checkpoint of [{}].", allocationId, checkpoint);
         }
     }
 
     /**
-     * Scans through the currently known local checkpoints and updates the global checkpoint accordingly.
+     * Scans through the currently known local checkpoint and updates the global checkpoint accordingly.
      *
-     * @return true if the checkpoint has been updated or if it can not be updated since one of the local checkpoints
-     * of one of the active allocations is not known.
+     * @return {@code true} if the checkpoint has been updated or if it can not be updated since one of the local checkpoints of one of the
+     * active allocations is not known.
      */
     synchronized boolean updateCheckpointOnPrimary() {
         long minCheckpoint = Long.MAX_VALUE;
         if (inSyncLocalCheckpoints.isEmpty()) {
             return false;
         }
-        for (ObjectLongCursor<String> cp : inSyncLocalCheckpoints) {
+        for (final ObjectLongCursor<String> cp : inSyncLocalCheckpoints) {
             if (cp.value == UNASSIGNED_SEQ_NO) {
                 logger.trace("unknown local checkpoint for active allocationId [{}], requesting a sync", cp.key);
                 return true;
@@ -127,8 +129,9 @@ public class GlobalCheckpointService extends AbstractIndexShardComponent {
             minCheckpoint = Math.min(cp.value, minCheckpoint);
         }
         if (minCheckpoint < globalCheckpoint) {
-            throw new IllegalStateException(shardId + " new global checkpoint [" + minCheckpoint
-                + "] is lower than previous one [" + globalCheckpoint + "]");
+            final String message =
+                String.format(Locale.ROOT, "new global checkpoint [%d] is lower than previous one [%d]", minCheckpoint, globalCheckpoint);
+            throw new IllegalStateException(message);
         }
         if (globalCheckpoint != minCheckpoint) {
             logger.trace("global checkpoint updated to [{}]", minCheckpoint);
@@ -139,36 +142,39 @@ public class GlobalCheckpointService extends AbstractIndexShardComponent {
     }
 
     /**
-     * gets the current global checkpoint. See java docs for {@link GlobalCheckpointService} for more details
+     * Returns the global checkpoint for the shard.
+     *
+     * @return the global checkpoint
      */
     public synchronized long getCheckpoint() {
         return globalCheckpoint;
     }
 
     /**
-     * updates the global checkpoint on a replica shard (after it has been updated by the primary).
+     * Updates the global checkpoint on a replica shard after it has been updated by the primary.
+     *
+     * @param checkpoint the global checkpoint
      */
-    synchronized void updateCheckpointOnReplica(long globalCheckpoint) {
+    synchronized void updateCheckpointOnReplica(final long checkpoint) {
         /*
          * The global checkpoint here is a local knowledge which is updated under the mandate of the primary. It can happen that the primary
          * information is lagging compared to a replica (e.g., if a replica is promoted to primary but has stale info relative to other
          * replica shards). In these cases, the local knowledge of the global checkpoint could be higher than sync from the lagging primary.
          */
-        if (this.globalCheckpoint <= globalCheckpoint) {
-            this.globalCheckpoint = globalCheckpoint;
-            logger.trace("global checkpoint updated from primary to [{}]", globalCheckpoint);
+        if (this.globalCheckpoint <= checkpoint) {
+            this.globalCheckpoint = checkpoint;
+            logger.trace("global checkpoint updated from primary to [{}]", checkpoint);
         }
     }
 
     /**
-     * Notifies the service of the current allocation ids in the cluster state. This method trims any shards that
-     * have been removed.
+     * Notifies the service of the current allocation ids in the cluster state. This method trims any shards that have been removed.
      *
-     * @param activeAllocationIds       the allocation ids of the currently active shard copies
-     * @param initializingAllocationIds the allocation ids of the currently initializing shard copies
+     * @param activeAllocationIds       the allocation IDs of the currently active shard copies
+     * @param initializingAllocationIds the allocation IDs of the currently initializing shard copies
      */
-    public synchronized void updateAllocationIdsFromMaster(Set<String> activeAllocationIds,
-                                                           Set<String> initializingAllocationIds) {
+    public synchronized void updateAllocationIdsFromMaster(final Set<String> activeAllocationIds,
+                                                           final Set<String> initializingAllocationIds) {
         assignedAllocationIds.removeIf(
             aId -> activeAllocationIds.contains(aId) == false && initializingAllocationIds.contains(aId) == false);
         assignedAllocationIds.addAll(activeAllocationIds);
@@ -182,22 +188,28 @@ public class GlobalCheckpointService extends AbstractIndexShardComponent {
     }
 
     /**
-     * marks the allocationId as "in sync" with the primary shard. This should be called at the end of recovery
-     * where the primary knows all operation below the global checkpoint have been completed on this shard.
+     * Marks the shard with the provided allocation ID as in-sync with the primary shard. This should be called at the end of recovery where
+     * the primary knows all operations below the global checkpoint have been completed on this shard.
      *
-     * @param allocationId    allocationId of the recovering shard
+     * @param allocationId the allocation ID of the shard to mark as in-sync
      */
-    public synchronized void markAllocationIdAsInSync(String allocationId) {
+    public synchronized void markAllocationIdAsInSync(final String allocationId) {
         if (assignedAllocationIds.contains(allocationId) == false) {
-            // master have change it's mind and removed this allocation, ignore.
+            // master has removed this allocation, ignore
             return;
         }
         logger.trace("marked [{}] as in sync", allocationId);
         inSyncLocalCheckpoints.put(allocationId, UNASSIGNED_SEQ_NO);
     }
 
-    // for testing
-    synchronized long getLocalCheckpointForAllocation(String allocationId) {
+    /**
+     * Returns the local checkpoint for the shard with the specified allocation ID, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO} if
+     * the shard is not in-sync.
+     *
+     * @param allocationId the allocation ID of the shard to obtain the local checkpoint for
+     * @return the local checkpoint, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}
+     */
+    synchronized long getLocalCheckpointForAllocationId(final String allocationId) {
         if (inSyncLocalCheckpoints.containsKey(allocationId)) {
             return inSyncLocalCheckpoints.get(allocationId);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointService.java b/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointService.java
deleted file mode 100644
index a03daf2ff23..00000000000
--- a/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointService.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.seqno;
-
-import org.apache.lucene.util.FixedBitSet;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.shard.AbstractIndexShardComponent;
-import org.elasticsearch.index.shard.ShardId;
-
-import java.util.LinkedList;
-
-/**
- * This class generates sequences numbers and keeps track of the so called local checkpoint - the highest number for which
- * all previous seqNo have been processed (including)
- */
-public class LocalCheckpointService extends AbstractIndexShardComponent {
-
-    /**
-     * we keep a bit for each seq No that is still pending. to optimize allocation, we do so in multiple arrays
-     * allocating them on demand and cleaning up while completed. This setting controls the size of the arrays
-     */
-    public static Setting<Integer> SETTINGS_BIT_ARRAYS_SIZE = Setting.intSetting("index.seq_no.checkpoint.bit_arrays_size", 1024,
-        4, Setting.Property.IndexScope);
-
-    /**
-     * an ordered list of bit arrays representing pending seq nos. The list is "anchored" in {@link #firstProcessedSeqNo}
-     * which marks the seqNo the fist bit in the first array corresponds to.
-     */
-    final LinkedList<FixedBitSet> processedSeqNo;
-    private final int bitArraysSize;
-    long firstProcessedSeqNo;
-
-    /** the current local checkpoint, i.e., all seqNo lower (&lt;=) than this number have been completed */
-    volatile long checkpoint;
-
-    /** the next available seqNo - used for seqNo generation */
-    private volatile long nextSeqNo;
-
-    /**
-     * Initialize the local checkpoint service. The {@code maxSeqNo} should be
-     * set to the last sequence number assigned by this shard, or
-     * {@link SequenceNumbersService#NO_OPS_PERFORMED} and
-     * {@code localCheckpoint} should be set to the last known local checkpoint
-     * for this shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}.
-     *
-     * @param shardId         the shard this service is providing tracking
-     *                        local checkpoints for
-     * @param indexSettings   the index settings
-     * @param maxSeqNo        the last sequence number assigned by this shard, or
-     *                        {@link SequenceNumbersService#NO_OPS_PERFORMED}
-     * @param localCheckpoint the last known local checkpoint for this shard, or
-     *                        {@link SequenceNumbersService#NO_OPS_PERFORMED}
-     */
-    LocalCheckpointService(final ShardId shardId, final IndexSettings indexSettings, final long maxSeqNo, final long localCheckpoint) {
-        super(shardId, indexSettings);
-        if (localCheckpoint < 0 && localCheckpoint != SequenceNumbersService.NO_OPS_PERFORMED) {
-            throw new IllegalArgumentException(
-                "local checkpoint must be non-negative or [" + SequenceNumbersService.NO_OPS_PERFORMED + "] "
-                    + "but was [" +  localCheckpoint + "]");
-        }
-        if (maxSeqNo < 0 && maxSeqNo != SequenceNumbersService.NO_OPS_PERFORMED) {
-            throw new IllegalArgumentException(
-                "max seq. no. must be non-negative or [" + SequenceNumbersService.NO_OPS_PERFORMED + "] but was [" + maxSeqNo + "]");
-        }
-        bitArraysSize = SETTINGS_BIT_ARRAYS_SIZE.get(indexSettings.getSettings());
-        processedSeqNo = new LinkedList<>();
-        firstProcessedSeqNo = localCheckpoint == SequenceNumbersService.NO_OPS_PERFORMED ? 0 : localCheckpoint + 1;
-        this.nextSeqNo = maxSeqNo == SequenceNumbersService.NO_OPS_PERFORMED ? 0 : maxSeqNo + 1;
-        this.checkpoint = localCheckpoint;
-    }
-
-    /**
-     * issue the next sequence number
-     **/
-    synchronized long generateSeqNo() {
-        return nextSeqNo++;
-    }
-
-    /**
-     * marks the processing of the given seqNo have been completed
-     **/
-    synchronized void markSeqNoAsCompleted(long seqNo) {
-        // make sure we track highest seen seqNo
-        if (seqNo >= nextSeqNo) {
-            nextSeqNo = seqNo + 1;
-        }
-        if (seqNo <= checkpoint) {
-            // this is possible during recovery where we might replay an op that was also replicated
-            return;
-        }
-        FixedBitSet bitSet = getBitSetForSeqNo(seqNo);
-        int offset = seqNoToBitSetOffset(seqNo);
-        bitSet.set(offset);
-        if (seqNo == checkpoint + 1) {
-            updateCheckpoint();
-        }
-    }
-
-    /** gets the current check point */
-    public long getCheckpoint() {
-        return checkpoint;
-    }
-
-    /** gets the maximum seqno seen so far */
-    long getMaxSeqNo() {
-        return nextSeqNo - 1;
-    }
-
-    /**
-     * moves the checkpoint to the last consecutively processed seqNo
-     * Note: this method assumes that the seqNo following the current checkpoint is processed.
-     */
-    private void updateCheckpoint() {
-        assert Thread.holdsLock(this);
-        assert checkpoint < firstProcessedSeqNo + bitArraysSize - 1 :
-            "checkpoint should be below the end of the first bit set (o.w. current bit set is completed and shouldn't be there)";
-        assert getBitSetForSeqNo(checkpoint + 1) == processedSeqNo.getFirst() :
-            "checkpoint + 1 doesn't point to the first bit set (o.w. current bit set is completed and shouldn't be there)";
-        assert getBitSetForSeqNo(checkpoint + 1).get(seqNoToBitSetOffset(checkpoint + 1)) :
-            "updateCheckpoint is called but the bit following the checkpoint is not set";
-        // keep it simple for now, get the checkpoint one by one. in the future we can optimize and read words
-        FixedBitSet current = processedSeqNo.getFirst();
-        do {
-            checkpoint++;
-            // the checkpoint always falls in the first bit set or just before. If it falls
-            // on the last bit of the current bit set, we can clean it.
-            if (checkpoint == firstProcessedSeqNo + bitArraysSize - 1) {
-                processedSeqNo.removeFirst();
-                firstProcessedSeqNo += bitArraysSize;
-                assert checkpoint - firstProcessedSeqNo < bitArraysSize;
-                current = processedSeqNo.peekFirst();
-            }
-        } while (current != null && current.get(seqNoToBitSetOffset(checkpoint + 1)));
-    }
-
-    /**
-     * gets the bit array for the given seqNo, allocating new ones if needed.
-     */
-    private FixedBitSet getBitSetForSeqNo(long seqNo) {
-        assert Thread.holdsLock(this);
-        assert seqNo >= firstProcessedSeqNo : "seqNo: " + seqNo + " firstProcessedSeqNo: " + firstProcessedSeqNo;
-        final long bitSetOffset = (seqNo - firstProcessedSeqNo) / bitArraysSize;
-        if (bitSetOffset > Integer.MAX_VALUE) {
-            throw new IndexOutOfBoundsException("seqNo too high. got [" + seqNo + "], firstProcessedSeqNo [" + firstProcessedSeqNo + "]");
-        }
-        while (bitSetOffset >= processedSeqNo.size()) {
-            processedSeqNo.add(new FixedBitSet(bitArraysSize));
-        }
-        return processedSeqNo.get((int)bitSetOffset);
-    }
-
-    /** maps the given seqNo to a position in the bit set returned by {@link #getBitSetForSeqNo} */
-    private int seqNoToBitSetOffset(long seqNo) {
-        assert Thread.holdsLock(this);
-        assert seqNo >= firstProcessedSeqNo;
-        return ((int) (seqNo - firstProcessedSeqNo)) % bitArraysSize;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java b/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
new file mode 100644
index 00000000000..a1ad5dfa6ff
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.seqno;
+
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.index.IndexSettings;
+
+import java.util.LinkedList;
+
+/**
+ * This class generates sequences numbers and keeps track of the so-called "local checkpoint" which is the highest number for which all
+ * previous sequence numbers have been processed (inclusive).
+ */
+public class LocalCheckpointTracker {
+
+    /**
+     * We keep a bit for each sequence number that is still pending. To optimize allocation, we do so in multiple arrays allocating them on
+     * demand and cleaning up while completed. This setting controls the size of the arrays.
+     */
+    public static Setting<Integer> SETTINGS_BIT_ARRAYS_SIZE =
+        Setting.intSetting("index.seq_no.checkpoint.bit_arrays_size", 1024, 4, Setting.Property.IndexScope);
+
+    /**
+     * An ordered list of bit arrays representing pending sequence numbers. The list is "anchored" in {@link #firstProcessedSeqNo} which
+     * marks the sequence number the fist bit in the first array corresponds to.
+     */
+    final LinkedList<FixedBitSet> processedSeqNo = new LinkedList<>();
+
+    /**
+     * The size of each bit set representing processed sequence numbers.
+     */
+    private final int bitArraysSize;
+
+    /**
+     * The sequence number that the first bit in the first array corresponds to.
+     */
+    long firstProcessedSeqNo;
+
+    /**
+     * The current local checkpoint, i.e., all sequence numbers no more than this number have been completed.
+     */
+    volatile long checkpoint;
+
+    /**
+     * The next available sequence number.
+     */
+    private volatile long nextSeqNo;
+
+    /**
+     * Initialize the local checkpoint service. The {@code maxSeqNo} should be set to the last sequence number assigned, or
+     * {@link SequenceNumbersService#NO_OPS_PERFORMED} and {@code localCheckpoint} should be set to the last known local checkpoint,
+     * or {@link SequenceNumbersService#NO_OPS_PERFORMED}.
+     *
+     * @param indexSettings   the index settings
+     * @param maxSeqNo        the last sequence number assigned, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
+     * @param localCheckpoint the last known local checkpoint, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
+     */
+    public LocalCheckpointTracker(final IndexSettings indexSettings, final long maxSeqNo, final long localCheckpoint) {
+        if (localCheckpoint < 0 && localCheckpoint != SequenceNumbersService.NO_OPS_PERFORMED) {
+            throw new IllegalArgumentException(
+                "local checkpoint must be non-negative or [" + SequenceNumbersService.NO_OPS_PERFORMED + "] "
+                    + "but was [" + localCheckpoint + "]");
+        }
+        if (maxSeqNo < 0 && maxSeqNo != SequenceNumbersService.NO_OPS_PERFORMED) {
+            throw new IllegalArgumentException(
+                "max seq. no. must be non-negative or [" + SequenceNumbersService.NO_OPS_PERFORMED + "] but was [" + maxSeqNo + "]");
+        }
+        bitArraysSize = SETTINGS_BIT_ARRAYS_SIZE.get(indexSettings.getSettings());
+        firstProcessedSeqNo = localCheckpoint == SequenceNumbersService.NO_OPS_PERFORMED ? 0 : localCheckpoint + 1;
+        nextSeqNo = maxSeqNo == SequenceNumbersService.NO_OPS_PERFORMED ? 0 : maxSeqNo + 1;
+        checkpoint = localCheckpoint;
+    }
+
+    /**
+     * Issue the next sequence number.
+     *
+     * @return the next assigned sequence number
+     */
+    synchronized long generateSeqNo() {
+        return nextSeqNo++;
+    }
+
+    /**
+     * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible.
+     *
+     * @param seqNo the sequence number to mark as completed
+     */
+    public synchronized void markSeqNoAsCompleted(final long seqNo) {
+        // make sure we track highest seen sequence number
+        if (seqNo >= nextSeqNo) {
+            nextSeqNo = seqNo + 1;
+        }
+        if (seqNo <= checkpoint) {
+            // this is possible during recovery where we might replay an operation that was also replicated
+            return;
+        }
+        final FixedBitSet bitSet = getBitSetForSeqNo(seqNo);
+        final int offset = seqNoToBitSetOffset(seqNo);
+        bitSet.set(offset);
+        if (seqNo == checkpoint + 1) {
+            updateCheckpoint();
+        }
+    }
+
+    /**
+     * The current checkpoint which can be advanced by {@link #markSeqNoAsCompleted(long)}.
+     *
+     * @return the current checkpoint
+     */
+    public long getCheckpoint() {
+        return checkpoint;
+    }
+
+    /**
+     * The maximum sequence number issued so far.
+     *
+     * @return the maximum sequence number
+     */
+    long getMaxSeqNo() {
+        return nextSeqNo - 1;
+    }
+
+
+    /**
+     * constructs a {@link SeqNoStats} object, using local state and the supplied global checkpoint
+     *
+     * @implNote this is needed to make sure the local checkpoint and max seq no are consistent
+     */
+    synchronized SeqNoStats getStats(final long globalCheckpoint) {
+        return new SeqNoStats(getMaxSeqNo(), getCheckpoint(), globalCheckpoint);
+    }
+
+    /**
+     * Waits for all operations up to the provided sequence number to complete.
+     *
+     * @param seqNo the sequence number that the checkpoint must advance to before this method returns
+     * @throws InterruptedException if the thread was interrupted while blocking on the condition
+     */
+    @SuppressForbidden(reason = "Object#wait")
+    synchronized void waitForOpsToComplete(final long seqNo) throws InterruptedException {
+        while (checkpoint < seqNo) {
+            // notified by updateCheckpoint
+            this.wait();
+        }
+    }
+
+    /**
+     * Moves the checkpoint to the last consecutively processed sequence number. This method assumes that the sequence number following the
+     * current checkpoint is processed.
+     */
+    @SuppressForbidden(reason = "Object#notifyAll")
+    private void updateCheckpoint() {
+        assert Thread.holdsLock(this);
+        assert checkpoint < firstProcessedSeqNo + bitArraysSize - 1 :
+            "checkpoint should be below the end of the first bit set (o.w. current bit set is completed and shouldn't be there)";
+        assert getBitSetForSeqNo(checkpoint + 1) == processedSeqNo.getFirst() :
+            "checkpoint + 1 doesn't point to the first bit set (o.w. current bit set is completed and shouldn't be there)";
+        assert getBitSetForSeqNo(checkpoint + 1).get(seqNoToBitSetOffset(checkpoint + 1)) :
+            "updateCheckpoint is called but the bit following the checkpoint is not set";
+        try {
+            // keep it simple for now, get the checkpoint one by one; in the future we can optimize and read words
+            FixedBitSet current = processedSeqNo.getFirst();
+            do {
+                checkpoint++;
+                // the checkpoint always falls in the first bit set or just before. If it falls
+                // on the last bit of the current bit set, we can clean it.
+                if (checkpoint == firstProcessedSeqNo + bitArraysSize - 1) {
+                    processedSeqNo.removeFirst();
+                    firstProcessedSeqNo += bitArraysSize;
+                    assert checkpoint - firstProcessedSeqNo < bitArraysSize;
+                    current = processedSeqNo.peekFirst();
+                }
+            } while (current != null && current.get(seqNoToBitSetOffset(checkpoint + 1)));
+        } finally {
+            // notifies waiters in waitForOpsToComplete
+            this.notifyAll();
+        }
+    }
+
+    /**
+     * Return the bit array for the provided sequence number, possibly allocating a new array if needed.
+     *
+     * @param seqNo the sequence number to obtain the bit array for
+     * @return the bit array corresponding to the provided sequence number
+     */
+    private FixedBitSet getBitSetForSeqNo(final long seqNo) {
+        assert Thread.holdsLock(this);
+        assert seqNo >= firstProcessedSeqNo : "seqNo: " + seqNo + " firstProcessedSeqNo: " + firstProcessedSeqNo;
+        final long bitSetOffset = (seqNo - firstProcessedSeqNo) / bitArraysSize;
+        if (bitSetOffset > Integer.MAX_VALUE) {
+            throw new IndexOutOfBoundsException(
+                "sequence number too high; got [" + seqNo + "], firstProcessedSeqNo [" + firstProcessedSeqNo + "]");
+        }
+        while (bitSetOffset >= processedSeqNo.size()) {
+            processedSeqNo.add(new FixedBitSet(bitArraysSize));
+        }
+        return processedSeqNo.get((int) bitSetOffset);
+    }
+
+    /**
+     * Obtain the position in the bit array corresponding to the provided sequence number. The bit array corresponding to the sequence
+     * number can be obtained via {@link #getBitSetForSeqNo(long)}.
+     *
+     * @param seqNo the sequence number to obtain the position for
+     * @return the position in the bit array corresponding to the provided sequence number
+     */
+    private int seqNoToBitSetOffset(final long seqNo) {
+        assert Thread.holdsLock(this);
+        assert seqNo >= firstProcessedSeqNo;
+        return ((int) (seqNo - firstProcessedSeqNo)) % bitArraysSize;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java b/core/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
index 157fea98d91..363e7f498d1 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/SeqNoStats.java
@@ -30,17 +30,19 @@ import java.io.IOException;
 public class SeqNoStats implements ToXContent, Writeable {
 
     private static final String SEQ_NO = "seq_no";
-    private static final String MAX_SEQ_NO = "max";
+    private static final String MAX_SEQ_NO = "max_seq_no";
     private static final String LOCAL_CHECKPOINT = "local_checkpoint";
     private static final String GLOBAL_CHECKPOINT = "global_checkpoint";
 
-    public static final SeqNoStats PROTO = new SeqNoStats(0, 0, 0);
-
     private final long maxSeqNo;
     private final long localCheckpoint;
     private final long globalCheckpoint;
 
     public SeqNoStats(long maxSeqNo, long localCheckpoint, long globalCheckpoint) {
+        assert localCheckpoint <= maxSeqNo:
+            "local checkpoint [" + localCheckpoint + "] is above maximum seq no [" + maxSeqNo + "]";
+        // note that the the global checkpoint can be higher from both maxSeqNo and localCheckpoint
+        // as we use this stats object to describe lucene commits as well as live statistic.
         this.maxSeqNo = maxSeqNo;
         this.localCheckpoint = localCheckpoint;
         this.globalCheckpoint = globalCheckpoint;
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java b/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java
new file mode 100644
index 00000000000..c3950e1012a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbers.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.seqno;
+
+import java.util.Map;
+
+/**
+ * A utility class for handling sequence numbers.
+ */
+public class SequenceNumbers {
+
+    public static final String LOCAL_CHECKPOINT_KEY = "local_checkpoint";
+    public static final String MAX_SEQ_NO = "max_seq_no";
+
+    /**
+     * Reads the sequence number stats from the commit data (maximum sequence number and local checkpoint) and uses the specified global
+     * checkpoint.
+     *
+     * @param globalCheckpoint the global checkpoint to use
+     * @param commitData       the commit data
+     * @return the sequence number stats
+     */
+    public static SeqNoStats loadSeqNoStatsFromLuceneCommit(
+        final long globalCheckpoint,
+        final Iterable<Map.Entry<String, String>> commitData) {
+        long maxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
+        long localCheckpoint = SequenceNumbersService.NO_OPS_PERFORMED;
+
+        for (final Map.Entry<String, String> entry : commitData) {
+            final String key = entry.getKey();
+            if (key.equals(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) {
+                assert localCheckpoint == SequenceNumbersService.NO_OPS_PERFORMED : localCheckpoint;
+                localCheckpoint = Long.parseLong(entry.getValue());
+            } else if (key.equals(SequenceNumbers.MAX_SEQ_NO)) {
+                assert maxSeqNo == SequenceNumbersService.NO_OPS_PERFORMED : maxSeqNo;
+                maxSeqNo = Long.parseLong(entry.getValue());
+            }
+        }
+
+        return new SeqNoStats(maxSeqNo, localCheckpoint, globalCheckpoint);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java b/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java
index 09ed72fd92a..3cdcde13c0b 100644
--- a/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java
+++ b/core/src/main/java/org/elasticsearch/index/seqno/SequenceNumbersService.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.seqno;
 
 import org.elasticsearch.index.IndexSettings;
@@ -25,10 +26,13 @@ import org.elasticsearch.index.shard.ShardId;
 import java.util.Set;
 
 /**
- * a very light weight implementation. will be replaced with proper machinery later
+ * Encapsulates the local and global checkpoints into a single service for use as a shard component.
  */
 public class SequenceNumbersService extends AbstractIndexShardComponent {
 
+    /**
+     * Represents an unassigned sequence number (e.g., can be used on primary operations before they are executed).
+     */
     public static final long UNASSIGNED_SEQ_NO = -2L;
 
     /**
@@ -36,8 +40,8 @@ public class SequenceNumbersService extends AbstractIndexShardComponent {
      */
     public static final long NO_OPS_PERFORMED = -1L;
 
-    final LocalCheckpointService localCheckpointService;
-    final GlobalCheckpointService globalCheckpointService;
+    private final LocalCheckpointTracker localCheckpointTracker;
+    private final GlobalCheckpointTracker globalCheckpointTracker;
 
     /**
      * Initialize the sequence number service. The {@code maxSeqNo} should be set to the last sequence number assigned by this shard, or
@@ -45,11 +49,11 @@ public class SequenceNumbersService extends AbstractIndexShardComponent {
      * shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}, and {@code globalCheckpoint} should be set to the last known global
      * checkpoint for this shard, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}.
      *
-     * @param shardId                  the shard this service is providing tracking local checkpoints for
-     * @param indexSettings            the index settings
-     * @param maxSeqNo                 the last sequence number assigned by this shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
-     * @param localCheckpoint          the last known local checkpoint for this shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
-     * @param globalCheckpoint         the last known global checkpoint for this shard, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}
+     * @param shardId          the shard this service is providing tracking local checkpoints for
+     * @param indexSettings    the index settings
+     * @param maxSeqNo         the last sequence number assigned by this shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
+     * @param localCheckpoint  the last known local checkpoint for this shard, or {@link SequenceNumbersService#NO_OPS_PERFORMED}
+     * @param globalCheckpoint the last known global checkpoint for this shard, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO}
      */
     public SequenceNumbersService(
         final ShardId shardId,
@@ -58,93 +62,125 @@ public class SequenceNumbersService extends AbstractIndexShardComponent {
         final long localCheckpoint,
         final long globalCheckpoint) {
         super(shardId, indexSettings);
-        localCheckpointService = new LocalCheckpointService(shardId, indexSettings, maxSeqNo, localCheckpoint);
-        globalCheckpointService = new GlobalCheckpointService(shardId, indexSettings, globalCheckpoint);
+        localCheckpointTracker = new LocalCheckpointTracker(indexSettings, maxSeqNo, localCheckpoint);
+        globalCheckpointTracker = new GlobalCheckpointTracker(shardId, indexSettings, globalCheckpoint);
     }
 
     /**
-     * generates a new sequence number.
-     * Note: you must call {@link #markSeqNoAsCompleted(long)} after the operation for which this seq# was generated
-     * was completed (whether successfully or with a failure)
+     * Issue the next sequence number. Note that you must call {@link #markSeqNoAsCompleted(long)} after the operation for which the
+     * issued sequence number completes (whether or not the operation completes successfully).
+     *
+     * @return the next assigned sequence number
      */
     public long generateSeqNo() {
-        return localCheckpointService.generateSeqNo();
+        return localCheckpointTracker.generateSeqNo();
     }
 
     /**
-     * Gets the maximum sequence number seen so far.  See {@link LocalCheckpointService#getMaxSeqNo()} for details.
+     * The maximum sequence number issued so far. See {@link LocalCheckpointTracker#getMaxSeqNo()} for additional details.
+     *
+     * @return the maximum sequence number
      */
     public long getMaxSeqNo() {
-        return localCheckpointService.getMaxSeqNo();
+        return localCheckpointTracker.getMaxSeqNo();
     }
 
     /**
-     * marks the given seqNo as completed. See {@link LocalCheckpointService#markSeqNoAsCompleted(long)}
-     * more details
+     * Waits for all operations up to the provided sequence number to complete.
+     *
+     * @param seqNo the sequence number that the checkpoint must advance to before this method returns
+     * @throws InterruptedException if the thread was interrupted while blocking on the condition
      */
-    public void markSeqNoAsCompleted(long seqNo) {
-        localCheckpointService.markSeqNoAsCompleted(seqNo);
+    public void waitForOpsToComplete(final long seqNo) throws InterruptedException {
+        localCheckpointTracker.waitForOpsToComplete(seqNo);
     }
 
     /**
-     * Gets sequence number related stats
+     * Marks the processing of the provided sequence number as completed as updates the checkpoint if possible.
+     * See {@link LocalCheckpointTracker#markSeqNoAsCompleted(long)} for additional details.
+     *
+     * @param seqNo the sequence number to mark as completed
+     */
+    public void markSeqNoAsCompleted(final long seqNo) {
+        localCheckpointTracker.markSeqNoAsCompleted(seqNo);
+    }
+
+    /**
+     * The current sequence number stats.
+     *
+     * @return stats encapuslating the maximum sequence number, the local checkpoint and the global checkpoint
      */
     public SeqNoStats stats() {
-        return new SeqNoStats(getMaxSeqNo(), getLocalCheckpoint(), getGlobalCheckpoint());
+        return localCheckpointTracker.getStats(getGlobalCheckpoint());
     }
 
     /**
-     * notifies the service of a local checkpoint.
-     * see {@link GlobalCheckpointService#updateLocalCheckpoint(String, long)} for details.
-     */
-    public void updateLocalCheckpointForShard(String allocationId, long checkpoint) {
-        globalCheckpointService.updateLocalCheckpoint(allocationId, checkpoint);
-    }
-
-    /**
-     * marks the allocationId as "in sync" with the primary shard.
-     * see {@link GlobalCheckpointService#markAllocationIdAsInSync(String)} for details.
+     * Notifies the service to update the local checkpoint for the shard with the provided allocation ID. See
+     * {@link GlobalCheckpointTracker#updateLocalCheckpoint(String, long)} for details.
      *
-     * @param allocationId    allocationId of the recovering shard
+     * @param allocationId the allocation ID of the shard to update the local checkpoint for
+     * @param checkpoint   the local checkpoint for the shard
      */
-    public void markAllocationIdAsInSync(String allocationId) {
-        globalCheckpointService.markAllocationIdAsInSync(allocationId);
+    public void updateLocalCheckpointForShard(final String allocationId, final long checkpoint) {
+        globalCheckpointTracker.updateLocalCheckpoint(allocationId, checkpoint);
     }
 
+    /**
+     * Marks the shard with the provided allocation ID as in-sync with the primary shard. See
+     * {@link GlobalCheckpointTracker#markAllocationIdAsInSync(String)} for additional details.
+     *
+     * @param allocationId the allocation ID of the shard to mark as in-sync
+     */
+    public void markAllocationIdAsInSync(final String allocationId) {
+        globalCheckpointTracker.markAllocationIdAsInSync(allocationId);
+    }
+
+    /**
+     * Returns the local checkpoint for the shard.
+     *
+     * @return the local checkpoint
+     */
     public long getLocalCheckpoint() {
-        return localCheckpointService.getCheckpoint();
+        return localCheckpointTracker.getCheckpoint();
     }
 
+    /**
+     * Returns the global checkpoint for the shard.
+     *
+     * @return the global checkpoint
+     */
     public long getGlobalCheckpoint() {
-        return globalCheckpointService.getCheckpoint();
+        return globalCheckpointTracker.getCheckpoint();
     }
 
     /**
      * Scans through the currently known local checkpoint and updates the global checkpoint accordingly.
      *
-     * @return true if the checkpoint has been updated or if it can not be updated since one of the local checkpoints
-     * of one of the active allocations is not known.
+     * @return {@code true} if the checkpoint has been updated or if it can not be updated since one of the local checkpoints of one of the
+     * active allocations is not known.
      */
     public boolean updateGlobalCheckpointOnPrimary() {
-        return globalCheckpointService.updateCheckpointOnPrimary();
+        return globalCheckpointTracker.updateCheckpointOnPrimary();
     }
 
     /**
-     * updates the global checkpoint on a replica shard (after it has been updated by the primary).
-     */
-    public void updateGlobalCheckpointOnReplica(long checkpoint) {
-        globalCheckpointService.updateCheckpointOnReplica(checkpoint);
-    }
-
-    /**
-     * Notifies the service of the current allocation ids in the cluster state.
-     * see {@link GlobalCheckpointService#updateAllocationIdsFromMaster(Set, Set)} for details.
+     * Updates the global checkpoint on a replica shard after it has been updated by the primary.
      *
-     * @param activeAllocationIds       the allocation ids of the currently active shard copies
-     * @param initializingAllocationIds the allocation ids of the currently initializing shard copies
+     * @param checkpoint the global checkpoint
      */
-    public void updateAllocationIdsFromMaster(Set<String> activeAllocationIds, Set<String> initializingAllocationIds) {
-        globalCheckpointService.updateAllocationIdsFromMaster(activeAllocationIds, initializingAllocationIds);
+    public void updateGlobalCheckpointOnReplica(final long checkpoint) {
+        globalCheckpointTracker.updateCheckpointOnReplica(checkpoint);
+    }
+
+    /**
+     * Notifies the service of the current allocation IDs in the cluster state. See
+     * {@link GlobalCheckpointTracker#updateAllocationIdsFromMaster(Set, Set)} for details.
+     *
+     * @param activeAllocationIds       the allocation IDs of the currently active shard copies
+     * @param initializingAllocationIds the allocation IDs of the currently initializing shard copies
+     */
+    public void updateAllocationIdsFromMaster(final Set<String> activeAllocationIds, final Set<String> initializingAllocationIds) {
+        globalCheckpointTracker.updateAllocationIdsFromMaster(activeAllocationIds, initializingAllocationIds);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
index 8291298a54d..03afc356f31 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
@@ -61,7 +61,7 @@ public class CommitPoint {
         }
     }
 
-    public static enum Type {
+    public enum Type {
         GENERATED,
         SAVED
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
index e3b6817f13d..12a31076228 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.shard;
 
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -118,7 +119,8 @@ public class CommitPoints implements Iterable<CommitPoint> {
     }
 
     public static CommitPoint fromXContent(byte[] data) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(data)) {
+        // EMPTY is safe here because we never call namedObject
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, data)) {
             String currentFieldName = null;
             XContentParser.Token token = parser.nextToken();
             if (token == null) {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index fc6eac196d8..0e6054deccd 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -75,7 +75,6 @@ import org.elasticsearch.index.cache.request.ShardRequestCache;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.engine.EngineFactory;
@@ -101,7 +100,7 @@ import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.search.stats.ShardSearchStats;
-import org.elasticsearch.index.seqno.GlobalCheckpointService;
+import org.elasticsearch.index.seqno.GlobalCheckpointTracker;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -385,7 +384,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
      * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted
      * @throws IOException                  if shard state could not be persisted
      */
-    public void updateRoutingEntry(final ShardRouting newRouting) throws IOException {
+    public void updateRoutingEntry(ShardRouting newRouting) throws IOException {
         final ShardRouting currentRouting;
         synchronized (mutex) {
             currentRouting = this.shardRouting;
@@ -544,26 +543,27 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return new Engine.Index(uid, doc, seqNo, primaryTerm, version, versionType, origin, startTime, autoGeneratedIdTimestamp, isRetry);
     }
 
-    public Engine.IndexResult index(Engine.Index index) {
+    public Engine.IndexResult index(Engine.Index index) throws IOException {
         ensureWriteAllowed(index);
         Engine engine = getEngine();
         return index(engine, index);
     }
 
-    private Engine.IndexResult index(Engine engine, Engine.Index index) {
+    private Engine.IndexResult index(Engine engine, Engine.Index index) throws IOException {
         active.set(true);
         final Engine.IndexResult result;
-        index = indexingOperationListeners.preIndex(index);
+        index = indexingOperationListeners.preIndex(shardId, index);
         try {
             if (logger.isTraceEnabled()) {
-                logger.trace("index [{}][{}]{}", index.type(), index.id(), index.docs());
+                // don't use index.source().utf8ToString() here source might not be valid UTF-8
+                logger.trace("index [{}][{}] (seq# [{}])",  index.type(), index.id(), index.seqNo());
             }
             result = engine.index(index);
         } catch (Exception e) {
-            indexingOperationListeners.postIndex(index, e);
+            indexingOperationListeners.postIndex(shardId, index, e);
             throw e;
         }
-        indexingOperationListeners.postIndex(index, result);
+        indexingOperationListeners.postIndex(shardId, index, result);
         return result;
     }
 
@@ -593,26 +593,26 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return new Engine.Delete(type, id, uid, seqNo, primaryTerm, version, versionType, origin, startTime);
     }
 
-    public Engine.DeleteResult delete(Engine.Delete delete) {
+    public Engine.DeleteResult delete(Engine.Delete delete) throws IOException {
         ensureWriteAllowed(delete);
         Engine engine = getEngine();
         return delete(engine, delete);
     }
 
-    private Engine.DeleteResult delete(Engine engine, Engine.Delete delete) {
+    private Engine.DeleteResult delete(Engine engine, Engine.Delete delete) throws IOException {
         active.set(true);
         final Engine.DeleteResult result;
-        delete = indexingOperationListeners.preDelete(delete);
+        delete = indexingOperationListeners.preDelete(shardId, delete);
         try {
             if (logger.isTraceEnabled()) {
-                logger.trace("delete [{}]", delete.uid().text());
+                logger.trace("delete [{}] (seq no [{}])", delete.uid().text(), delete.seqNo());
             }
             result = engine.delete(delete);
         } catch (Exception e) {
-            indexingOperationListeners.postDelete(delete, e);
+            indexingOperationListeners.postDelete(shardId, delete, e);
             throw e;
         }
-        indexingOperationListeners.postDelete(delete, result);
+        indexingOperationListeners.postDelete(shardId, delete, result);
         return result;
     }
 
@@ -622,7 +622,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     /**
-     * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link EngineClosedException}.
+     * Writes all indexing changes to disk and opens a new searcher reflecting all changes.  This can throw {@link AlreadyClosedException}.
      */
     public void refresh(String source) {
         verifyNotClosed();
@@ -661,7 +661,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     public RefreshStats refreshStats() {
-        return new RefreshStats(refreshMetric.count(), TimeUnit.NANOSECONDS.toMillis(refreshMetric.sum()));
+        // Null refreshListeners means this shard doesn't support them so there can't be any.
+        int listeners = refreshListeners == null ? 0 : refreshListeners.pendingCount();
+        return new RefreshStats(refreshMetric.count(), TimeUnit.NANOSECONDS.toMillis(refreshMetric.sum()), listeners);
     }
 
     public FlushStats flushStats() {
@@ -669,9 +671,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     public DocsStats docStats() {
-        readAllowed();
-        final Engine engine = getEngine();
-        return engine.getDocStats();
+        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
+            return new DocsStats(searcher.reader().numDocs(), searcher.reader().numDeletedDocs());
+        }
     }
 
     /**
@@ -752,7 +754,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     public CompletionStats completionStats(String... fields) {
         CompletionStats completionStats = new CompletionStats();
-        try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
+        try (Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
         }
         return completionStats;
@@ -932,8 +934,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     if (engine != null && flushEngine) {
                         engine.flushAndClose();
                     }
-                } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
-                    IOUtils.close(engine);
+                } finally {
+                    // playing safe here and close the engine even if the above succeeds - close can be called multiple times
+                    // Also closing refreshListeners to prevent us from accumulating any more listeners
+                    IOUtils.close(engine, refreshListeners);
                     indexShardOperationsLock.close();
                 }
             }
@@ -1005,7 +1009,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         }
         recoveryState.setStage(RecoveryState.Stage.VERIFY_INDEX);
         // also check here, before we apply the translog
-        if (Booleans.parseBoolean(checkIndexOnStartup, false)) {
+        if (Booleans.isTrue(checkIndexOnStartup)) {
             try {
                 checkIndex();
             } catch (IOException ex) {
@@ -1261,7 +1265,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             try {
                 Translog translog = engine.getTranslog();
                 return translog.sizeInBytes() > indexSettings.getFlushThresholdSize().getBytes();
-            } catch (AlreadyClosedException | EngineClosedException ex) {
+            } catch (AlreadyClosedException ex) {
                 // that's fine we are already close - no need to flush
             }
         }
@@ -1300,7 +1304,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     public void activateThrottling() {
         try {
             getEngine().activateThrottling();
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // ignore
         }
     }
@@ -1308,13 +1312,13 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     public void deactivateThrottling() {
         try {
             getEngine().deactivateThrottling();
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // ignore
         }
     }
 
     private void handleRefreshException(Exception e) {
-        if (e instanceof EngineClosedException) {
+        if (e instanceof AlreadyClosedException) {
             // ignore
         } else if (e instanceof RefreshFailedEngineException) {
             RefreshFailedEngineException rfee = (RefreshFailedEngineException) e;
@@ -1364,36 +1368,59 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     /**
-     * notifies the service of a local checkpoint. see {@link GlobalCheckpointService#updateLocalCheckpoint(String, long)} for details.
+     * Notifies the service to update the local checkpoint for the shard with the provided allocation ID. See
+     * {@link GlobalCheckpointTracker#updateLocalCheckpoint(String, long)} for details.
+     *
+     * @param allocationId the allocation ID of the shard to update the local checkpoint for
+     * @param checkpoint   the local checkpoint for the shard
      */
-    public void updateLocalCheckpointForShard(String allocationId, long checkpoint) {
+    public void updateLocalCheckpointForShard(final String allocationId, final long checkpoint) {
         verifyPrimary();
         getEngine().seqNoService().updateLocalCheckpointForShard(allocationId, checkpoint);
     }
 
     /**
-     * marks the allocationId as "in sync" with the primary shard. see {@link GlobalCheckpointService#markAllocationIdAsInSync(String)}
-     * for details.
+     * Waits for all operations up to the provided sequence number to complete.
      *
-     * @param allocationId    allocationId of the recovering shard
+     * @param seqNo the sequence number that the checkpoint must advance to before this method returns
+     * @throws InterruptedException if the thread was interrupted while blocking on the condition
      */
-    public void markAllocationIdAsInSync(String allocationId) {
+    public void waitForOpsToComplete(final long seqNo) throws InterruptedException {
+        getEngine().seqNoService().waitForOpsToComplete(seqNo);
+    }
+
+    /**
+     * Marks the shard with the provided allocation ID as in-sync with the primary shard. See
+     * {@link GlobalCheckpointTracker#markAllocationIdAsInSync(String)} for additional details.
+     *
+     * @param allocationId the allocation ID of the shard to mark as in-sync
+     */
+    public void markAllocationIdAsInSync(final String allocationId) {
         verifyPrimary();
         getEngine().seqNoService().markAllocationIdAsInSync(allocationId);
     }
 
+    /**
+     * Returns the local checkpoint for the shard.
+     *
+     * @return the local checkpoint
+     */
     public long getLocalCheckpoint() {
         return getEngine().seqNoService().getLocalCheckpoint();
     }
 
+    /**
+     * Returns the global checkpoint for the shard.
+     *
+     * @return the global checkpoint
+     */
     public long getGlobalCheckpoint() {
         return getEngine().seqNoService().getGlobalCheckpoint();
     }
 
     /**
-     * Checks whether the global checkpoint can be updated based on current knowledge of local checkpoints on the different
-     * shard copies. The checkpoint is updated or more information is required from the replica, a global checkpoint sync
-     * is initiated.
+     * Checks whether the global checkpoint can be updated based on current knowledge of local checkpoints on the different shard copies.
+     * The checkpoint is updated or if more information is required from the replica, a global checkpoint sync is initiated.
      */
     public void updateGlobalCheckpointOnPrimary() {
         verifyPrimary();
@@ -1403,24 +1430,26 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     /**
-     * updates the global checkpoint on a replica shard (after it has been updated by the primary).
+     * Updates the global checkpoint on a replica shard after it has been updated by the primary.
+     *
+     * @param checkpoint the global checkpoint
      */
-    public void updateGlobalCheckpointOnReplica(long checkpoint) {
+    public void updateGlobalCheckpointOnReplica(final long checkpoint) {
         verifyReplicationTarget();
         getEngine().seqNoService().updateGlobalCheckpointOnReplica(checkpoint);
     }
 
     /**
-     * Notifies the service of the current allocation ids in the cluster state.
-     * see {@link GlobalCheckpointService#updateAllocationIdsFromMaster(Set, Set)} for details.
+     * Notifies the service of the current allocation IDs in the cluster state. See
+     * {@link GlobalCheckpointTracker#updateAllocationIdsFromMaster(Set, Set)} for details.
      *
-     * @param activeAllocationIds       the allocation ids of the currently active shard copies
-     * @param initializingAllocationIds the allocation ids of the currently initializing shard copies
+     * @param activeAllocationIds       the allocation IDs of the currently active shard copies
+     * @param initializingAllocationIds the allocation IDs of the currently initializing shard copies
      */
-    public void updateAllocationIdsFromMaster(Set<String> activeAllocationIds, Set<String> initializingAllocationIds) {
+    public void updateAllocationIdsFromMaster(final Set<String> activeAllocationIds, final Set<String> initializingAllocationIds) {
         verifyPrimary();
-        Engine engine = getEngineOrNull();
-        // if engine is not yet started, we are not ready yet and can just ignore this
+        final Engine engine = getEngineOrNull();
+        // if the engine is not yet started, we are not ready yet and can just ignore this
         if (engine != null) {
             engine.seqNoService().updateAllocationIdsFromMaster(activeAllocationIds, initializingAllocationIds);
         }
@@ -1435,7 +1464,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         internalIndexingStats.noopUpdate(type);
     }
 
-
     private void checkIndex() throws IOException {
         if (store.tryIncRef()) {
             try {
@@ -1512,7 +1540,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     Engine getEngine() {
         Engine engine = getEngineOrNull();
         if (engine == null) {
-            throw new EngineClosedException(shardId);
+            throw new AlreadyClosedException("engine is closed");
         }
         return engine;
     }
@@ -1649,7 +1677,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private Engine createNewEngine(EngineConfig config) {
         synchronized (mutex) {
             if (state == IndexShardState.CLOSED) {
-                throw new EngineClosedException(shardId);
+                throw new AlreadyClosedException(shardId + " can't create engine - shard is closed");
             }
             assert this.currentEngineReference.get() == null;
             Engine engine = newEngine(config);
@@ -1751,7 +1779,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             try {
                 final Engine engine = getEngine();
                 engine.getTranslog().ensureSynced(candidates.stream().map(Tuple::v1));
-            } catch (EngineClosedException ex) {
+            } catch (AlreadyClosedException ex) {
                 // that's fine since we already synced everything on engine close - this also is conform with the methods
                 // documentation
             } catch (IOException ex) { // if this fails we are in deep shit - fail the request
@@ -1866,8 +1894,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
      * refresh listeners.
      * Otherwise <code>false</code>.
      *
-     * @throws EngineClosedException  if the engine is already closed
-     * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed
+     * @throws AlreadyClosedException if the engine or internal indexwriter in the engine is already closed
      */
     public boolean isRefreshNeeded() {
         return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded());
@@ -1906,12 +1933,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         }
 
         @Override
-        protected void index(Engine engine, Engine.Index engineIndex) {
+        protected void index(Engine engine, Engine.Index engineIndex) throws IOException {
             IndexShard.this.index(engine, engineIndex);
         }
 
         @Override
-        protected void delete(Engine engine, Engine.Delete engineDelete) {
+        protected void delete(Engine engine, Engine.Delete engineDelete) throws IOException {
             IndexShard.this.delete(engine, engineDelete);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
index 36f2765222a..335196fe681 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
@@ -33,29 +33,29 @@ public interface IndexingOperationListener {
     /**
      * Called before the indexing occurs.
      */
-    default Engine.Index preIndex(Engine.Index operation) {
+    default Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
         return operation;
     }
 
     /**
      * Called after the indexing operation occurred. Note that this is
      * also called when indexing a document did not succeed due to document
-     * related failures. See {@link #postIndex(Engine.Index, Exception)}
+     * related failures. See {@link #postIndex(ShardId, Engine.Index, Exception)}
      * for engine level failures
      */
-    default void postIndex(Engine.Index index, Engine.IndexResult result) {}
+    default void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {}
 
     /**
      * Called after the indexing operation occurred with engine level exception.
-     * See {@link #postIndex(Engine.Index, Engine.IndexResult)} for document
+     * See {@link #postIndex(ShardId, Engine.Index, Engine.IndexResult)} for document
      * related failures
      */
-    default void postIndex(Engine.Index index, Exception ex) {}
+    default void postIndex(ShardId shardId, Engine.Index index, Exception ex) {}
 
     /**
      * Called before the delete occurs.
      */
-    default Engine.Delete preDelete(Engine.Delete delete) {
+    default Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
         return delete;
     }
 
@@ -63,17 +63,17 @@ public interface IndexingOperationListener {
     /**
      * Called after the delete operation occurred. Note that this is
      * also called when deleting a document did not succeed due to document
-     * related failures. See {@link #postDelete(Engine.Delete, Exception)}
+     * related failures. See {@link #postDelete(ShardId, Engine.Delete, Exception)}
      * for engine level failures
      */
-    default void postDelete(Engine.Delete delete, Engine.DeleteResult result) {}
+    default void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {}
 
     /**
      * Called after the delete operation occurred with engine level exception.
-     * See {@link #postDelete(Engine.Delete, Engine.DeleteResult)} for document
+     * See {@link #postDelete(ShardId, Engine.Delete, Engine.DeleteResult)} for document
      * related failures
      */
-    default void postDelete(Engine.Delete delete, Exception ex) {}
+    default void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {}
 
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
@@ -88,11 +88,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public Engine.Index preIndex(Engine.Index operation) {
+        public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
             assert operation != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.preIndex(operation);
+                    listener.preIndex(shardId, operation);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("preIndex listener [{}] failed", listener), e);
                 }
@@ -101,11 +101,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public void postIndex(Engine.Index index, Engine.IndexResult result) {
+        public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
             assert index != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postIndex(index, result);
+                    listener.postIndex(shardId, index, result);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postIndex listener [{}] failed", listener), e);
                 }
@@ -113,11 +113,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public void postIndex(Engine.Index index, Exception ex) {
+        public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
             assert index != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postIndex(index, ex);
+                    listener.postIndex(shardId, index, ex);
                 } catch (Exception inner) {
                     inner.addSuppressed(ex);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postIndex listener [{}] failed", listener), inner);
@@ -126,11 +126,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public Engine.Delete preDelete(Engine.Delete delete) {
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
             assert delete != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.preDelete(delete);
+                    listener.preDelete(shardId, delete);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("preDelete listener [{}] failed", listener), e);
                 }
@@ -139,11 +139,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+        public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
             assert delete != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postDelete(delete, result);
+                    listener.postDelete(shardId, delete, result);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postDelete listener [{}] failed", listener), e);
                 }
@@ -151,11 +151,11 @@ public interface IndexingOperationListener {
         }
 
         @Override
-        public void postDelete(Engine.Delete delete, Exception ex) {
+        public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
             assert delete != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
-                    listener.postDelete(delete, ex);
+                    listener.postDelete(shardId, delete, ex);
                 } catch (Exception inner) {
                     inner.addSuppressed(ex);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("postDelete listener [{}] failed", listener), inner);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
index 32868a7368a..ada869a1d9c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
@@ -65,7 +65,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public Engine.Index preIndex(Engine.Index operation) {
+    public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
         if (!operation.origin().isRecovery()) {
             totalStats.indexCurrent.inc();
             typeStats(operation.type()).indexCurrent.inc();
@@ -74,7 +74,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public void postIndex(Engine.Index index, Engine.IndexResult result) {
+    public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
         if (result.hasFailure() == false) {
             if (!index.origin().isRecovery()) {
                 long took = result.getTook();
@@ -85,12 +85,12 @@ final class InternalIndexingStats implements IndexingOperationListener {
                 typeStats.indexCurrent.dec();
             }
         } else {
-            postIndex(index, result.getFailure());
+            postIndex(shardId, index, result.getFailure());
         }
     }
 
     @Override
-    public void postIndex(Engine.Index index, Exception ex) {
+    public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
         if (!index.origin().isRecovery()) {
             totalStats.indexCurrent.dec();
             typeStats(index.type()).indexCurrent.dec();
@@ -100,7 +100,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public Engine.Delete preDelete(Engine.Delete delete) {
+    public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
         if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.inc();
             typeStats(delete.type()).deleteCurrent.inc();
@@ -110,7 +110,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+    public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
         if (result.hasFailure() == false) {
             if (!delete.origin().isRecovery()) {
                 long took = result.getTook();
@@ -121,12 +121,12 @@ final class InternalIndexingStats implements IndexingOperationListener {
                 typeStats.deleteCurrent.dec();
             }
         } else {
-            postDelete(delete, result.getFailure());
+            postDelete(shardId, delete, result.getFailure());
         }
     }
 
     @Override
-    public void postDelete(Engine.Delete delete, Exception ex) {
+    public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
         if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.dec();
             typeStats(delete.type()).deleteCurrent.dec();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
index d576f4d2ab5..302cdda4804 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
@@ -26,8 +26,7 @@ import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.NoLockFactory;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.store.Store;
 
@@ -42,7 +41,7 @@ final class LocalShardSnapshot implements Closeable {
     private final IndexCommit indexCommit;
     private final AtomicBoolean closed = new AtomicBoolean(false);
 
-    public LocalShardSnapshot(IndexShard shard) {
+    LocalShardSnapshot(IndexShard shard) {
         this.shard = shard;
         store = shard.store();
         store.incRef();
@@ -123,8 +122,8 @@ final class LocalShardSnapshot implements Closeable {
         }
     }
 
-    ImmutableOpenMap<String, MappingMetaData> getMappings() {
-        return shard.indexSettings.getIndexMetaData().getMappings();
+    IndexMetaData getIndexMetaData() {
+        return shard.indexSettings.getIndexMetaData();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index ca94f1ea961..f0df6e12b8c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.ReferenceManager;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.index.translog.Translog;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,18 +36,26 @@ import static java.util.Objects.requireNonNull;
 
 /**
  * Allows for the registration of listeners that are called when a change becomes visible for search. This functionality is exposed from
- * {@link IndexShard} but kept here so it can be tested without standing up the entire thing. 
+ * {@link IndexShard} but kept here so it can be tested without standing up the entire thing.
+ *
+ * When {@link Closeable#close()}d it will no longer accept listeners and flush any existing listeners.
  */
-public final class RefreshListeners implements ReferenceManager.RefreshListener {
+public final class RefreshListeners implements ReferenceManager.RefreshListener, Closeable {
     private final IntSupplier getMaxRefreshListeners;
     private final Runnable forceRefresh;
     private final Executor listenerExecutor;
     private final Logger logger;
 
+    /**
+     * Is this closed? If true then we won't add more listeners and have flushed all pending listeners.
+     */
+    private volatile boolean closed = false;
     /**
      * List of refresh listeners. Defaults to null and built on demand because most refresh cycles won't need it. Entries are never removed
      * from it, rather, it is nulled and rebuilt when needed again. The (hopefully) rare entries that didn't make the current refresh cycle
      * are just added back to the new list. Both the reference and the contents are always modified while synchronized on {@code this}.
+     *
+     * We never set this to non-null while closed it {@code true}.
      */
     private volatile List<Tuple<Translog.Location, Consumer<Boolean>>> refreshListeners = null;
     /**
@@ -80,12 +89,17 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
             return true;
         }
         synchronized (this) {
-            if (refreshListeners == null) {
-                refreshListeners = new ArrayList<>();
+            List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
+            if (listeners == null) {
+                if (closed) {
+                    throw new IllegalStateException("can't wait for refresh on a closed index");
+                }
+                listeners = new ArrayList<>();
+                refreshListeners = listeners;
             }
-            if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) {
+            if (listeners.size() < getMaxRefreshListeners.getAsInt()) {
                 // We have a free slot so register the listener
-                refreshListeners.add(new Tuple<>(location, listener));
+                listeners.add(new Tuple<>(location, listener));
                 return false;
             }
         }
@@ -95,12 +109,34 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
         return true;
     }
 
+    @Override
+    public void close() throws IOException {
+        List<Tuple<Translog.Location, Consumer<Boolean>>> oldListeners;
+        synchronized (this) {
+            oldListeners = refreshListeners;
+            refreshListeners = null;
+            closed = true;
+        }
+        // Fire any listeners we might have had
+        fireListeners(oldListeners);
+    }
+
     /**
      * Returns true if there are pending listeners.
      */
     public boolean refreshNeeded() {
+        // A null list doesn't need a refresh. If we're closed we don't need a refresh either.
+        return refreshListeners != null && false == closed;
+    }
+
+    /**
+     * The number of pending listeners.
+     */
+    public int pendingCount() {
         // No need to synchronize here because we're doing a single volatile read
-        return refreshListeners != null;
+        List<Tuple<Translog.Location, Consumer<Boolean>>> listeners = refreshListeners;
+        // A null list means we haven't accumulated any listeners. Otherwise we need the size.
+        return listeners == null ? 0 : listeners.size();
     }
 
     /**
@@ -125,33 +161,25 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
 
     @Override
     public void afterRefresh(boolean didRefresh) throws IOException {
-        /*
-         * We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made
+        /* We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made
          * it into the refresh, but the way we snapshot the translog position before the refresh, things can sneak into the refresh that we
-         * don't know about.
-         */
+         * don't know about. */
         if (null == currentRefreshLocation) {
-            /*
-             * The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This
-             * usually happens during recovery. The next refresh cycle out to pick up this refresh.
-             */
+            /* The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This
+             * usually happens during recovery. The next refresh cycle out to pick up this refresh. */
             return;
         }
-        /*
-         * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
+        /* Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
          * around with refreshListeners or synchronizing at all. Note that it is not safe for us to abort early if we haven't advanced the
          * position here because we set and read lastRefreshedLocation outside of a synchronized block. We do that so that waiting for a
          * refresh that has already passed is just a volatile read but the cost is that any check whether or not we've advanced the
          * position will introduce a race between adding the listener and the position check. We could work around this by moving this
          * assignment into the synchronized block below and double checking lastRefreshedLocation in addOrNotify's synchronized block but
-         * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate.
-         */
+         * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate. */
         lastRefreshedLocation = currentRefreshLocation;
-        /*
-         * Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be
+        /* Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be
          * in the list we iterate over and very likely won't be candidates for refresh anyway because we've already moved the
-         * lastRefreshedLocation.
-         */
+         * lastRefreshedLocation. */
         List<Tuple<Translog.Location, Consumer<Boolean>>> candidates;
         synchronized (this) {
             candidates = refreshListeners;
@@ -162,16 +190,15 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
             refreshListeners = null;
         }
         // Iterate the list of listeners, copying the listeners to fire to one list and those to preserve to another list.
-        List<Consumer<Boolean>> listenersToFire = null;
+        List<Tuple<Translog.Location, Consumer<Boolean>>> listenersToFire = null;
         List<Tuple<Translog.Location, Consumer<Boolean>>> preservedListeners = null;
         for (Tuple<Translog.Location, Consumer<Boolean>> tuple : candidates) {
             Translog.Location location = tuple.v1();
-            Consumer<Boolean> listener = tuple.v2();
             if (location.compareTo(currentRefreshLocation) <= 0) {
                 if (listenersToFire == null) {
                     listenersToFire = new ArrayList<>();
                 }
-                listenersToFire.add(listener);
+                listenersToFire.add(tuple);
             } else {
                 if (preservedListeners == null) {
                     preservedListeners = new ArrayList<>();
@@ -179,27 +206,36 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
                 preservedListeners.add(tuple);
             }
         }
-        /*
-         * Now add any preserved listeners back to the running list of refresh listeners while under lock. We'll try them next time. While
-         * we were iterating the list of listeners new listeners could have come in. That means that adding all of our preserved listeners
-         * might push our list of listeners above the maximum number of slots allowed. This seems unlikely because we expect few listeners
-         * to be preserved. And the next listener while we're full will trigger a refresh anyway.
-         */
+        /* Now deal with the listeners that it isn't time yet to fire. We need to do this under lock so we don't miss a concurrent close or
+         * newly registered listener. If we're not closed we just add the listeners to the list of listeners we check next time. If we are
+         * closed we fire the listeners even though it isn't time for them. */
         if (preservedListeners != null) {
             synchronized (this) {
                 if (refreshListeners == null) {
-                    refreshListeners = new ArrayList<>();
+                    if (closed) {
+                        listenersToFire.addAll(preservedListeners);
+                    } else {
+                        refreshListeners = preservedListeners;
+                    }
+                } else {
+                    assert closed == false : "Can't be closed and have non-null refreshListeners";
+                    refreshListeners.addAll(preservedListeners);
                 }
-                refreshListeners.addAll(preservedListeners);
             }
         }
         // Lastly, fire the listeners that are ready on the listener thread pool
+        fireListeners(listenersToFire);
+    }
+
+    /**
+     * Fire some listeners. Does nothing if the list of listeners is null.
+     */
+    private void fireListeners(List<Tuple<Translog.Location, Consumer<Boolean>>> listenersToFire) {
         if (listenersToFire != null) {
-            final List<Consumer<Boolean>> finalListenersToFire = listenersToFire;
             listenerExecutor.execute(() -> {
-                for (Consumer<Boolean> listener : finalListenersToFire) {
+                for (Tuple<Translog.Location, Consumer<Boolean>> listener : listenersToFire) {
                     try {
-                        listener.accept(false);
+                        listener.v2().accept(false);
                     } catch (Exception e) {
                         logger.warn("Error firing refresh listener", e);
                     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
index a9bc63ae44f..a806c414e9a 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardId.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.shard;
 
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -71,6 +72,22 @@ public class ShardId implements Streamable, Comparable<ShardId> {
         return "[" + index.getName() + "][" + shardId + "]";
     }
 
+    /**
+     * Parse the string representation of this shardId back to an object.
+     * We lose index uuid information here, but since we use toString in
+     * rest responses, this is the best we can do to reconstruct the object
+     * on the client side.
+     */
+    public static ShardId fromString(String shardIdString) {
+        int splitPosition = shardIdString.indexOf("][");
+        if (splitPosition <= 0 || shardIdString.charAt(0) != '[' || shardIdString.charAt(shardIdString.length() - 1) != ']') {
+            throw new IllegalArgumentException("Unexpected shardId string format, expected [indexName][shardId] but got " + shardIdString);
+        }
+        String indexName = shardIdString.substring(1, splitPosition);
+        int shardId = Integer.parseInt(shardIdString.substring(splitPosition + 2, shardIdString.length() - 1));
+        return new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
index 23b17c290f1..eb2d530fd99 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.shard;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.IndexSettings;
@@ -113,7 +114,8 @@ public final class ShardPath {
         final Path[] paths = env.availableShardPaths(shardId);
         Path loadedPath = null;
         for (Path path : paths) {
-            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
+            // EMPTY is safe here because we never call namedObject
+            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
             if (load != null) {
                 if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                     logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
@@ -150,7 +152,8 @@ public final class ShardPath {
         final String indexUUID = indexSettings.getUUID();
         final Path[] paths = env.availableShardPaths(lock.getShardId());
         for (Path path : paths) {
-            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
+            // EMPTY is safe here because we never call namedObject
+            ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, path);
             if (load != null) {
                 if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
                     logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
index c7d1938cd6d..32ddcd1c273 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.shard;
 
 public class SnapshotStatus {
 
-    public static enum Stage {
+    public enum Stage {
         NONE,
         INDEX,
         TRANSLOG,
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 2e28b4051e5..04c2113dea3 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -104,12 +104,11 @@ final class StoreRecovery {
             if (indices.size() > 1) {
                 throw new IllegalArgumentException("can't add shards from more than one index");
             }
-            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
+            IndexMetaData indexMetaData = shards.get(0).getIndexMetaData();
+            for (ObjectObjectCursor<String, MappingMetaData> mapping : indexMetaData.getMappings()) {
                 mappingUpdateConsumer.accept(mapping.key, mapping.value);
             }
-            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
-                indexShard.mapperService().merge(mapping.key,mapping.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
-            }
+            indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
             return executeRecovery(indexShard, () -> {
                 logger.debug("starting recovery from local shards {}", shards);
                 try {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index f27958b71f5..ad00e39097e 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -22,6 +22,8 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
@@ -29,6 +31,7 @@ import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.rest.RestStatus;
 
@@ -94,6 +97,7 @@ public class TranslogRecoveryPerformer {
                 }
             }
         }
+
         return opsRecovered;
     }
 
@@ -146,7 +150,7 @@ public class TranslogRecoveryPerformer {
      *                            cause a {@link MapperException} to be thrown if an update
      *                            is encountered.
      */
-    private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) {
+    private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) throws IOException {
 
         try {
             switch (operation.opType()) {
@@ -154,26 +158,33 @@ public class TranslogRecoveryPerformer {
                     Translog.Index index = (Translog.Index) operation;
                     // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
                     // autoGeneratedID docs that are coming from the primary are updated correctly.
-                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source())
+                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()),
+                        source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentFactory.xContentType(index.source()))
                             .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(),
                             index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true);
                     maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates);
-                    if (logger.isTraceEnabled()) {
-                        logger.trace("[translog] recover [index] op of [{}][{}]", index.type(), index.id());
-                    }
+                    logger.trace("[translog] recover [index] op [({}, {})] of [{}][{}]", index.seqNo(), index.primaryTerm(), index.type(), index.id());
                     index(engine, engineIndex);
                     break;
                 case DELETE:
                     Translog.Delete delete = (Translog.Delete) operation;
                     Uid uid = Uid.createUid(delete.uid().text());
-                    if (logger.isTraceEnabled()) {
-                        logger.trace("[translog] recover [delete] op of [{}][{}]", uid.type(), uid.id());
-                    }
+                    logger.trace("[translog] recover [delete] op [({}, {})] of [{}][{}]", delete.seqNo(), delete.primaryTerm(), uid.type(), uid.id());
                     final Engine.Delete engineDelete = new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.seqNo(),
                             delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(),
                             origin, System.nanoTime());
                     delete(engine, engineDelete);
                     break;
+                case NO_OP:
+                    final Translog.NoOp noOp = (Translog.NoOp) operation;
+                    final long seqNo = noOp.seqNo();
+                    final long primaryTerm = noOp.primaryTerm();
+                    final String reason = noOp.reason();
+                    logger.trace("[translog] recover [no_op] op [({}, {})] of [{}]", seqNo, primaryTerm, reason);
+                    final Engine.NoOp engineNoOp =
+                        new Engine.NoOp(null, seqNo, primaryTerm, 0, VersionType.INTERNAL, origin, System.nanoTime(), reason);
+                    noOp(engine, engineNoOp);
+                    break;
                 default:
                     throw new IllegalStateException("No operation defined for [" + operation + "]");
             }
@@ -198,14 +209,17 @@ public class TranslogRecoveryPerformer {
         operationProcessed();
     }
 
-    protected void index(Engine engine, Engine.Index engineIndex) {
+    protected void index(Engine engine, Engine.Index engineIndex) throws IOException {
         engine.index(engineIndex);
     }
 
-    protected void delete(Engine engine, Engine.Delete engineDelete) {
+    protected void delete(Engine engine, Engine.Delete engineDelete) throws IOException {
         engine.delete(engineDelete);
     }
 
+    protected void noOp(Engine engine, Engine.NoOp engineNoOp) {
+        engine.noOp(engineNoOp);
+    }
 
     /**
      * Called once for every processed operation by this recovery performer.
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
index 68e50da1348..a1323a95984 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/BM25SimilarityProvider.java
@@ -21,6 +21,9 @@ package org.elasticsearch.index.similarity;
 
 import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -38,11 +41,13 @@ public class BM25SimilarityProvider extends AbstractSimilarityProvider {
 
     private final BM25Similarity similarity;
 
-    public BM25SimilarityProvider(String name, Settings settings) {
+    public BM25SimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
         float k1 = settings.getAsFloat("k1", 1.2f);
         float b = settings.getAsFloat("b", 0.75f);
-        boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
+        final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(getClass()));
+        boolean discountOverlaps =
+            settings.getAsBooleanLenientForPreEs6Indices(Version.indexCreated(indexSettings), "discount_overlaps", true, deprecationLogger);
 
         this.similarity = new BM25Similarity(k1, b);
         this.similarity.setDiscountOverlaps(discountOverlaps);
@@ -55,4 +60,5 @@ public class BM25SimilarityProvider extends AbstractSimilarityProvider {
     public Similarity get() {
         return similarity;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/ClassicSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/ClassicSimilarityProvider.java
index f9a6ff2f5fb..e031c5b3dac 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/ClassicSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/ClassicSimilarityProvider.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.index.similarity;
 
 import org.apache.lucene.search.similarities.ClassicSimilarity;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 
 /**
@@ -35,9 +38,10 @@ public class ClassicSimilarityProvider extends AbstractSimilarityProvider {
 
     private final ClassicSimilarity similarity = new ClassicSimilarity();
 
-    public ClassicSimilarityProvider(String name, Settings settings) {
+    public ClassicSimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
-        boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
+        boolean discountOverlaps = settings.getAsBooleanLenientForPreEs6Indices(
+            Version.indexCreated(indexSettings), "discount_overlaps", true, new DeprecationLogger(ESLoggerFactory.getLogger(getClass())));
         this.similarity.setDiscountOverlaps(discountOverlaps);
     }
 
@@ -48,4 +52,5 @@ public class ClassicSimilarityProvider extends AbstractSimilarityProvider {
     public ClassicSimilarity get() {
         return similarity;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/DFISimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/DFISimilarityProvider.java
index 54b0aab7588..b1148fbb2d5 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/DFISimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/DFISimilarityProvider.java
@@ -25,6 +25,9 @@ import org.apache.lucene.search.similarities.IndependenceChiSquared;
 import org.apache.lucene.search.similarities.IndependenceSaturated;
 import org.apache.lucene.search.similarities.IndependenceStandardized;
 import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 
 import java.util.HashMap;
@@ -55,9 +58,10 @@ public class DFISimilarityProvider extends AbstractSimilarityProvider {
 
     private final DFISimilarity similarity;
 
-    public DFISimilarityProvider(String name, Settings settings) {
+    public DFISimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
-        boolean discountOverlaps = settings.getAsBoolean("discount_overlaps", true);
+        boolean discountOverlaps = settings.getAsBooleanLenientForPreEs6Indices(
+            Version.indexCreated(indexSettings), "discount_overlaps", true, new DeprecationLogger(ESLoggerFactory.getLogger(getClass())));
         Independence measure = parseIndependence(settings);
         this.similarity = new DFISimilarity(measure);
         this.similarity.setDiscountOverlaps(discountOverlaps);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
index 2aeb685de34..0d47e86da01 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/DFRSimilarityProvider.java
@@ -75,7 +75,7 @@ public class DFRSimilarityProvider extends AbstractSimilarityProvider {
 
     private final DFRSimilarity similarity;
 
-    public DFRSimilarityProvider(String name, Settings settings) {
+    public DFRSimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
         BasicModel basicModel = parseBasicModel(settings);
         AfterEffect afterEffect = parseAfterEffect(settings);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
index 4b83bc838f2..a43276bbfaa 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/IBSimilarityProvider.java
@@ -65,7 +65,7 @@ public class IBSimilarityProvider extends AbstractSimilarityProvider {
 
     private final IBSimilarity similarity;
 
-    public IBSimilarityProvider(String name, Settings settings) {
+    public IBSimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
         Distribution distribution = parseDistribution(settings);
         Lambda lambda = parseLambda(settings);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
index 24494dc0b75..170a7e42133 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/LMDirichletSimilarityProvider.java
@@ -36,7 +36,7 @@ public class LMDirichletSimilarityProvider extends AbstractSimilarityProvider {
 
     private final LMDirichletSimilarity similarity;
 
-    public LMDirichletSimilarityProvider(String name, Settings settings) {
+    public LMDirichletSimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
         float mu = settings.getAsFloat("mu", 2000f);
         this.similarity = new LMDirichletSimilarity(mu);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java b/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
index cd10d297355..2ee04b78ec2 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/LMJelinekMercerSimilarityProvider.java
@@ -36,7 +36,7 @@ public class LMJelinekMercerSimilarityProvider extends AbstractSimilarityProvide
 
     private final LMJelinekMercerSimilarity similarity;
 
-    public LMJelinekMercerSimilarityProvider(String name, Settings settings) {
+    public LMJelinekMercerSimilarityProvider(String name, Settings settings, Settings indexSettings) {
         super(name);
         float lambda = settings.getAsFloat("lambda", 0.1f);
         this.similarity = new LMJelinekMercerSimilarity(lambda);
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index a8b7fafb980..54aa940a71f 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.similarity;
 import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
 import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexModule;
@@ -32,7 +33,6 @@ import org.elasticsearch.index.mapper.MapperService;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.function.BiFunction;
 
 public final class SimilarityService extends AbstractIndexComponent {
 
@@ -40,11 +40,11 @@ public final class SimilarityService extends AbstractIndexComponent {
     private final Similarity defaultSimilarity;
     private final Similarity baseSimilarity;
     private final Map<String, SimilarityProvider> similarities;
-    static final Map<String, BiFunction<String, Settings, SimilarityProvider>> DEFAULTS;
-    public static final Map<String, BiFunction<String, Settings, SimilarityProvider>> BUILT_IN;
+    private static final Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> DEFAULTS;
+    public static final Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> BUILT_IN;
     static {
-        Map<String, BiFunction<String, Settings, SimilarityProvider>> defaults = new HashMap<>();
-        Map<String, BiFunction<String, Settings, SimilarityProvider>> buildIn = new HashMap<>();
+        Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> defaults = new HashMap<>();
+        Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> buildIn = new HashMap<>();
         defaults.put("classic", ClassicSimilarityProvider::new);
         defaults.put("BM25", BM25SimilarityProvider::new);
         buildIn.put("classic", ClassicSimilarityProvider::new);
@@ -58,7 +58,8 @@ public final class SimilarityService extends AbstractIndexComponent {
         BUILT_IN = Collections.unmodifiableMap(buildIn);
     }
 
-    public SimilarityService(IndexSettings indexSettings, Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities) {
+    public SimilarityService(IndexSettings indexSettings,
+                             Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> similarities) {
         super(indexSettings);
         Map<String, SimilarityProvider> providers = new HashMap<>(similarities.size());
         Map<String, Settings> similaritySettings = this.indexSettings.getSettings().getGroups(IndexModule.SIMILARITY_SETTINGS_PREFIX);
@@ -68,20 +69,22 @@ public final class SimilarityService extends AbstractIndexComponent {
             if(BUILT_IN.containsKey(name) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) {
                 throw new IllegalArgumentException("Cannot redefine built-in Similarity [" + name + "]");
             }
-            Settings settings = entry.getValue();
-            String typeName = settings.get("type");
+            Settings providerSettings = entry.getValue();
+            String typeName = providerSettings.get("type");
             if (typeName == null) {
                 throw new IllegalArgumentException("Similarity [" + name + "] must have an associated type");
             } else if ((similarities.containsKey(typeName) || BUILT_IN.containsKey(typeName)) == false) {
                 throw new IllegalArgumentException("Unknown Similarity type [" + typeName + "] for [" + name + "]");
             }
-            BiFunction<String, Settings, SimilarityProvider> factory = similarities.getOrDefault(typeName, BUILT_IN.get(typeName));
-            if (settings == null) {
-                settings = Settings.Builder.EMPTY_SETTINGS;
+            TriFunction<String, Settings, Settings, SimilarityProvider> defaultFactory = BUILT_IN.get(typeName);
+            TriFunction<String, Settings, Settings, SimilarityProvider> factory = similarities.getOrDefault(typeName, defaultFactory);
+            if (providerSettings == null) {
+                providerSettings = Settings.Builder.EMPTY_SETTINGS;
             }
-            providers.put(name, factory.apply(name, settings));
+            providers.put(name, factory.apply(name, providerSettings, indexSettings.getSettings()));
         }
-        for (Map.Entry<String, SimilarityProvider> entry : addSimilarities(similaritySettings, DEFAULTS).entrySet()) {
+        Map<String, SimilarityProvider> providerMapping = addSimilarities(similaritySettings, indexSettings.getSettings(), DEFAULTS);
+        for (Map.Entry<String, SimilarityProvider> entry : providerMapping.entrySet()) {
             // Avoid overwriting custom providers for indices older that v5.0
             if (providers.containsKey(entry.getKey()) && indexSettings.getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) {
                 continue;
@@ -102,17 +105,17 @@ public final class SimilarityService extends AbstractIndexComponent {
                 defaultSimilarity;
     }
 
-    private Map<String, SimilarityProvider> addSimilarities(Map<String, Settings>  similaritySettings,
-                                 Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities)  {
+    private Map<String, SimilarityProvider> addSimilarities(Map<String, Settings>  similaritySettings, Settings indexSettings,
+                                 Map<String, TriFunction<String, Settings, Settings, SimilarityProvider>> similarities)  {
         Map<String, SimilarityProvider> providers = new HashMap<>(similarities.size());
-        for (Map.Entry<String, BiFunction<String, Settings, SimilarityProvider>> entry : similarities.entrySet()) {
+        for (Map.Entry<String, TriFunction<String, Settings, Settings, SimilarityProvider>> entry : similarities.entrySet()) {
             String name = entry.getKey();
-            BiFunction<String, Settings, SimilarityProvider> factory = entry.getValue();
-            Settings settings = similaritySettings.get(name);
-            if (settings == null) {
-                settings = Settings.Builder.EMPTY_SETTINGS;
+            TriFunction<String, Settings, Settings, SimilarityProvider> factory = entry.getValue();
+            Settings providerSettings = similaritySettings.get(name);
+            if (providerSettings == null) {
+                providerSettings = Settings.Builder.EMPTY_SETTINGS;
             }
-            providers.put(name, factory.apply(name, settings));
+            providers.put(name, factory.apply(name, providerSettings, indexSettings));
         }
         return providers;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
index 91636b53292..37b728d43d6 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
@@ -23,11 +23,9 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -41,9 +39,7 @@ import java.util.List;
 /**
  * Shard snapshot metadata
  */
-public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuilder<BlobStoreIndexShardSnapshot> {
-
-    public static final BlobStoreIndexShardSnapshot PROTO = new BlobStoreIndexShardSnapshot();
+public class BlobStoreIndexShardSnapshot implements ToXContent {
 
     /**
      * Information about snapshotted file
@@ -507,8 +503,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
      * @param parser parser
      * @return shard snapshot metadata
      */
-    public BlobStoreIndexShardSnapshot fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-
+    public static BlobStoreIndexShardSnapshot fromXContent(XContentParser parser) throws IOException {
         String snapshot = null;
         long indexVersion = -1;
         long startTime = 0;
@@ -527,24 +522,24 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
                     String currentFieldName = parser.currentName();
                     token = parser.nextToken();
                     if (token.isValue()) {
-                        if (parseFieldMatcher.match(currentFieldName, PARSE_NAME)) {
+                        if (PARSE_NAME.match(currentFieldName)) {
                             snapshot = parser.text();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_INDEX_VERSION)) {
+                        } else if (PARSE_INDEX_VERSION.match(currentFieldName)) {
                             // The index-version is needed for backward compatibility with v 1.0
                             indexVersion = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_START_TIME)) {
+                        } else if (PARSE_START_TIME.match(currentFieldName)) {
                             startTime = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TIME)) {
+                        } else if (PARSE_TIME.match(currentFieldName)) {
                             time = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_NUMBER_OF_FILES)) {
+                        } else if (PARSE_NUMBER_OF_FILES.match(currentFieldName)) {
                             numberOfFiles = parser.intValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TOTAL_SIZE)) {
+                        } else if (PARSE_TOTAL_SIZE.match(currentFieldName)) {
                             totalSize = parser.longValue();
                         } else {
                             throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
                         }
                     } else if (token == XContentParser.Token.START_ARRAY) {
-                        if (parseFieldMatcher.match(currentFieldName, PARSE_FILES)) {
+                        if (PARSE_FILES.match(currentFieldName)) {
                             while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                                 indexFiles.add(FileInfo.fromXContent(parser));
                             }
@@ -562,5 +557,4 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
         return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles),
                                                startTime, time, numberOfFiles, totalSize);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
index 5b66d9b6f6f..359c3165f53 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
@@ -21,8 +21,6 @@ package org.elasticsearch.index.snapshots.blobstore;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -44,9 +42,7 @@ import static java.util.Collections.unmodifiableMap;
  * This class is used to find files that were already snapshotted and clear out files that no longer referenced by any
  * snapshots
  */
-public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContent, FromXContentBuilder<BlobStoreIndexShardSnapshots> {
-
-    public static final BlobStoreIndexShardSnapshots PROTO = new BlobStoreIndexShardSnapshots();
+public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContent {
 
     private final List<SnapshotFiles> shardSnapshots;
     private final Map<String, FileInfo> files;
@@ -232,8 +228,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
         return builder;
     }
 
-    @Override
-    public BlobStoreIndexShardSnapshots fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static BlobStoreIndexShardSnapshots fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         if (token == null) { // New parser
             token = parser.nextToken();
@@ -248,7 +243,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
                 String currentFieldName = parser.currentName();
                 token = parser.nextToken();
                 if (token == XContentParser.Token.START_ARRAY) {
-                    if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES) == false) {
+                    if (ParseFields.FILES.match(currentFieldName) == false) {
                         throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
                     }
                     while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
@@ -256,7 +251,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
                         files.put(fileInfo.name(), fileInfo);
                     }
                 } else if (token == XContentParser.Token.START_OBJECT) {
-                    if (parseFieldMatcher.match(currentFieldName, ParseFields.SNAPSHOTS) == false) {
+                    if (ParseFields.SNAPSHOTS.match(currentFieldName) == false) {
                         throw new ElasticsearchParseException("unknown object [{}]", currentFieldName);
                     }
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -271,7 +266,7 @@ public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, To
                             if (token == XContentParser.Token.FIELD_NAME) {
                                 currentFieldName = parser.currentName();
                                 if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
-                                    if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES) == false) {
+                                    if (ParseFields.FILES.match(currentFieldName) == false) {
                                         throw new ElasticsearchParseException("unknown array [{}]", currentFieldName);
                                     }
                                     List<String> fileNames = new ArrayList<>();
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index b6bc8c60815..bf8e8466dae 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -29,7 +29,6 @@ import org.apache.lucene.store.NativeFSLockFactory;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.store.SimpleFSLockFactory;
 import org.apache.lucene.store.SleepingLockWrapper;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -75,7 +74,7 @@ public class FsDirectoryService extends DirectoryService {
         Set<String> preLoadExtensions = new HashSet<>(
                 indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
         wrapped = setPreload(wrapped, location, lockFactory, preLoadExtensions);
-        if (IndexMetaData.isOnSharedFilesystem(indexSettings.getSettings())) {
+        if (indexSettings.isOnSharedFilesystem()) {
             wrapped = new SleepingLockWrapper(wrapped, 5000);
         }
         return wrapped;
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index fcc079d7909..d4761dccbb6 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -24,6 +24,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexFormatTooNewException;
@@ -75,6 +76,8 @@ import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.env.ShardLockObtainFailedException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -207,6 +210,20 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
 
     }
 
+    /**
+     * Loads the local checkpoint and the maximum sequence number from the latest Lucene commit point and returns the triplet of local and
+     * global checkpoints, and maximum sequence number as an instance of {@link SeqNoStats}. The global checkpoint must be provided
+     * externally as it is not stored in the commit point.
+     *
+     * @param globalCheckpoint the provided global checkpoint
+     * @return an instance of {@link SeqNoStats} populated with the local and global checkpoints, and the maximum sequence number
+     * @throws IOException if an I/O exception occurred reading the latest Lucene commit point from disk
+     */
+    public SeqNoStats loadSeqNoStats(final long globalCheckpoint) throws IOException {
+        final Map<String, String> userData = SegmentInfos.readLatestCommit(directory).getUserData();
+        return SequenceNumbers.loadSeqNoStatsFromLuceneCommit(globalCheckpoint, userData.entrySet());
+    }
+
     final void ensureOpen() {
         if (this.refCounter.refCount() <= 0) {
             throw new AlreadyClosedException("store is already closed");
@@ -841,7 +858,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                 Logger logger, Version version, boolean readFileAsHash) throws IOException {
             final String checksum;
             final BytesRefBuilder fileHash = new BytesRefBuilder();
-            try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) {
+            try (IndexInput in = directory.openInput(file, IOContext.READONCE)) {
                 final long length;
                 try {
                     length = in.length();
@@ -1177,11 +1194,11 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         private final byte[] checksum = new byte[8];
         private long verifiedPosition = 0;
 
-        public VerifyingIndexInput(IndexInput input) {
+        VerifyingIndexInput(IndexInput input) {
             this(input, new BufferedChecksum(new CRC32()));
         }
 
-        public VerifyingIndexInput(IndexInput input, Checksum digest) {
+        VerifyingIndexInput(IndexInput input, Checksum digest) {
             super("VerifyingIndexInput(" + input + ")");
             this.input = input;
             this.digest = digest;
@@ -1349,7 +1366,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
     private static class StoreStatsCache extends SingleObjectCache<StoreStats> {
         private final Directory directory;
 
-        public StoreStatsCache(TimeValue refreshInterval, Directory directory) throws IOException {
+        StoreStatsCache(TimeValue refreshInterval, Directory directory) throws IOException {
             super(refreshInterval, new StoreStats(estimateSize(directory)));
             this.directory = directory;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 520cb13390f..7981c891242 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetField;
@@ -270,7 +271,8 @@ public class TermVectorsService  {
 
     private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
         // parse the document, at the moment we do update the mapping, just like percolate
-        ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc());
+        ParsedDocument parsedDocument =
+            parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType());
 
         // select the right fields and generate term vectors
         ParseContext.Document doc = parsedDocument.rootDoc();
@@ -293,13 +295,15 @@ public class TermVectorsService  {
             String[] values = doc.getValues(field.name());
             getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
         }
-        return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true).v2(), getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
+        return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(),
+            getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
-    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) {
+    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc,
+                                                XContentType xContentType) {
         MapperService mapperService = indexShard.mapperService();
         DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
-        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc));
+        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc, xContentType));
         if (docMapper.getMapping() != null) {
             parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
         }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
index 932bf24a0ac..6f5d7fa8414 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
@@ -106,7 +106,7 @@ class Checkpoint {
 
     public static Checkpoint read(Path path) throws IOException {
         try (Directory dir = new SimpleFSDirectory(path.getParent())) {
-            try (final IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
+            try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
                 if (indexInput.length() == LEGACY_NON_CHECKSUMMED_FILE_LENGTH) {
                     // OLD unchecksummed file that was written < ES 5.0.0
                     return Checkpoint.readNonChecksummed(indexInput);
@@ -136,7 +136,7 @@ class Checkpoint {
             }
         };
         final String resourceDesc = "checkpoint(path=\"" + checkpointFile + "\", gen=" + checkpoint + ")";
-        try (final OutputStreamIndexOutput indexOutput =
+        try (OutputStreamIndexOutput indexOutput =
                  new OutputStreamIndexOutput(resourceDesc, checkpointFile.toString(), byteOutputStream, FILE_SIZE)) {
             CodecUtil.writeHeader(indexOutput, CHECKPOINT_CODEC, CURRENT_VERSION);
             checkpoint.write(indexOutput);
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index f7560960660..3e650638fec 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -464,7 +464,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
      * @return the last synced checkpoint
      */
     public long getLastSyncedGlobalCheckpoint() {
-        try (final ReleasableLock ignored = readLock.acquire()) {
+        try (ReleasableLock ignored = readLock.acquire()) {
             return current.getLastSyncedCheckpoint().globalCheckpoint;
         }
     }
@@ -743,7 +743,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             @Deprecated
             CREATE((byte) 1),
             INDEX((byte) 2),
-            DELETE((byte) 3);
+            DELETE((byte) 3),
+            NO_OP((byte) 4);
 
             private final byte id;
 
@@ -763,6 +764,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                         return INDEX;
                     case 3:
                         return DELETE;
+                    case 4:
+                        return NO_OP;
                     default:
                         throw new IllegalArgumentException("No type mapped for [" + id + "]");
                 }
@@ -775,6 +778,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
         Source getSource();
 
+        long seqNo();
+
         /**
          * Reads the type and the operation from the given stream. The operatino must be written with
          * {@link Operation#writeType(Operation, StreamOutput)}
@@ -786,9 +791,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                     // the deserialization logic in Index was identical to that of Create when create was deprecated
                     return new Index(input);
                 case DELETE:
-                    return new Translog.Delete(input);
+                    return new Delete(input);
                 case INDEX:
                     return new Index(input);
+                case NO_OP:
+                    return new NoOp(input);
                 default:
                     throw new IOException("No type for [" + type + "]");
             }
@@ -805,6 +812,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     }
 
     public static class Source {
+
         public final BytesReference source;
         public final String routing;
         public final String parent;
@@ -814,13 +822,16 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             this.routing = routing;
             this.parent = parent;
         }
+
     }
 
     public static class Index implements Operation {
-        public static final int FORMAT_2x = 6; // since 2.0-beta1 and 1.1
-        public static final int FORMAT_AUTO_GENERATED_IDS = 7; // since 5.0.0-beta1
+
+        public static final int FORMAT_2_X = 6; // since 2.0-beta1 and 1.1
+        public static final int FORMAT_AUTO_GENERATED_IDS = FORMAT_2_X + 1; // since 5.0.0-beta1
         public static final int FORMAT_SEQ_NO = FORMAT_AUTO_GENERATED_IDS + 1; // since 6.0.0
         public static final int SERIALIZATION_FORMAT = FORMAT_SEQ_NO;
+
         private final String id;
         private final long autoGeneratedIdTimestamp;
         private final String type;
@@ -834,7 +845,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
         public Index(StreamInput in) throws IOException {
             final int format = in.readVInt(); // SERIALIZATION_FORMAT
-            assert format >= FORMAT_2x : "format was: " + format;
+            assert format >= FORMAT_2_X : "format was: " + format;
             id = in.readString();
             type = in.readString();
             source = in.readBytesReference();
@@ -846,15 +857,15 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 in.readLong(); // ttl
             }
             this.versionType = VersionType.fromValue(in.readByte());
-            assert versionType.validateVersionForWrites(this.version);
+            assert versionType.validateVersionForWrites(this.version) : "invalid version for writes: " + this.version;
             if (format >= FORMAT_AUTO_GENERATED_IDS) {
                 this.autoGeneratedIdTimestamp = in.readLong();
             } else {
                 this.autoGeneratedIdTimestamp = IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP;
             }
             if (format >= FORMAT_SEQ_NO) {
-                seqNo = in.readVLong();
-                primaryTerm = in.readVLong();
+                seqNo = in.readLong();
+                primaryTerm = in.readLong();
             }
         }
 
@@ -913,6 +924,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return this.source;
         }
 
+        @Override
         public long seqNo() {
             return seqNo;
         }
@@ -943,11 +955,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             out.writeOptionalString(routing);
             out.writeOptionalString(parent);
             out.writeLong(version);
-            
+
             out.writeByte(versionType.getValue());
             out.writeLong(autoGeneratedIdTimestamp);
-            out.writeVLong(seqNo);
-            out.writeVLong(primaryTerm);
+            out.writeLong(seqNo);
+            out.writeLong(primaryTerm);
         }
 
         @Override
@@ -1004,10 +1016,12 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         public long getAutoGeneratedIdTimestamp() {
             return autoGeneratedIdTimestamp;
         }
+
     }
 
     public static class Delete implements Operation {
-        private static final int FORMAT_5_X = 3;
+
+        private static final int FORMAT_5_X = 2;
         private static final int FORMAT_SEQ_NO = FORMAT_5_X + 1;
         public static final int SERIALIZATION_FORMAT = FORMAT_SEQ_NO;
 
@@ -1019,14 +1033,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
         public Delete(StreamInput in) throws IOException {
             final int format = in.readVInt();// SERIALIZATION_FORMAT
-            assert format >= SERIALIZATION_FORMAT - 1: "format was: " + format;
+            assert format >= FORMAT_5_X : "format was: " + format;
             uid = new Term(in.readString(), in.readString());
             this.version = in.readLong();
             this.versionType = VersionType.fromValue(in.readByte());
             assert versionType.validateVersionForWrites(this.version);
             if (format >= FORMAT_SEQ_NO) {
-                seqNo = in.readVLong();
-                primaryTerm = in.readVLong();
+                seqNo = in.readLong();
+                primaryTerm = in.readLong();
             }
         }
 
@@ -1061,6 +1075,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return this.uid;
         }
 
+        @Override
         public long seqNo() {
             return seqNo;
         }
@@ -1089,8 +1104,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             out.writeString(uid.text());
             out.writeLong(version);
             out.writeByte(versionType.getValue());
-            out.writeVLong(seqNo);
-            out.writeVLong(primaryTerm);
+            out.writeLong(seqNo);
+            out.writeLong(primaryTerm);
         }
 
         @Override
@@ -1127,6 +1142,82 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                     "uid=" + uid +
                     '}';
         }
+
+    }
+
+    public static class NoOp implements Operation {
+
+        private final long seqNo;
+        private final long primaryTerm;
+        private final String reason;
+
+        @Override
+        public long seqNo() {
+            return seqNo;
+        }
+
+        public long primaryTerm() {
+            return primaryTerm;
+        }
+
+        public String reason() {
+            return reason;
+        }
+
+        NoOp(final StreamInput in) throws IOException {
+            seqNo = in.readLong();
+            primaryTerm = in.readLong();
+            reason = in.readString();
+        }
+
+        public NoOp(final long seqNo, final long primaryTerm, final String reason) {
+            assert seqNo > SequenceNumbersService.NO_OPS_PERFORMED;
+            assert primaryTerm >= 0;
+            assert reason != null;
+            this.seqNo = seqNo;
+            this.primaryTerm = primaryTerm;
+            this.reason = reason;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeLong(seqNo);
+            out.writeLong(primaryTerm);
+            out.writeString(reason);
+        }
+
+        @Override
+        public Type opType() {
+            return Type.NO_OP;
+        }
+
+        @Override
+        public long estimateSize() {
+            return 2 * reason.length() + 2 * Long.BYTES;
+        }
+
+        @Override
+        public Source getSource() {
+            throw new UnsupportedOperationException("source does not exist for a no-op");
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            final NoOp that = (NoOp) obj;
+            return seqNo == that.seqNo && primaryTerm == that.primaryTerm && reason.equals(that.reason);
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * 31 * 31 + 31 * 31 * Long.hashCode(seqNo) + 31 * Long.hashCode(primaryTerm) + reason().hashCode();
+        }
+
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
index a08259ef32d..ffbe1002eb1 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
@@ -40,7 +40,7 @@ final class TranslogSnapshot extends BaseTranslogReader implements Translog.Snap
      * Create a snapshot of translog file channel. The length parameter should be consistent with totalOperations and point
      * at the end of the last operation in this snapshot.
      */
-    public TranslogSnapshot(long generation, FileChannel channel, Path path, long firstOperationOffset, long length, int totalOperations) {
+    TranslogSnapshot(long generation, FileChannel channel, Path path, long firstOperationOffset, long length, int totalOperations) {
         super(generation, channel, path, firstOperationOffset);
         this.length = length;
         this.totalOperations = totalOperations;
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
index 3b77466a916..944296d6813 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogToolCli.java
@@ -21,10 +21,6 @@ package org.elasticsearch.index.translog;
 
 import org.elasticsearch.cli.MultiCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 /**
  * Class encapsulating and dispatching commands from the {@code elasticsearch-translog} command line tool
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 6870643d20b..785566b976c 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -374,7 +374,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
 
     private final class BufferedChannelOutputStream extends BufferedOutputStream {
 
-        public BufferedChannelOutputStream(OutputStream out, int size) throws IOException {
+        BufferedChannelOutputStream(OutputStream out, int size) throws IOException {
             super(out, size);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java b/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
index 967876348c8..40a75c16370 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TruncateTranslogCommand.java
@@ -34,10 +34,11 @@ import org.apache.lucene.store.OutputStreamDataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 
@@ -55,7 +56,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-public class TruncateTranslogCommand extends SettingCommand {
+public class TruncateTranslogCommand extends EnvironmentAwareCommand {
 
     private final OptionSpec<String> translogFolder;
     private final OptionSpec<Void> batchMode;
@@ -87,7 +88,7 @@ public class TruncateTranslogCommand extends SettingCommand {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         boolean batch = options.has(batchMode);
 
         Path translogPath = getTranslogPath(options);
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 11dbfb36f4f..1b960bb1599 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -21,6 +21,7 @@ package org.elasticsearch.indices;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -30,10 +31,10 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPool.Cancellable;
 import org.elasticsearch.threadpool.ThreadPool.Names;
@@ -200,12 +201,12 @@ public class IndexingMemoryController extends AbstractComponent implements Index
     }
 
     @Override
-    public void postIndex(Engine.Index index, Engine.IndexResult result) {
+    public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
         recordOperationBytes(index, result);
     }
 
     @Override
-    public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+    public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
         recordOperationBytes(delete, result);
     }
 
@@ -220,7 +221,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
         final long bytesUsed;
         final IndexShard shard;
 
-        public ShardAndBytesUsed(long bytesUsed, IndexShard shard) {
+        ShardAndBytesUsed(long bytesUsed, IndexShard shard) {
             this.bytesUsed = bytesUsed;
             this.shard = shard;
         }
@@ -383,7 +384,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
     protected void checkIdle(IndexShard shard, long inactiveTimeNS) {
         try {
             shard.checkIdle(inactiveTimeNS);
-        } catch (EngineClosedException e) {
+        } catch (AlreadyClosedException e) {
             logger.trace((Supplier<?>) () -> new ParameterizedMessage("ignore exception while checking if shard {} is inactive", shard.shardId()), e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 0fcda5c8fd5..e7bd76fb34d 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -130,6 +130,16 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
         return value;
     }
 
+    /**
+     * Invalidates the given the cache entry for the given key and it's context
+     * @param cacheEntity the cache entity to invalidate for
+     * @param reader the reader to invalidate the cache entry for
+     * @param cacheKey the cache key to invalidate
+     */
+    void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) {
+        cache.invalidate(new Key(cacheEntity, reader.getVersion(), cacheKey));
+    }
+
     private static class Loader implements CacheLoader<Key, BytesReference> {
 
         private final CacheEntity entity;
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index a3361f6b2ed..69ecdb5e54a 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.indices;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.index.DirectoryReader;
@@ -40,12 +39,11 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -68,6 +66,8 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
@@ -88,6 +88,7 @@ import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -103,7 +104,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.PluginsService;
@@ -133,7 +133,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
@@ -151,9 +150,9 @@ public class IndicesService extends AbstractLifecycleComponent
         Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope);
     private final PluginsService pluginsService;
     private final NodeEnvironment nodeEnv;
+    private final NamedXContentRegistry xContentRegistry;
     private final TimeValue shardsClosedTimeout;
     private final AnalysisRegistry analysisRegistry;
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopeSetting;
     private final IndicesFieldDataCache indicesFieldDataCache;
@@ -182,9 +181,9 @@ public class IndicesService extends AbstractLifecycleComponent
         threadPool.schedule(this.cleanInterval, ThreadPool.Names.SAME, this.cacheCleaner);
     }
 
-    public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv,
+    public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv, NamedXContentRegistry xContentRegistry,
                           ClusterSettings clusterSettings, AnalysisRegistry analysisRegistry,
-                          IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver,
+                          IndexNameExpressionResolver indexNameExpressionResolver,
                           MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry,
                           ThreadPool threadPool, IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService,
                           BigArrays bigArrays, ScriptService scriptService, ClusterService clusterService, Client client,
@@ -193,9 +192,9 @@ public class IndicesService extends AbstractLifecycleComponent
         this.threadPool = threadPool;
         this.pluginsService = pluginsService;
         this.nodeEnv = nodeEnv;
+        this.xContentRegistry = xContentRegistry;
         this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS));
         this.analysisRegistry = analysisRegistry;
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indicesRequestCache = new IndicesRequestCache(settings);
         this.indicesQueryCache = new IndicesQueryCache(settings);
@@ -441,12 +440,12 @@ public class IndicesService extends AbstractLifecycleComponent
         }
         return indexModule.newIndexService(
             nodeEnv,
+            xContentRegistry,
             this,
             circuitBreakerService,
             bigArrays,
             threadPool,
             scriptService,
-            indicesQueriesRegistry,
             clusterService,
             client,
             indicesQueryCache,
@@ -465,7 +464,7 @@ public class IndicesService extends AbstractLifecycleComponent
         final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopeSetting);
         final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry);
         pluginsService.onIndexModule(indexModule);
-        return indexModule.newIndexMapperService(mapperRegistry);
+        return indexModule.newIndexMapperService(xContentRegistry, mapperRegistry);
     }
 
     /**
@@ -485,11 +484,7 @@ public class IndicesService extends AbstractLifecycleComponent
             final IndexService service =
                 createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList(), s -> {});
             closeables.add(() -> service.close("metadata verification", false));
-            for (ObjectCursor<MappingMetaData> typeMapping : metaData.getMappings().values()) {
-                // don't apply the default mapping, it has been applied when the mapping was created
-                service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
-                    MapperService.MergeReason.MAPPING_RECOVERY, true);
-            }
+            service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
             if (metaData.equals(metaDataUpdate) == false) {
                 service.updateMetaData(metaDataUpdate);
             }
@@ -608,7 +603,7 @@ public class IndicesService extends AbstractLifecycleComponent
                                                     "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
                 }
                 deleteIndexStore(reason, metaData, clusterState);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete unassigned index (reason [{}])", metaData.getIndex(), reason), e);
             }
         }
@@ -770,14 +765,14 @@ public class IndicesService extends AbstractLifecycleComponent
             final IndexMetaData metaData;
             try {
                 metaData = metaStateService.loadIndexState(index);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to load state file from a stale deleted index, folders will be left on disk", index), e);
                 return null;
             }
             final IndexSettings indexSettings = buildIndexSettings(metaData);
             try {
                 deleteIndexStoreIfDeletionAllowed("stale deleted index", index, indexSettings, ALWAYS_TRUE);
-            } catch (IOException e) {
+            } catch (Exception e) {
                 // we just warn about the exception here because if deleteIndexStoreIfDeletionAllowed
                 // throws an exception, it gets added to the list of pending deletes to be tried again
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete index on disk", metaData.getIndex()), e);
@@ -885,7 +880,7 @@ public class IndicesService extends AbstractLifecycleComponent
         /**
          * Creates a new pending delete of an index
          */
-        public PendingDelete(ShardId shardId, IndexSettings settings) {
+        PendingDelete(ShardId shardId, IndexSettings settings) {
             this.index = shardId.getIndex();
             this.shardId = shardId.getId();
             this.settings = settings;
@@ -895,7 +890,7 @@ public class IndicesService extends AbstractLifecycleComponent
         /**
          * Creates a new pending delete of a shard
          */
-        public PendingDelete(Index index, IndexSettings settings) {
+        PendingDelete(Index index, IndexSettings settings) {
             this.index = index;
             this.shardId = -1;
             this.settings = settings;
@@ -1016,13 +1011,6 @@ public class IndicesService extends AbstractLifecycleComponent
         return numUncompletedDeletes.get() > 0;
     }
 
-    /**
-     * Returns this nodes {@link IndicesQueriesRegistry}
-     */
-    public IndicesQueriesRegistry getIndicesQueryRegistry() {
-        return indicesQueriesRegistry;
-    }
-
     public AnalysisRegistry getAnalysis() {
         return analysisRegistry;
     }
@@ -1042,7 +1030,7 @@ public class IndicesService extends AbstractLifecycleComponent
         private final AtomicBoolean closed = new AtomicBoolean(false);
         private final IndicesRequestCache requestCache;
 
-        public CacheCleaner(IndicesFieldDataCache cache, IndicesRequestCache requestCache, Logger logger, ThreadPool threadPool, TimeValue interval) {
+        CacheCleaner(IndicesFieldDataCache cache, IndicesRequestCache requestCache, Logger logger, ThreadPool threadPool, TimeValue interval) {
             this.cache = cache;
             this.requestCache = requestCache;
             this.logger = logger;
@@ -1146,17 +1134,28 @@ public class IndicesService extends AbstractLifecycleComponent
             queryPhase.execute(context);
             try {
                 context.queryResult().writeToNoId(out);
+
             } catch (IOException e) {
                 throw new AssertionError("Could not serialize response", e);
             }
             loadedFromCache[0] = false;
         });
+
         if (loadedFromCache[0]) {
             // restore the cached query result into the context
             final QuerySearchResult result = context.queryResult();
             StreamInput in = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), namedWriteableRegistry);
             result.readFromWithId(context.id(), in);
             result.shardTarget(context.shardTarget());
+        } else if (context.queryResult().searchTimedOut()) {
+            // we have to invalidate the cache entry if we cached a query result form a request that timed out.
+            // we can't really throw exceptions in the loading part to signal a timed out search to the outside world since if there are
+            // multiple requests that wait for the cache entry to be calculated they'd fail all with the same exception.
+            // instead we all caching such a result for the time being, return the timed out result for all other searches with that cache
+            // key invalidate the result in the thread that caused the timeout. This will end up to be simpler and eventually correct since
+            // running a search that times out concurrently will likely timeout again if it's run while we have this `stale` result in the
+            // cache. One other option is to not cache requests with a timeout at all...
+            indicesRequestCache.invalidate(new IndexShardCacheEntity(context.indexShard()), directoryReader, request.cacheKey());
         }
     }
 
@@ -1263,11 +1262,16 @@ public class IndicesService extends AbstractLifecycleComponent
     private final IndexDeletionAllowedPredicate ALWAYS_TRUE = (Index index, IndexSettings indexSettings) -> true;
 
     public AliasFilter buildAliasFilter(ClusterState state, String index, String... expressions) {
-        Function<XContentParser, QueryParseContext> factory =
-            (parser) -> new QueryParseContext(indicesQueriesRegistry, parser, new ParseFieldMatcher(settings));
+        /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch
+         * of dependencies we pass in a function that can perform the parsing. */
+        CheckedFunction<byte[], QueryBuilder, IOException> filterParser = bytes -> {
+            try (XContentParser parser = XContentFactory.xContent(bytes).createParser(xContentRegistry, bytes)) {
+                return new QueryParseContext(parser).parseInnerQueryBuilder();
+            }
+        };
         String[] aliases = indexNameExpressionResolver.filteringAliases(state, index, expressions);
         IndexMetaData indexMetaData = state.metaData().index(index);
-        return new AliasFilter(ShardSearchRequest.parseAliasFilter(factory, indexMetaData, aliases), aliases);
+        return new AliasFilter(ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliases), aliases);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index 89c9421198d..1aaf3077aea 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -60,6 +60,7 @@ import org.elasticsearch.index.analysis.EnglishAnalyzerProvider;
 import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider;
 import org.elasticsearch.index.analysis.FingerprintTokenFilterFactory;
 import org.elasticsearch.index.analysis.FinnishAnalyzerProvider;
+import org.elasticsearch.index.analysis.FlattenGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.FrenchAnalyzerProvider;
 import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.GalicianAnalyzerProvider;
@@ -170,8 +171,9 @@ public final class AnalysisModule {
         NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
         NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
         NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = setupNormalizers(plugins);
         analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(), tokenizers
-            .getRegistry(), analyzers.getRegistry());
+            .getRegistry(), analyzers.getRegistry(), normalizers.getRegistry());
     }
 
     HunspellService getHunspellService() {
@@ -225,6 +227,7 @@ public final class AnalysisModule {
         tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new);
         tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
         tokenFilters.register("elision", ElisionTokenFilterFactory::new);
+        tokenFilters.register("flatten_graph", FlattenGraphTokenFilterFactory::new);
         tokenFilters.register("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
         tokenFilters.register("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
         tokenFilters.register("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
@@ -334,6 +337,13 @@ public final class AnalysisModule {
         return analyzers;
     }
 
+    private NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> setupNormalizers(List<AnalysisPlugin> plugins) {
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = new NamedRegistry<>("normalizer");
+        // TODO: provide built-in normalizer providers?
+        // TODO: pluggability?
+        return normalizers;
+    }
+
     private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
         return new AnalysisModule.AnalysisProvider<T>() {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index de31a57283b..88cff7f559a 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -208,10 +208,11 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
         long parentLimit = this.parentSettings.getLimit();
         if (totalUsed > parentLimit) {
             this.parentTripCount.incrementAndGet();
-            throw new CircuitBreakingException("[parent] Data too large, data for [" +
-                            label + "] would be larger than limit of [" +
-                            parentLimit + "/" + new ByteSizeValue(parentLimit) + "]",
-                    totalUsed, parentLimit);
+            final String message = "[parent] Data too large, data for [" + label + "]" +
+                    " would be [" + totalUsed + "/" + new ByteSizeValue(totalUsed) + "]" +
+                    ", which is larger than the limit of [" +
+                    parentLimit + "/" + new ByteSizeValue(parentLimit) + "]";
+            throw new CircuitBreakingException(message, totalUsed, parentLimit);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index a49c2a97cb0..2307a711714 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -24,6 +24,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
@@ -52,7 +53,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexComponent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.seqno.GlobalCheckpointService;
+import org.elasticsearch.index.seqno.GlobalCheckpointTracker;
 import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
@@ -561,9 +562,15 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
             shard.updateRoutingEntry(shardRouting);
             if (shardRouting.primary()) {
                 IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId());
-                Set<String> activeIds = indexShardRoutingTable.activeShards().stream().map(r -> r.allocationId().getId())
+                Set<String> activeIds = indexShardRoutingTable.activeShards().stream()
+                    // filter to shards that track seq# and should be taken into consideration for checkpoint tracking
+                    // shards on old nodes will go through a file based recovery which will also transfer seq# information.
+                    .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))
+                    .map(r -> r.allocationId().getId())
                     .collect(Collectors.toSet());
-                Set<String> initializingIds = indexShardRoutingTable.getAllInitializingShards().stream().map(r -> r.allocationId().getId())
+                Set<String> initializingIds = indexShardRoutingTable.getAllInitializingShards().stream()
+                    .filter(sr -> nodes.get(sr.currentNodeId()).getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED))
+                    .map(r -> r.allocationId().getId())
                     .collect(Collectors.toSet());
                shard.updateAllocationIdsFromMaster(activeIds, initializingIds);
             }
@@ -732,7 +739,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
         /**
          * Notifies the service of the current allocation ids in the cluster state.
-         * See {@link GlobalCheckpointService#updateAllocationIdsFromMaster(Set, Set)} for details.
+         * See {@link GlobalCheckpointTracker#updateAllocationIdsFromMaster(Set, Set)} for details.
          *
          * @param activeAllocationIds       the allocation ids of the currently active shard copies
          * @param initializingAllocationIds the allocation ids of the currently initializing shard copies
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 273682db312..3dce0ecdfd4 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -658,10 +658,10 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
 
         int opCount;
 
-        public InFlightOpsResponse() {
+        InFlightOpsResponse() {
         }
 
-        public InFlightOpsResponse(int opCount) {
+        InFlightOpsResponse(int opCount) {
             this.opCount = opCount;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
index 6a14681a164..e2113957690 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoverySourceService.java
@@ -199,10 +199,10 @@ public class PeerRecoverySourceService extends AbstractComponent implements Inde
                 Supplier<Long> currentClusterStateVersionSupplier = () -> clusterService.state().getVersion();
                 if (shard.indexSettings().isOnSharedFilesystem()) {
                     handler = new SharedFSRecoverySourceHandler(shard, recoveryTarget, request, currentClusterStateVersionSupplier,
-                        this::delayNewRecoveries, logger);
+                        this::delayNewRecoveries, settings);
                 } else {
                     handler = new RecoverySourceHandler(shard, recoveryTarget, request, currentClusterStateVersionSupplier,
-                        this::delayNewRecoveries, recoverySettings.getChunkSize().bytesAsInt(), logger);
+                        this::delayNewRecoveries, recoverySettings.getChunkSize().bytesAsInt(), settings);
                 }
                 return handler;
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
index 5f59af04b50..70cf722f761 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetService.java
@@ -41,6 +41,8 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.RecoveryEngineException;
 import org.elasticsearch.index.mapper.MapperException;
+import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.IllegalIndexShardStateException;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
@@ -48,6 +50,8 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.recovery.RecoveriesCollection.RecoveryRef;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -123,11 +127,11 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
     }
 
     /**
-     * cancel all ongoing recoveries for the given shard, if their status match a predicate
+     * Cancel all ongoing recoveries for the given shard.
      *
-     * @param reason       reason for cancellation
-     * @param shardId      shardId for which to cancel recoveries
-     * @return true if a recovery was cancelled
+     * @param reason  reason for cancellation
+     * @param shardId shard ID for which to cancel recoveries
+     * @return {@code true} if a recovery was cancelled
      */
     public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
         return onGoingRecoveries.cancelRecoveriesForShard(shardId, reason);
@@ -139,65 +143,56 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
         threadPool.generic().execute(new RecoveryRunner(recoveryId));
     }
 
-    protected void retryRecovery(final RecoveryTarget recoveryTarget, final Throwable reason, TimeValue retryAfter, final
-    StartRecoveryRequest currentRequest) {
+    protected void retryRecovery(final long recoveryId, final Throwable reason, TimeValue retryAfter, TimeValue activityTimeout) {
         logger.trace(
             (Supplier<?>) () -> new ParameterizedMessage(
-                "will retry recovery with id [{}] in [{}]", recoveryTarget.recoveryId(), retryAfter), reason);
-        retryRecovery(recoveryTarget, retryAfter, currentRequest);
+                "will retry recovery with id [{}] in [{}]", recoveryId, retryAfter), reason);
+        retryRecovery(recoveryId, retryAfter, activityTimeout);
     }
 
-    protected void retryRecovery(final RecoveryTarget recoveryTarget, final String reason, TimeValue retryAfter, final
-    StartRecoveryRequest currentRequest) {
-        logger.trace("will retry recovery with id [{}] in [{}] (reason [{}])", recoveryTarget.recoveryId(), retryAfter, reason);
-        retryRecovery(recoveryTarget, retryAfter, currentRequest);
+    protected void retryRecovery(final long recoveryId, final String reason, TimeValue retryAfter, TimeValue activityTimeout) {
+        logger.trace("will retry recovery with id [{}] in [{}] (reason [{}])", recoveryId, retryAfter, reason);
+        retryRecovery(recoveryId, retryAfter, activityTimeout);
     }
 
-    private void retryRecovery(final RecoveryTarget recoveryTarget, TimeValue retryAfter, final StartRecoveryRequest currentRequest) {
-        try {
-            onGoingRecoveries.resetRecovery(recoveryTarget.recoveryId(), recoveryTarget.shardId());
-        } catch (Exception e) {
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(currentRequest, e), true);
+    private void retryRecovery(final long recoveryId, final TimeValue retryAfter, final TimeValue activityTimeout) {
+        RecoveryTarget newTarget = onGoingRecoveries.resetRecovery(recoveryId, activityTimeout);
+        if (newTarget != null) {
+            threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(newTarget.recoveryId()));
         }
-        threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(recoveryTarget.recoveryId()));
     }
 
-    private void doRecovery(final RecoveryTarget recoveryTarget) {
-        assert recoveryTarget.sourceNode() != null : "can't do a recovery without a source node";
+    private void doRecovery(final long recoveryId) {
+        final StartRecoveryRequest request;
+        final CancellableThreads cancellableThreads;
+        final RecoveryState.Timer timer;
 
-        logger.trace("collecting local files for {}", recoveryTarget);
-        Store.MetadataSnapshot metadataSnapshot = null;
-        try {
-            if (recoveryTarget.indexShard().indexSettings().isOnSharedFilesystem()) {
-                // we are not going to copy any files, so don't bother listing files, potentially running
-                // into concurrency issues with the primary changing files underneath us.
-                metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-            } else {
-                metadataSnapshot = recoveryTarget.indexShard().snapshotStoreMetadata();
+        try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
+            if (recoveryRef == null) {
+                logger.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
+                return;
+            }
+            final RecoveryTarget recoveryTarget = recoveryRef.target();
+            cancellableThreads = recoveryTarget.cancellableThreads();
+            timer = recoveryTarget.state().getTimer();
+            try {
+                assert recoveryTarget.sourceNode() != null : "can not do a recovery without a source node";
+                request = getStartRecoveryRequest(recoveryTarget);
+                logger.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
+                recoveryTarget.indexShard().prepareForIndexRecovery();
+            } catch (final Exception e) {
+                // this will be logged as warning later on...
+                logger.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
+                onGoingRecoveries.failRecovery(recoveryId,
+                    new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), true);
+                return;
             }
-        } catch (org.apache.lucene.index.IndexNotFoundException e) {
-            // happens on an empty folder. no need to log
-            metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-        } catch (IOException e) {
-            logger.warn("error while listing local files, recover as if there are none", e);
-            metadataSnapshot = Store.MetadataSnapshot.EMPTY;
-        } catch (Exception e) {
-            // this will be logged as warning later on...
-            logger.trace("unexpected error while listing local files, failing recovery", e);
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(),
-                    new RecoveryFailedException(recoveryTarget.state(), "failed to list local files", e), true);
-            return;
         }
-        logger.trace("{} local file count: [{}]", recoveryTarget, metadataSnapshot.size());
-        final StartRecoveryRequest request = new StartRecoveryRequest(recoveryTarget.shardId(), recoveryTarget.sourceNode(),
-                clusterService.localNode(), metadataSnapshot, recoveryTarget.state().getPrimary(), recoveryTarget.recoveryId());
 
-        final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
         try {
-            logger.trace("[{}][{}] starting recovery from {}", request.shardId().getIndex().getName(), request.shardId().id(), request
-                    .sourceNode());
-            recoveryTarget.indexShard().prepareForIndexRecovery();
-            recoveryTarget.CancellableThreads().execute(() -> responseHolder.set(
+            logger.trace("{} starting recovery from {}", request.shardId(), request.sourceNode());
+            final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
+            cancellableThreads.execute(() -> responseHolder.set(
                     transportService.submitRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request,
                             new FutureTransportResponseHandler<RecoveryResponse>() {
                                 @Override
@@ -206,10 +201,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
                                 }
                             }).txGet()));
             final RecoveryResponse recoveryResponse = responseHolder.get();
-            assert responseHolder != null;
-            final TimeValue recoveryTime = new TimeValue(recoveryTarget.state().getTimer().time());
+            final TimeValue recoveryTime = new TimeValue(timer.time());
             // do this through ongoing recoveries to remove it from the collection
-            onGoingRecoveries.markRecoveryAsDone(recoveryTarget.recoveryId());
+            onGoingRecoveries.markRecoveryAsDone(recoveryId);
             if (logger.isTraceEnabled()) {
                 StringBuilder sb = new StringBuilder();
                 sb.append('[').append(request.shardId().getIndex().getName()).append(']').append('[').append(request.shardId().id())
@@ -229,7 +223,7 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
                         .append("\n");
                 logger.trace("{}", sb);
             } else {
-                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), recoveryTarget.sourceNode(), recoveryTime);
+                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
             }
         } catch (CancellableThreads.ExecutionCancelledException e) {
             logger.trace("recovery cancelled", e);
@@ -245,8 +239,8 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
             Throwable cause = ExceptionsHelper.unwrapCause(e);
             if (cause instanceof CancellableThreads.ExecutionCancelledException) {
                 // this can also come from the source wrapped in a RemoteTransportException
-                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source has canceled the" +
-                        " recovery", cause), false);
+                onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request,
+                    "source has canceled the recovery", cause), false);
                 return;
             }
             if (cause instanceof RecoveryEngineException) {
@@ -262,31 +256,129 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
             // here, we would add checks against exception that need to be retried (and not removeAndClean in this case)
 
-            if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof
-                    ShardNotFoundException) {
+            if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException ||
+                cause instanceof ShardNotFoundException) {
                 // if the target is not ready yet, retry
-                retryRecovery(recoveryTarget, "remote shard not ready", recoverySettings.retryDelayStateSync(), request);
+                retryRecovery(
+                    recoveryId,
+                    "remote shard not ready",
+                    recoverySettings.retryDelayStateSync(),
+                    recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof DelayRecoveryException) {
-                retryRecovery(recoveryTarget, cause, recoverySettings.retryDelayStateSync(), request);
+                retryRecovery(recoveryId, cause, recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof ConnectTransportException) {
-                logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", recoveryTarget.shardId(), recoverySettings
-                        .retryDelayNetwork(), cause.getMessage());
-                retryRecovery(recoveryTarget, cause.getMessage(), recoverySettings.retryDelayNetwork(), request);
+                logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", request.shardId(),
+                    recoverySettings.retryDelayNetwork(), cause.getMessage());
+                retryRecovery(recoveryId, cause.getMessage(), recoverySettings.retryDelayNetwork(), recoverySettings.activityTimeout());
                 return;
             }
 
             if (cause instanceof AlreadyClosedException) {
-                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source shard is " +
-                        "closed", cause), false);
+                onGoingRecoveries.failRecovery(recoveryId,
+                    new RecoveryFailedException(request, "source shard is closed", cause), false);
                 return;
             }
-            onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, e), true);
+
+            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, e), true);
+        }
+    }
+
+    /**
+     * Obtains a snapshot of the store metadata for the recovery target.
+     *
+     * @param recoveryTarget the target of the recovery
+     * @return a snapshot of the store metdata
+     */
+    private Store.MetadataSnapshot getStoreMetadataSnapshot(final RecoveryTarget recoveryTarget) {
+        try {
+            if (recoveryTarget.indexShard().indexSettings().isOnSharedFilesystem()) {
+                // we are not going to copy any files, so don't bother listing files, potentially running into concurrency issues with the
+                // primary changing files underneath us
+                return Store.MetadataSnapshot.EMPTY;
+            } else {
+                return recoveryTarget.indexShard().snapshotStoreMetadata();
+            }
+        } catch (final org.apache.lucene.index.IndexNotFoundException e) {
+            // happens on an empty folder. no need to log
+            logger.trace("{} shard folder empty, recovering all files", recoveryTarget);
+            return Store.MetadataSnapshot.EMPTY;
+        } catch (final IOException e) {
+            logger.warn("error while listing local files, recovering as if there are none", e);
+            return Store.MetadataSnapshot.EMPTY;
+        }
+    }
+
+    /**
+     * Prepare the start recovery request.
+     *
+     * @param recoveryTarget the target of the recovery
+     * @return a start recovery request
+     */
+    private StartRecoveryRequest getStartRecoveryRequest(final RecoveryTarget recoveryTarget) {
+        final StartRecoveryRequest request;
+        logger.trace("{} collecting local files for [{}]", recoveryTarget.shardId(), recoveryTarget.sourceNode());
+
+        final Store.MetadataSnapshot metadataSnapshot = getStoreMetadataSnapshot(recoveryTarget);
+        logger.trace("{} local file count [{}]", recoveryTarget.shardId(), metadataSnapshot.size());
+
+        final long startingSeqNo;
+        if (metadataSnapshot.size() > 0) {
+            startingSeqNo = getStartingSeqNo(recoveryTarget);
+        } else {
+            startingSeqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        }
+
+        if (startingSeqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO) {
+            logger.trace("{} preparing for file-based recovery from [{}]", recoveryTarget.shardId(), recoveryTarget.sourceNode());
+        } else {
+            logger.trace(
+                "{} preparing for sequence-number-based recovery starting at local checkpoint [{}] from [{}]",
+                recoveryTarget.shardId(),
+                startingSeqNo,
+                recoveryTarget.sourceNode());
+        }
+
+        request = new StartRecoveryRequest(
+            recoveryTarget.shardId(),
+            recoveryTarget.sourceNode(),
+            clusterService.localNode(),
+            metadataSnapshot,
+            recoveryTarget.state().getPrimary(),
+            recoveryTarget.recoveryId(),
+            startingSeqNo);
+        return request;
+    }
+
+    /**
+     * Get the starting sequence number for a sequence-number-based request.
+     *
+     * @param recoveryTarget the target of the recovery
+     * @return the starting sequence number or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO} if obtaining the starting sequence number
+     * failed
+     */
+    public static long getStartingSeqNo(final RecoveryTarget recoveryTarget) {
+        try {
+            final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.indexShard().shardPath().resolveTranslog());
+            final SeqNoStats seqNoStats = recoveryTarget.store().loadSeqNoStats(globalCheckpoint);
+            if (seqNoStats.getMaxSeqNo() <= seqNoStats.getGlobalCheckpoint()) {
+                // commit point is good for seq no based recovery as the maximum seq# including in it
+                // is below the global checkpoint (i.e., it excludes any ops thay may not be on the primary)
+                // Recovery will start at the first op after the local check point stored in the commit.
+                return seqNoStats.getLocalCheckpoint() + 1;
+            } else {
+                return SequenceNumbersService.UNASSIGNED_SEQ_NO;
+            }
+        } catch (final IOException e) {
+            // this can happen, for example, if a phase one of the recovery completed successfully, a network partition happens before the
+            // translog on the recovery target is opened, the recovery enters a retry loop seeing now that the index files are on disk and
+            // proceeds to attempt a sequence-number-based recovery
+            return SequenceNumbersService.UNASSIGNED_SEQ_NO;
         }
     }
 
@@ -300,9 +392,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(RecoveryPrepareForTranslogOperationsRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().prepareForTranslogOperations(request.totalTranslogOps(), request.getMaxUnsafeAutoIdTimestamp());
+                recoveryRef.target().prepareForTranslogOperations(request.totalTranslogOps(), request.getMaxUnsafeAutoIdTimestamp());
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -312,9 +404,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(RecoveryFinalizeRecoveryRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef =
+            try (RecoveryRef recoveryRef =
                      onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
-                recoveryRef.status().finalizeRecovery(request.globalCheckpoint());
+                recoveryRef.target().finalizeRecovery(request.globalCheckpoint());
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -324,9 +416,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(RecoveryWaitForClusterStateRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().ensureClusterStateVersion(request.clusterStateVersion());
+                recoveryRef.target().ensureClusterStateVersion(request.clusterStateVersion());
             }
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
@@ -336,10 +428,10 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(final RecoveryTranslogOperationsRequest request, final TransportChannel channel) throws IOException {
-            try (RecoveriesCollection.RecoveryRef recoveryRef =
+            try (RecoveryRef recoveryRef =
                          onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
                 final ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
-                final RecoveryTarget recoveryTarget = recoveryRef.status();
+                final RecoveryTarget recoveryTarget = recoveryRef.target();
                 try {
                     recoveryTarget.indexTranslogOperations(request.operations(), request.totalTranslogOps());
                     channel.sendResponse(TransportResponse.Empty.INSTANCE);
@@ -397,21 +489,21 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
     }
 
     private void waitForClusterState(long clusterStateVersion) {
-        ClusterStateObserver observer = new ClusterStateObserver(clusterService, TimeValue.timeValueMinutes(5), logger,
+        final ClusterState clusterState = clusterService.state();
+        ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, TimeValue.timeValueMinutes(5), logger,
             threadPool.getThreadContext());
-        final ClusterState clusterState = observer.observedState();
         if (clusterState.getVersion() >= clusterStateVersion) {
             logger.trace("node has cluster state with version higher than {} (current: {})", clusterStateVersion,
                 clusterState.getVersion());
             return;
         } else {
             logger.trace("waiting for cluster state version {} (current: {})", clusterStateVersion, clusterState.getVersion());
-            final PlainActionFuture<Void> future = new PlainActionFuture<>();
+            final PlainActionFuture<Long> future = new PlainActionFuture<>();
             observer.waitForNextChange(new ClusterStateObserver.Listener() {
 
                 @Override
                 public void onNewClusterState(ClusterState state) {
-                    future.onResponse(null);
+                    future.onResponse(state.getVersion());
                 }
 
                 @Override
@@ -425,15 +517,14 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
                 }
             }, newState -> newState.getVersion() >= clusterStateVersion);
             try {
-                future.get();
-                logger.trace("successfully waited for cluster state with version {} (current: {})", clusterStateVersion,
-                    observer.observedState().getVersion());
+                long currentVersion = future.get();
+                logger.trace("successfully waited for cluster state with version {} (current: {})", clusterStateVersion, currentVersion);
             } catch (Exception e) {
                 logger.debug(
                     (Supplier<?>) () -> new ParameterizedMessage(
                         "failed waiting for cluster state with version {} (current: {})",
                         clusterStateVersion,
-                        observer.observedState().getVersion()),
+                        clusterService.state().getVersion()),
                     e);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
@@ -444,9 +535,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().receiveFileInfo(request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames,
+                recoveryRef.target().receiveFileInfo(request.phase1FileNames, request.phase1FileSizes, request.phase1ExistingFileNames,
                         request.phase1ExistingFileSizes, request.totalTranslogOps);
                 channel.sendResponse(TransportResponse.Empty.INSTANCE);
             }
@@ -457,9 +548,9 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                recoveryRef.status().cleanFiles(request.totalTranslogOps(), request.sourceMetaSnapshot());
+                recoveryRef.target().cleanFiles(request.totalTranslogOps(), request.sourceMetaSnapshot());
                 channel.sendResponse(TransportResponse.Empty.INSTANCE);
             }
         }
@@ -472,10 +563,10 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel) throws Exception {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId()
             )) {
-                final RecoveryTarget status = recoveryRef.status();
-                final RecoveryState.Index indexState = status.state().getIndex();
+                final RecoveryTarget recoveryTarget = recoveryRef.target();
+                final RecoveryState.Index indexState = recoveryTarget.state().getIndex();
                 if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) {
                     indexState.addSourceThrottling(request.sourceThrottleTimeInNanos());
                 }
@@ -488,11 +579,11 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
                         bytesSinceLastPause.addAndGet(-bytes);
                         long throttleTimeInNanos = rateLimiter.pause(bytes);
                         indexState.addTargetThrottling(throttleTimeInNanos);
-                        status.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
+                        recoveryTarget.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
                     }
                 }
 
-                status.writeFileChunk(request.metadata(), request.position(), request.content(),
+                recoveryTarget.writeFileChunk(request.metadata(), request.position(), request.content(),
                         request.lastChunk(), request.totalTranslogOps()
                 );
             }
@@ -510,13 +601,13 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void onFailure(Exception e) {
-            try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
+            try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
                 if (recoveryRef != null) {
                     logger.error(
                         (Supplier<?>) () -> new ParameterizedMessage(
                             "unexpected error during recovery [{}], failing shard", recoveryId), e);
                     onGoingRecoveries.failRecovery(recoveryId,
-                            new RecoveryFailedException(recoveryRef.status().state(), "unexpected error", e),
+                            new RecoveryFailedException(recoveryRef.target().state(), "unexpected error", e),
                             true // be safe
                     );
                 } else {
@@ -529,16 +620,7 @@ public class PeerRecoveryTargetService extends AbstractComponent implements Inde
 
         @Override
         public void doRun() {
-            RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId);
-            if (recoveryRef == null) {
-                logger.trace("not running recovery with id [{}] - can't find it (probably finished)", recoveryId);
-                return;
-            }
-            try {
-                doRecovery(recoveryRef.status());
-            } finally {
-                recoveryRef.close();
-            }
+            doRecovery(recoveryId);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 65a48b18e22..4fb2e398e52 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -26,6 +26,7 @@ import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.shard.IndexShard;
@@ -33,9 +34,12 @@ import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 
 /**
  * This class holds a collection of all on going recoveries on the current node (i.e., the node is the target node
@@ -65,36 +69,67 @@ public class RecoveriesCollection {
      */
     public long startRecovery(IndexShard indexShard, DiscoveryNode sourceNode,
                               PeerRecoveryTargetService.RecoveryListener listener, TimeValue activityTimeout) {
-        RecoveryTarget status = new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
-        RecoveryTarget existingStatus = onGoingRecoveries.putIfAbsent(status.recoveryId(), status);
-        assert existingStatus == null : "found two RecoveryStatus instances with the same id";
-        logger.trace("{} started recovery from {}, id [{}]", indexShard.shardId(), sourceNode, status.recoveryId());
-        threadPool.schedule(activityTimeout, ThreadPool.Names.GENERIC,
-                new RecoveryMonitor(status.recoveryId(), status.lastAccessTime(), activityTimeout));
-        return status.recoveryId();
+        RecoveryTarget recoveryTarget = new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
+        startRecoveryInternal(recoveryTarget, activityTimeout);
+        return recoveryTarget.recoveryId();
     }
 
+    private void startRecoveryInternal(RecoveryTarget recoveryTarget, TimeValue activityTimeout) {
+        RecoveryTarget existingTarget = onGoingRecoveries.putIfAbsent(recoveryTarget.recoveryId(), recoveryTarget);
+        assert existingTarget == null : "found two RecoveryStatus instances with the same id";
+        logger.trace("{} started recovery from {}, id [{}]", recoveryTarget.shardId(), recoveryTarget.sourceNode(),
+            recoveryTarget.recoveryId());
+        threadPool.schedule(activityTimeout, ThreadPool.Names.GENERIC,
+                new RecoveryMonitor(recoveryTarget.recoveryId(), recoveryTarget.lastAccessTime(), activityTimeout));
+    }
 
     /**
      * Resets the recovery and performs a recovery restart on the currently recovering index shard
      *
      * @see IndexShard#performRecoveryRestart()
+     * @return newly created RecoveryTarget
      */
-    public void resetRecovery(long id, ShardId shardId) throws IOException {
-        try (RecoveryRef ref = getRecoverySafe(id, shardId)) {
-            // instead of adding complicated state to RecoveryTarget we just flip the
-            // target instance when we reset a recovery, that way we have only one cleanup
-            // path on the RecoveryTarget and are always within the bounds of ref-counting
-            // which is important since we verify files are on disk etc. after we have written them etc.
-            RecoveryTarget status = ref.status();
-            RecoveryTarget resetRecovery = status.resetRecovery();
-            if (onGoingRecoveries.replace(id, status, resetRecovery) == false) {
-                resetRecovery.cancel("replace failed"); // this is important otherwise we leak a reference to the store
-                throw new IllegalStateException("failed to replace recovery target");
+    public RecoveryTarget resetRecovery(final long recoveryId, final TimeValue activityTimeout) {
+        RecoveryTarget oldRecoveryTarget = null;
+        final RecoveryTarget newRecoveryTarget;
+
+        try {
+            synchronized (onGoingRecoveries) {
+                // swap recovery targets in a synchronized block to ensure that the newly added recovery target is picked up by
+                // cancelRecoveriesForShard whenever the old recovery target is picked up
+                oldRecoveryTarget = onGoingRecoveries.remove(recoveryId);
+                if (oldRecoveryTarget == null) {
+                    return null;
+                }
+
+                newRecoveryTarget = oldRecoveryTarget.retryCopy();
+                startRecoveryInternal(newRecoveryTarget, activityTimeout);
             }
+
+            // Closes the current recovery target
+            boolean successfulReset = oldRecoveryTarget.resetRecovery(newRecoveryTarget.cancellableThreads());
+            if (successfulReset) {
+                logger.trace("{} restarted recovery from {}, id [{}], previous id [{}]", newRecoveryTarget.shardId(),
+                    newRecoveryTarget.sourceNode(), newRecoveryTarget.recoveryId(), oldRecoveryTarget.recoveryId());
+                return newRecoveryTarget;
+            } else {
+                logger.trace("{} recovery could not be reset as it is already cancelled, recovery from {}, id [{}], previous id [{}]",
+                    newRecoveryTarget.shardId(), newRecoveryTarget.sourceNode(), newRecoveryTarget.recoveryId(),
+                    oldRecoveryTarget.recoveryId());
+                cancelRecovery(newRecoveryTarget.recoveryId(), "recovery cancelled during reset");
+                return null;
+            }
+        } catch (Exception e) {
+            // fail shard to be safe
+            oldRecoveryTarget.notifyListener(new RecoveryFailedException(oldRecoveryTarget.state(), "failed to retry recovery", e), true);
+            return null;
         }
     }
 
+    public RecoveryTarget getRecoveryTarget(long id) {
+        return onGoingRecoveries.get(id);
+    }
+
     /**
      * gets the {@link RecoveryTarget } for a given id. The RecoveryStatus returned has it's ref count already incremented
      * to make sure it's safe to use. However, you must call {@link RecoveryTarget#decRef()} when you are done with it, typically
@@ -116,7 +151,7 @@ public class RecoveriesCollection {
         if (recoveryRef == null) {
             throw new IndexShardClosedException(shardId);
         }
-        assert recoveryRef.status().shardId().equals(shardId);
+        assert recoveryRef.target().shardId().equals(shardId);
         return recoveryRef;
     }
 
@@ -143,7 +178,8 @@ public class RecoveriesCollection {
     public void failRecovery(long id, RecoveryFailedException e, boolean sendShardFailure) {
         RecoveryTarget removed = onGoingRecoveries.remove(id);
         if (removed != null) {
-            logger.trace("{} failing recovery from {}, id [{}]. Send shard failure: [{}]", removed.shardId(), removed.sourceNode(), removed.recoveryId(), sendShardFailure);
+            logger.trace("{} failing recovery from {}, id [{}]. Send shard failure: [{}]", removed.shardId(), removed.sourceNode(),
+                removed.recoveryId(), sendShardFailure);
             removed.fail(e, sendShardFailure);
         }
     }
@@ -171,11 +207,22 @@ public class RecoveriesCollection {
      */
     public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
         boolean cancelled = false;
-        for (RecoveryTarget status : onGoingRecoveries.values()) {
-            if (status.shardId().equals(shardId)) {
-                cancelled |= cancelRecovery(status.recoveryId(), reason);
+        List<RecoveryTarget> matchedRecoveries = new ArrayList<>();
+        synchronized (onGoingRecoveries) {
+            for (Iterator<RecoveryTarget> it = onGoingRecoveries.values().iterator(); it.hasNext(); ) {
+                RecoveryTarget status = it.next();
+                if (status.shardId().equals(shardId)) {
+                    matchedRecoveries.add(status);
+                    it.remove();
+                }
             }
         }
+        for (RecoveryTarget removed : matchedRecoveries) {
+            logger.trace("{} canceled recovery from {}, id [{}] (reason [{}])",
+                removed.shardId(), removed.sourceNode(), removed.recoveryId(), reason);
+            removed.cancel(reason);
+            cancelled = true;
+        }
         return cancelled;
     }
 
@@ -205,7 +252,7 @@ public class RecoveriesCollection {
             }
         }
 
-        public RecoveryTarget status() {
+        public RecoveryTarget target() {
             return status;
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
index d4ddccd8742..e238277b698 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySettings.java
@@ -69,7 +69,7 @@ public class RecoverySettings extends AbstractComponent {
      */
     public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
         Setting.timeSetting("indices.recovery.recovery_activity_timeout",
-                INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING::get, TimeValue.timeValueSeconds(0),
+            INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING::get, TimeValue.timeValueSeconds(0),
             Property.Dynamic, Property.NodeScope);
 
     public static final ByteSizeValue DEFAULT_CHUNK_SIZE = new ByteSizeValue(512, ByteSizeUnit.KB);
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index fa1a9a7979a..a6aa47492e1 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -30,17 +30,20 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.RateLimiter;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.index.engine.RecoveryEngineException;
+import org.elasticsearch.index.seqno.LocalCheckpointTracker;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexShardRelocatedException;
@@ -107,15 +110,15 @@ public class RecoverySourceHandler {
                                  final Supplier<Long> currentClusterStateVersionSupplier,
                                  Function<String, Releasable> delayNewRecoveries,
                                  final int fileChunkSizeInBytes,
-                                 final Logger logger) {
+                                 final Settings nodeSettings) {
         this.shard = shard;
         this.recoveryTarget = recoveryTarget;
         this.request = request;
         this.currentClusterStateVersionSupplier = currentClusterStateVersionSupplier;
         this.delayNewRecoveries = delayNewRecoveries;
-        this.logger = logger;
         this.indexName = this.request.shardId().getIndex().getName();
         this.shardId = this.request.shardId().id();
+        this.logger = Loggers.getLogger(getClass(), nodeSettings, request.shardId(), "recover to " + request.targetNode().getName());
         this.chunkSizeInBytes = fileChunkSizeInBytes;
         this.response = new RecoveryResponse();
     }
@@ -126,45 +129,61 @@ public class RecoverySourceHandler {
     public RecoveryResponse recoverToTarget() throws IOException {
         try (Translog.View translogView = shard.acquireTranslogView()) {
             logger.trace("captured translog id [{}] for recovery", translogView.minTranslogGeneration());
-            final IndexCommit phase1Snapshot;
-            try {
-                phase1Snapshot = shard.acquireIndexCommit(false);
-            } catch (Exception e) {
-                IOUtils.closeWhileHandlingException(translogView);
-                throw new RecoveryEngineException(shard.shardId(), 1, "Snapshot failed", e);
-            }
 
-            try {
-                phase1(phase1Snapshot, translogView);
-            } catch (Exception e) {
-                throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
-            } finally {
+            boolean isSequenceNumberBasedRecoveryPossible = request.startingSeqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO &&
+                isTranslogReadyForSequenceNumberBasedRecovery(translogView);
+
+            if (isSequenceNumberBasedRecoveryPossible) {
+                logger.trace("performing sequence numbers based recovery. starting at [{}]", request.startingSeqNo());
+            } else {
+                final IndexCommit phase1Snapshot;
                 try {
-                    shard.releaseIndexCommit(phase1Snapshot);
-                } catch (IOException ex) {
-                    logger.warn("releasing snapshot caused exception", ex);
+                    phase1Snapshot = shard.acquireIndexCommit(false);
+                } catch (final Exception e) {
+                    IOUtils.closeWhileHandlingException(translogView);
+                    throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e);
+                }
+                try {
+                    phase1(phase1Snapshot, translogView);
+                } catch (final Exception e) {
+                    throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
+                } finally {
+                    try {
+                        shard.releaseIndexCommit(phase1Snapshot);
+                    } catch (final IOException ex) {
+                        logger.warn("releasing snapshot caused exception", ex);
+                    }
                 }
             }
 
-            // engine was just started at the end of phase 1
+            try {
+                prepareTargetForTranslog(translogView.totalOperations(), shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp());
+            } catch (final Exception e) {
+                throw new RecoveryEngineException(shard.shardId(), 1, "prepare target for translog failed", e);
+            }
+
+            // engine was just started at the end of phase1
             if (shard.state() == IndexShardState.RELOCATED) {
-                /**
+                assert request.isPrimaryRelocation() == false :
+                    "recovery target should not retry primary relocation if previous attempt made it past finalization step";
+                /*
                  * The primary shard has been relocated while we copied files. This means that we can't guarantee any more that all
                  * operations that were replicated during the file copy (when the target engine was not yet opened) will be present in the
-                 * local translog and thus will be resent on phase 2. The reason is that an operation replicated by the target primary is
+                 * local translog and thus will be resent on phase2. The reason is that an operation replicated by the target primary is
                  * sent to the recovery target and the local shard (old primary) concurrently, meaning it may have arrived at the recovery
                  * target before we opened the engine and is still in-flight on the local shard.
                  *
                  * Checking the relocated status here, after we opened the engine on the target, is safe because primary relocation waits
                  * for all ongoing operations to complete and be fully replicated. Therefore all future operation by the new primary are
-                 * guaranteed to reach the target shard when it's engine is open.
+                 * guaranteed to reach the target shard when its engine is open.
                  */
                 throw new IndexShardRelocatedException(request.shardId());
             }
 
-            logger.trace("{} snapshot translog for recovery. current size is [{}]", shard.shardId(), translogView.totalOperations());
+            logger.trace("snapshot translog for recovery; current size is [{}]", translogView.totalOperations());
             try {
-                phase2(translogView.snapshot());
+                phase2(isSequenceNumberBasedRecoveryPossible ? request.startingSeqNo() : SequenceNumbersService.UNASSIGNED_SEQ_NO,
+                    translogView.snapshot());
             } catch (Exception e) {
                 throw new RecoveryEngineException(shard.shardId(), 2, "phase2 failed", e);
             }
@@ -174,6 +193,46 @@ public class RecoverySourceHandler {
         return response;
     }
 
+    /**
+     * Determines if the source translog is ready for a sequence-number-based peer recovery. The main condition here is that the source
+     * translog contains all operations between the local checkpoint on the target and the current maximum sequence number on the source.
+     *
+     * @param translogView a view of the translog on the source
+     * @return {@code true} if the source is ready for a sequence-number-based recovery
+     * @throws IOException if an I/O exception occurred reading the translog snapshot
+     */
+    boolean isTranslogReadyForSequenceNumberBasedRecovery(final Translog.View translogView) throws IOException {
+        final long startingSeqNo = request.startingSeqNo();
+        assert startingSeqNo >= 0;
+        final long endingSeqNo = shard.seqNoStats().getMaxSeqNo();
+        logger.trace("testing sequence numbers in range: [{}, {}]", startingSeqNo, endingSeqNo);
+        // the start recovery request is initialized with the starting sequence number set to the target shard's local checkpoint plus one
+        if (startingSeqNo - 1 <= endingSeqNo) {
+            /*
+             * We need to wait for all operations up to the current max to complete, otherwise we can not guarantee that all
+             * operations in the required range will be available for replaying from the translog of the source.
+             */
+            cancellableThreads.execute(() -> shard.waitForOpsToComplete(endingSeqNo));
+
+            logger.trace("all operations up to [{}] completed, checking translog content", endingSeqNo);
+
+            final LocalCheckpointTracker tracker = new LocalCheckpointTracker(shard.indexSettings(), startingSeqNo, startingSeqNo - 1);
+            final Translog.Snapshot snapshot = translogView.snapshot();
+            Translog.Operation operation;
+            while ((operation = snapshot.next()) != null) {
+                if (operation.seqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO) {
+                    tracker.markSeqNoAsCompleted(operation.seqNo());
+                }
+            }
+            return tracker.getCheckpoint() >= endingSeqNo;
+        } else {
+            // norelease this can currently happen if a snapshot restore rolls the primary back to a previous commit point; in this
+            // situation the local checkpoint on the replica can be far in advance of the maximum sequence number on the primary violating
+            // all assumptions regarding local and global checkpoints
+            return false;
+        }
+    }
+
     /**
      * Perform phase1 of the recovery operations. Once this {@link IndexCommit}
      * snapshot has been performed no commit operations (files being fsync'd)
@@ -220,14 +279,12 @@ public class RecoverySourceHandler {
                 final long numDocsSource = recoverySourceMetadata.getNumDocs();
                 if (numDocsTarget != numDocsSource) {
                     throw new IllegalStateException("try to recover " + request.shardId() + " from primary shard with sync id but number " +
-                            "of docs differ: " + numDocsTarget + " (" + request.sourceNode().getName() + ", primary) vs " + numDocsSource
+                            "of docs differ: " + numDocsSource + " (" + request.sourceNode().getName() + ", primary) vs " + numDocsTarget
                             + "(" + request.targetNode().getName() + ")");
                 }
                 // we shortcut recovery here because we have nothing to copy. but we must still start the engine on the target.
                 // so we don't return here
-                logger.trace("[{}][{}] skipping [phase1] to {} - identical sync id [{}] found on both source and target", indexName,
-                        shardId,
-                        request.targetNode(), recoverySourceSyncId);
+                logger.trace("skipping [phase1]- identical sync id [{}] found on both source and target", recoverySourceSyncId);
             } else {
                 final Store.RecoveryDiff diff = recoverySourceMetadata.recoveryDiff(request.metadataSnapshot());
                 for (StoreFileMetaData md : diff.identical) {
@@ -235,9 +292,8 @@ public class RecoverySourceHandler {
                     response.phase1ExistingFileSizes.add(md.length());
                     existingTotalSize += md.length();
                     if (logger.isTraceEnabled()) {
-                        logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has checksum [{}]," +
-                                        " size [{}]",
-                                indexName, shardId, request.targetNode(), md.name(), md.checksum(), md.length());
+                        logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," +
+                                        " size [{}]", md.name(), md.checksum(), md.length());
                     }
                     totalSize += md.length();
                 }
@@ -246,12 +302,10 @@ public class RecoverySourceHandler {
                 phase1Files.addAll(diff.missing);
                 for (StoreFileMetaData md : phase1Files) {
                     if (request.metadataSnapshot().asMap().containsKey(md.name())) {
-                        logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but is different: remote " +
-                                        "[{}], local [{}]",
-                                indexName, shardId, request.targetNode(), md.name(), request.metadataSnapshot().asMap().get(md.name()), md);
+                        logger.trace("recovery [phase1]: recovering [{}], exists in local store, but is different: remote [{}], local [{}]",
+                            md.name(), request.metadataSnapshot().asMap().get(md.name()), md);
                     } else {
-                        logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], does not exists in remote",
-                                indexName, shardId, request.targetNode(), md.name());
+                        logger.trace("recovery [phase1]: recovering [{}], does not exist in remote", md.name());
                     }
                     response.phase1FileNames.add(md.name());
                     response.phase1FileSizes.add(md.length());
@@ -261,9 +315,8 @@ public class RecoverySourceHandler {
                 response.phase1TotalSize = totalSize;
                 response.phase1ExistingTotalSize = existingTotalSize;
 
-                logger.trace("[{}][{}] recovery [phase1] to {}: recovering_files [{}] with total_size [{}], reusing_files [{}] with " +
-                                "total_size [{}]",
-                        indexName, shardId, request.targetNode(), response.phase1FileNames.size(),
+                logger.trace("recovery [phase1]: recovering_files [{}] with total_size [{}], reusing_files [{}] with total_size [{}]",
+                        response.phase1FileNames.size(),
                         new ByteSizeValue(totalSize), response.phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSize));
                 cancellableThreads.execute(() ->
                         recoveryTarget.receiveFileInfo(response.phase1FileNames, response.phase1FileSizes, response.phase1ExistingFileNames,
@@ -299,10 +352,10 @@ public class RecoverySourceHandler {
                             });
                             for (StoreFileMetaData md : metadata) {
                                 cancellableThreads.checkForCancel();
-                                logger.debug("{} checking integrity for file {} after remove corruption exception", shard.shardId(), md);
+                                logger.debug("checking integrity for file {} after remove corruption exception", md);
                                 if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
                                     shard.failShard("recovery", corruptIndexException);
-                                    logger.warn("{} Corrupted file detected {} checksum mismatch", shard.shardId(), md);
+                                    logger.warn("Corrupted file detected {} checksum mismatch", md);
                                     throw corruptIndexException;
                                 }
                             }
@@ -327,9 +380,7 @@ public class RecoverySourceHandler {
                 }
             }
 
-            prepareTargetForTranslog(translogView.totalOperations());
-
-            logger.trace("[{}][{}] recovery [phase1] to {}: took [{}]", indexName, shardId, request.targetNode(), stopWatch.totalTime());
+            logger.trace("recovery [phase1]: took [{}]", stopWatch.totalTime());
             response.phase1Time = stopWatch.totalTime().millis();
         } catch (Exception e) {
             throw new RecoverFilesRecoveryException(request.shardId(), response.phase1FileNames.size(), new ByteSizeValue(totalSize), e);
@@ -338,49 +389,50 @@ public class RecoverySourceHandler {
         }
     }
 
-
-    protected void prepareTargetForTranslog(final int totalTranslogOps) throws IOException {
+    void prepareTargetForTranslog(final int totalTranslogOps, final long maxUnsafeAutoIdTimestamp) throws IOException {
         StopWatch stopWatch = new StopWatch().start();
-        logger.trace("{} recovery [phase1] to {}: prepare remote engine for translog", request.shardId(), request.targetNode());
+        logger.trace("recovery [phase1]: prepare remote engine for translog");
         final long startEngineStart = stopWatch.totalTime().millis();
-        // Send a request preparing the new shard's translog to receive
-        // operations. This ensures the shard engine is started and disables
-        // garbage collection (not the JVM's GC!) of tombstone deletes
-        cancellableThreads.executeIO(() -> recoveryTarget.prepareForTranslogOperations(totalTranslogOps,
-            shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp()));
+        // Send a request preparing the new shard's translog to receive operations. This ensures the shard engine is started and disables
+        // garbage collection (not the JVM's GC!) of tombstone deletes.
+        cancellableThreads.executeIO(() -> recoveryTarget.prepareForTranslogOperations(totalTranslogOps, maxUnsafeAutoIdTimestamp));
         stopWatch.stop();
 
         response.startTime = stopWatch.totalTime().millis() - startEngineStart;
-        logger.trace("{} recovery [phase1] to {}: remote engine start took [{}]",
-                request.shardId(), request.targetNode(), stopWatch.totalTime());
+        logger.trace("recovery [phase1]: remote engine start took [{}]", stopWatch.totalTime());
     }
 
     /**
-     * Perform phase2 of the recovery process
+     * Perform phase two of the recovery process.
      * <p>
-     * Phase2 takes a snapshot of the current translog *without* acquiring the
-     * write lock (however, the translog snapshot is a point-in-time view of
-     * the translog). It then sends each translog operation to the target node
-     * so it can be replayed into the new shard.
+     * Phase two uses a snapshot of the current translog *without* acquiring the write lock (however, the translog snapshot is
+     * point-in-time view of the translog). It then sends each translog operation to the target node so it can be replayed into the new
+     * shard.
+     *
+     * @param startingSeqNo the sequence number to start recovery from, or {@link SequenceNumbersService#UNASSIGNED_SEQ_NO} if all
+     *                      ops should be sent
+     * @param snapshot      a snapshot of the translog
      */
-    public void phase2(Translog.Snapshot snapshot) {
+    void phase2(final long startingSeqNo, final Translog.Snapshot snapshot) throws IOException {
         if (shard.state() == IndexShardState.CLOSED) {
             throw new IndexShardClosedException(request.shardId());
         }
         cancellableThreads.checkForCancel();
 
-        StopWatch stopWatch = new StopWatch().start();
+        final StopWatch stopWatch = new StopWatch().start();
+
+        logger.trace("recovery [phase2]: sending transaction log operations");
+
+        // send all the snapshot's translog operations to the target
+        final int totalOperations = sendSnapshot(startingSeqNo, snapshot);
 
-        logger.trace("{} recovery [phase2] to {}: sending transaction log operations", request.shardId(), request.targetNode());
-        // Send all the snapshot's translog operations to the target
-        int totalOperations = sendSnapshot(snapshot);
         stopWatch.stop();
-        logger.trace("{} recovery [phase2] to {}: took [{}]", request.shardId(), request.targetNode(), stopWatch.totalTime());
+        logger.trace("recovery [phase2]: took [{}]", stopWatch.totalTime());
         response.phase2Time = stopWatch.totalTime().millis();
         response.phase2Operations = totalOperations;
     }
 
-    /**
+    /*
      * finalizes the recovery process
      */
     public void finalizeRecovery() {
@@ -389,7 +441,7 @@ public class RecoverySourceHandler {
         }
         cancellableThreads.checkForCancel();
         StopWatch stopWatch = new StopWatch().start();
-        logger.trace("[{}][{}] finalizing recovery to {}", indexName, shardId, request.targetNode());
+        logger.trace("finalizing recovery");
         cancellableThreads.execute(() -> {
             shard.markAllocationIdAsInSync(recoveryTarget.getTargetAllocationId());
             recoveryTarget.finalizeRecovery(shard.getGlobalCheckpoint());
@@ -402,95 +454,81 @@ public class RecoverySourceHandler {
             // state. This means that no new recovery can be completed based on information of a newer cluster state than the current one.
             try (Releasable ignored = delayNewRecoveries.apply("primary relocation hand-off in progress or completed for " + shardId)) {
                 final long currentClusterStateVersion = currentClusterStateVersionSupplier.get();
-                logger.trace("[{}][{}] waiting on {} to have cluster state with version [{}]", indexName, shardId, request.targetNode(),
-                    currentClusterStateVersion);
+                logger.trace("waiting on remote node to have cluster state with version [{}]", currentClusterStateVersion);
                 cancellableThreads.execute(() -> recoveryTarget.ensureClusterStateVersion(currentClusterStateVersion));
 
-                logger.trace("[{}][{}] performing relocation hand-off to {}", indexName, shardId, request.targetNode());
+                logger.trace("performing relocation hand-off");
                 cancellableThreads.execute(() -> shard.relocated("to " + request.targetNode()));
             }
-            /**
+            /*
              * if the recovery process fails after setting the shard state to RELOCATED, both relocation source and
              * target are failed (see {@link IndexShard#updateRoutingEntry}).
              */
         }
         stopWatch.stop();
-        logger.trace("[{}][{}] finalizing recovery to {}: took [{}]",
-                indexName, shardId, request.targetNode(), stopWatch.totalTime());
+        logger.trace("finalizing recovery took [{}]", stopWatch.totalTime());
     }
 
     /**
-     * Send the given snapshot's operations to this handler's target node.
+     * Send the given snapshot's operations with a sequence number greater than the specified staring sequence number to this handler's
+     * target node.
      * <p>
-     * Operations are bulked into a single request depending on an operation
-     * count limit or size-in-bytes limit
+     * Operations are bulked into a single request depending on an operation count limit or size-in-bytes limit.
      *
+     * @param startingSeqNo the sequence number for which only operations with a sequence number greater than this will be sent
+     * @param snapshot      the translog snapshot to replay operations from
      * @return the total number of translog operations that were sent
+     * @throws IOException if an I/O exception occurred reading the translog snapshot
      */
-    protected int sendSnapshot(final Translog.Snapshot snapshot) {
+    protected int sendSnapshot(final long startingSeqNo, final Translog.Snapshot snapshot) throws IOException {
         int ops = 0;
         long size = 0;
         int totalOperations = 0;
         final List<Translog.Operation> operations = new ArrayList<>();
-        Translog.Operation operation;
-        try {
-            operation = snapshot.next(); // this ex should bubble up
-        } catch (IOException ex) {
-            throw new ElasticsearchException("failed to get next operation from translog", ex);
+
+        if (snapshot.totalOperations() == 0) {
+            logger.trace("no translog operations to send");
         }
 
-        if (operation == null) {
-            logger.trace("[{}][{}] no translog operations to send to {}",
-                    indexName, shardId, request.targetNode());
-        }
-        while (operation != null) {
+        // send operations in batches
+        Translog.Operation operation;
+        while ((operation = snapshot.next()) != null) {
             if (shard.state() == IndexShardState.CLOSED) {
                 throw new IndexShardClosedException(request.shardId());
             }
             cancellableThreads.checkForCancel();
+            // if we are doing a sequence-number-based recovery, we have to skip older ops for which no sequence number was assigned, and
+            // any ops before the starting sequence number
+            final long seqNo = operation.seqNo();
+            if (startingSeqNo >= 0 && (seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO || seqNo < startingSeqNo)) continue;
             operations.add(operation);
-            ops += 1;
+            ops++;
             size += operation.estimateSize();
             totalOperations++;
 
-            // Check if this request is past bytes threshold, and
-            // if so, send it off
+            // check if this request is past bytes threshold, and if so, send it off
             if (size >= chunkSizeInBytes) {
-
-                // don't throttle translog, since we lock for phase3 indexing,
-                // so we need to move it as fast as possible. Note, since we
-                // index docs to replicas while the index files are recovered
-                // the lock can potentially be removed, in which case, it might
-                // make sense to re-enable throttling in this phase
                 cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations()));
                 if (logger.isTraceEnabled()) {
-                    logger.trace("[{}][{}] sent batch of [{}][{}] (total: [{}]) translog operations to {}",
-                            indexName, shardId, ops, new ByteSizeValue(size),
-                            snapshot.totalOperations(),
-                            request.targetNode());
+                    logger.trace("sent batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size),
+                        snapshot.totalOperations());
                 }
-
                 ops = 0;
                 size = 0;
                 operations.clear();
             }
-            try {
-                operation = snapshot.next(); // this ex should bubble up
-            } catch (IOException ex) {
-                throw new ElasticsearchException("failed to get next operation from translog", ex);
-            }
         }
-        // send the leftover
+
+        // send the leftover operations
         if (!operations.isEmpty()) {
             cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations()));
+        }
 
-        }
         if (logger.isTraceEnabled()) {
-            logger.trace("[{}][{}] sent final batch of [{}][{}] (total: [{}]) translog operations to {}",
-                    indexName, shardId, ops, new ByteSizeValue(size),
-                    snapshot.totalOperations(),
-                    request.targetNode());
+            logger.trace("sent final batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size),
+                snapshot.totalOperations());
         }
+
         return totalOperations;
     }
 
@@ -550,7 +588,7 @@ public class RecoverySourceHandler {
             ArrayUtil.timSort(files, (a, b) -> Long.compare(a.length(), b.length())); // send smallest first
             for (int i = 0; i < files.length; i++) {
                 final StoreFileMetaData md = files[i];
-                try (final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
+                try (IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE)) {
                     // it's fine that we are only having the indexInput in the try/with block. The copy methods handles
                     // exceptions during close correctly and doesn't hide the original exception.
                     Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStreamFactory.apply(md));
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 919601de949..77d8b4d7077 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -45,7 +45,7 @@ import java.util.Map;
  */
 public class RecoveryState implements ToXContent, Streamable {
 
-    public static enum Stage {
+    public enum Stage {
         INIT((byte) 0),
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index 67ee1a5ac9a..d9886efa07b 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRefCounted;
@@ -48,6 +49,7 @@ import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.index.translog.Translog;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
@@ -55,6 +57,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -87,32 +92,28 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     // last time this status was accessed
     private volatile long lastAccessTime = System.nanoTime();
 
+    // latch that can be used to blockingly wait for RecoveryTarget to be closed
+    private final CountDownLatch closedLatch = new CountDownLatch(1);
+
     private final Map<String, String> tempFileNames = ConcurrentCollections.newConcurrentMap();
 
-    private RecoveryTarget(RecoveryTarget copyFrom) { // copy constructor
-        this(copyFrom.indexShard, copyFrom.sourceNode, copyFrom.listener, copyFrom.cancellableThreads, copyFrom.recoveryId,
-            copyFrom.ensureClusterStateVersionCallback);
-    }
-
-    public RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
-                          Callback<Long> ensureClusterStateVersionCallback) {
-        this(indexShard, sourceNode, listener, new CancellableThreads(), idGenerator.incrementAndGet(), ensureClusterStateVersionCallback);
-    }
     /**
-     * creates a new recovery target object that represents a recovery to the provided indexShard
+     * Creates a new recovery target object that represents a recovery to the provided shard.
      *
-     * @param indexShard local shard where we want to recover to
-     * @param sourceNode source node of the recovery where we recover from
-     * @param listener called when recovery is completed / failed
+     * @param indexShard                        local shard where we want to recover to
+     * @param sourceNode                        source node of the recovery where we recover from
+     * @param listener                          called when recovery is completed/failed
      * @param ensureClusterStateVersionCallback callback to ensure that the current node is at least on a cluster state with the provided
-     *                                          version. Necessary for primary relocation so that new primary knows about all other ongoing
-     *                                          replica recoveries when replicating documents (see {@link RecoverySourceHandler}).
+     *                                          version; necessary for primary relocation so that new primary knows about all other ongoing
+     *                                          replica recoveries when replicating documents (see {@link RecoverySourceHandler})
      */
-    private RecoveryTarget(IndexShard indexShard, DiscoveryNode sourceNode, PeerRecoveryTargetService.RecoveryListener listener,
-                           CancellableThreads cancellableThreads, long recoveryId, Callback<Long> ensureClusterStateVersionCallback) {
+    public RecoveryTarget(final IndexShard indexShard,
+                   final DiscoveryNode sourceNode,
+                   final PeerRecoveryTargetService.RecoveryListener listener,
+                   final Callback<Long> ensureClusterStateVersionCallback) {
         super("recovery_status");
-        this.cancellableThreads = cancellableThreads;
-        this.recoveryId = recoveryId;
+        this.cancellableThreads = new CancellableThreads();
+        this.recoveryId = idGenerator.incrementAndGet();
         this.listener = listener;
         this.logger = Loggers.getLogger(getClass(), indexShard.indexSettings().getSettings(), indexShard.shardId());
         this.indexShard = indexShard;
@@ -126,6 +127,15 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
         indexShard.recoveryStats().incCurrentAsTarget();
     }
 
+    /**
+     * Returns a fresh recovery target to retry recovery from the same source node onto the same shard and using the same listener.
+     *
+     * @return a copy of this recovery target
+     */
+    public RecoveryTarget retryCopy() {
+        return new RecoveryTarget(indexShard, sourceNode, listener, ensureClusterStateVersionCallback);
+    }
+
     public long recoveryId() {
         return recoveryId;
     }
@@ -147,7 +157,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
         return indexShard.recoveryState();
     }
 
-    public CancellableThreads CancellableThreads() {
+    public CancellableThreads cancellableThreads() {
         return cancellableThreads;
     }
 
@@ -177,19 +187,37 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     }
 
     /**
-     * Closes the current recovery target and returns a
-     * clone to reset the ongoing recovery.
-     * Note: the returned target must be canceled, failed or finished
-     * in order to release all it's reference.
+     * Closes the current recovery target and waits up to a certain timeout for resources to be freed.
+     * Returns true if resetting the recovery was successful, false if the recovery target is already cancelled / failed or marked as done.
      */
-    RecoveryTarget resetRecovery() throws IOException {
-        ensureRefCount();
+    boolean resetRecovery(CancellableThreads newTargetCancellableThreads) throws IOException {
         if (finished.compareAndSet(false, true)) {
-            // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now
-            decRef();
+            try {
+                logger.debug("reset of recovery with shard {} and id [{}]", shardId, recoveryId);
+            } finally {
+                // release the initial reference. recovery files will be cleaned as soon as ref count goes to zero, potentially now.
+                decRef();
+            }
+            try {
+                newTargetCancellableThreads.execute(closedLatch::await);
+            } catch (CancellableThreads.ExecutionCancelledException e) {
+                logger.trace("new recovery target cancelled for shard {} while waiting on old recovery target with id [{}] to close",
+                    shardId, recoveryId);
+                return false;
+            }
+            RecoveryState.Stage stage = indexShard.recoveryState().getStage();
+            if (indexShard.recoveryState().getPrimary() && (stage == RecoveryState.Stage.FINALIZE || stage == RecoveryState.Stage.DONE)) {
+                // once primary relocation has moved past the finalization step, the relocation source can be moved to RELOCATED state
+                // and start indexing as primary into the target shard (see TransportReplicationAction). Resetting the target shard in this
+                // state could mean that indexing is halted until the recovery retry attempt is completed and could also destroy existing
+                // documents indexed and acknowledged before the reset.
+                assert stage != RecoveryState.Stage.DONE : "recovery should not have completed when it's being reset";
+                throw new IllegalStateException("cannot reset recovery as previous attempt made it past finalization step");
+            }
+            indexShard.performRecoveryRestart();
+            return true;
         }
-        indexShard.performRecoveryRestart();
-        return new RecoveryTarget(this);
+        return false;
     }
 
     /**
@@ -197,7 +225,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
      * unless this object is in use (in which case it will be cleaned once all ongoing users call
      * {@link #decRef()}
      * <p>
-     * if {@link #CancellableThreads()} was used, the threads will be interrupted.
+     * if {@link #cancellableThreads()} was used, the threads will be interrupted.
      */
     public void cancel(String reason) {
         if (finished.compareAndSet(false, true)) {
@@ -220,7 +248,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
     public void fail(RecoveryFailedException e, boolean sendShardFailure) {
         if (finished.compareAndSet(false, true)) {
             try {
-                listener.onRecoveryFailure(state(), e, sendShardFailure);
+                notifyListener(e, sendShardFailure);
             } finally {
                 try {
                     cancellableThreads.cancel("failed recovery [" + ExceptionsHelper.stackTrace(e) + "]");
@@ -232,6 +260,10 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
         }
     }
 
+    public void notifyListener(RecoveryFailedException e, boolean sendShardFailure) {
+        listener.onRecoveryFailure(state(), e, sendShardFailure);
+    }
+
     /** mark the current recovery as done */
     public void markAsDone() {
         if (finished.compareAndSet(false, true)) {
@@ -309,6 +341,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
             // free store. increment happens in constructor
             store.decRef();
             indexShard.recoveryStats().decCurrentAsTarget();
+            closedLatch.countDown();
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
index 591176f047a..fdf0de32f2f 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.indices.recovery;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
 
@@ -29,18 +29,17 @@ import java.util.function.Function;
 import java.util.function.Supplier;
 
 /**
- * A recovery handler that skips phase 1 as well as sending the snapshot. During phase 3 the shard is marked
- * as relocated an closed to ensure that the engine is closed and the target can acquire the IW write lock.
+ * A recovery handler that skips phase one as well as sending the translog snapshot.
  */
 public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
 
     private final IndexShard shard;
     private final StartRecoveryRequest request;
 
-    public SharedFSRecoverySourceHandler(IndexShard shard, RecoveryTargetHandler recoveryTarget, StartRecoveryRequest request,
-                                         Supplier<Long> currentClusterStateVersionSupplier,
-                                         Function<String, Releasable> delayNewRecoveries, Logger logger) {
-        super(shard, recoveryTarget, request, currentClusterStateVersionSupplier, delayNewRecoveries, -1, logger);
+    SharedFSRecoverySourceHandler(IndexShard shard, RecoveryTargetHandler recoveryTarget, StartRecoveryRequest request,
+                                  Supplier<Long> currentClusterStateVersionSupplier,
+                                  Function<String, Releasable> delayNewRecoveries, Settings nodeSettings) {
+        super(shard, recoveryTarget, request, currentClusterStateVersionSupplier, delayNewRecoveries, -1, nodeSettings);
         this.shard = shard;
         this.request = request;
     }
@@ -49,7 +48,8 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
     public RecoveryResponse recoverToTarget() throws IOException {
         boolean engineClosed = false;
         try {
-            logger.trace("{} recovery [phase1] to {}: skipping phase 1 for shared filesystem", request.shardId(), request.targetNode());
+            logger.trace("recovery [phase1]: skipping phase1 for shared filesystem");
+            final long maxUnsafeAutoIdTimestamp = shard.segmentStats(false).getMaxUnsafeAutoIdTimestamp();
             if (request.isPrimaryRelocation()) {
                 logger.debug("[phase1] closing engine on primary for shared filesystem recovery");
                 try {
@@ -62,7 +62,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
                     shard.failShard("failed to close engine (phase1)", e);
                 }
             }
-            prepareTargetForTranslog(0);
+            prepareTargetForTranslog(0, maxUnsafeAutoIdTimestamp);
             finalizeRecovery();
             return response;
         } catch (Exception e) {
@@ -82,9 +82,9 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
     }
 
     @Override
-    protected int sendSnapshot(Translog.Snapshot snapshot) {
-        logger.trace("{} skipping recovery of translog snapshot on shared filesystem to: {}",
-                shard.shardId(), request.targetNode());
+    protected int sendSnapshot(final long startingSeqNo, final Translog.Snapshot snapshot) {
+        logger.trace("skipping recovery of translog snapshot on shared filesystem");
         return 0;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
index bc8a73b5622..46a29d65464 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java
@@ -19,46 +19,60 @@
 
 package org.elasticsearch.indices.recovery;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 
+/**
+ * Represents a request for starting a peer recovery.
+ */
 public class StartRecoveryRequest extends TransportRequest {
 
     private long recoveryId;
-
     private ShardId shardId;
-
     private DiscoveryNode sourceNode;
-
     private DiscoveryNode targetNode;
-
     private Store.MetadataSnapshot metadataSnapshot;
-
     private boolean primaryRelocation;
+    private long startingSeqNo;
 
     public StartRecoveryRequest() {
     }
 
     /**
-     * Start recovery request.
+     * Construct a request for starting a peer recovery.
      *
-     * @param sourceNode       The node to recover from
-     * @param targetNode       The node to recover to
+     * @param shardId           the shard ID to recover
+     * @param sourceNode        the source node to remover from
+     * @param targetNode        the target node to recover to
+     * @param metadataSnapshot  the Lucene metadata
+     * @param primaryRelocation whether or not the recovery is a primary relocation
+     * @param recoveryId        the recovery ID
+     * @param startingSeqNo     the starting sequence number
      */
-    public StartRecoveryRequest(ShardId shardId, DiscoveryNode sourceNode, DiscoveryNode targetNode, Store.MetadataSnapshot metadataSnapshot, boolean primaryRelocation, long recoveryId) {
+    public StartRecoveryRequest(final ShardId shardId,
+                                final DiscoveryNode sourceNode,
+                                final DiscoveryNode targetNode,
+                                final Store.MetadataSnapshot metadataSnapshot,
+                                final boolean primaryRelocation,
+                                final long recoveryId,
+                                final long startingSeqNo) {
         this.recoveryId = recoveryId;
         this.shardId = shardId;
         this.sourceNode = sourceNode;
         this.targetNode = targetNode;
         this.metadataSnapshot = metadataSnapshot;
         this.primaryRelocation = primaryRelocation;
+        this.startingSeqNo = startingSeqNo;
     }
 
     public long recoveryId() {
@@ -85,6 +99,10 @@ public class StartRecoveryRequest extends TransportRequest {
         return metadataSnapshot;
     }
 
+    public long startingSeqNo() {
+        return startingSeqNo;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -94,6 +112,11 @@ public class StartRecoveryRequest extends TransportRequest {
         targetNode = new DiscoveryNode(in);
         metadataSnapshot = new Store.MetadataSnapshot(in);
         primaryRelocation = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            startingSeqNo = in.readLong();
+        } else {
+            startingSeqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        }
     }
 
     @Override
@@ -105,6 +128,9 @@ public class StartRecoveryRequest extends TransportRequest {
         targetNode.writeTo(out);
         metadataSnapshot.writeTo(out);
         out.writeBoolean(primaryRelocation);
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            out.writeLong(startingSeqNo);
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 589d8348981..4f0ee0e11e4 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -26,7 +26,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateObserver;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -233,7 +233,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
         private final AtomicInteger awaitingResponses;
         private final AtomicInteger activeCopies;
 
-        public ShardActiveResponseHandler(ShardId shardId, long clusterStateVersion, int expectedActiveCopies) {
+        ShardActiveResponseHandler(ShardId shardId, long clusterStateVersion, int expectedActiveCopies) {
             this.shardId = shardId;
             this.expectedActiveCopies = expectedActiveCopies;
             this.clusterStateVersion = clusterStateVersion;
@@ -283,24 +283,19 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
                 return;
             }
 
-            clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new ClusterStateUpdateTask() {
+            clusterService.submitStateUpdateTask("indices_store ([" + shardId + "] active fully on other nodes)", new LocalClusterUpdateTask() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public ClusterState execute(ClusterState currentState) throws Exception {
+                public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                     if (clusterStateVersion != currentState.getVersion()) {
                         logger.trace("not deleting shard {}, the update task state version[{}] is not equal to cluster state before shard active api call [{}]", shardId, currentState.getVersion(), clusterStateVersion);
-                        return currentState;
+                        return unchanged();
                     }
                     try {
                         indicesService.deleteShardStore("no longer used", shardId, currentState);
                     } catch (Exception ex) {
                         logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to delete unallocated shard, ignoring", shardId), ex);
                     }
-                    return currentState;
+                    return unchanged();
                 }
 
                 @Override
@@ -401,7 +396,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
         private String indexUUID;
         private ShardId shardId;
 
-        public ShardActiveRequest() {
+        ShardActiveRequest() {
         }
 
         ShardActiveRequest(ClusterName clusterName, String indexUUID, ShardId shardId, TimeValue timeout) {
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index 707df5feb1e..84d3354ba4e 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.AsyncShardFetch;
 import org.elasticsearch.index.IndexService;
@@ -130,7 +131,8 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
                 // we may send this requests while processing the cluster state that recovered the index
                 // sometimes the request comes in before the local node processed that cluster state
                 // in such cases we can load it from disk
-                metaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(shardId.getIndex()));
+                metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                    nodeEnv.indexPaths(shardId.getIndex()));
             }
             if (metaData == null) {
                 logger.trace("{} node doesn't have meta data for the requests index, responding with empty", shardId);
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 88105420e14..8eaf44048c1 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -280,6 +280,7 @@ public final class ConfigurationUtils {
 
     public static Processor readProcessor(Map<String, Processor.Factory> processorFactories,
                                            String type, Map<String, Object> config) throws Exception {
+        String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
         Processor.Factory factory = processorFactories.get(type);
         if (factory != null) {
             boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
@@ -287,7 +288,6 @@ public final class ConfigurationUtils {
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorFactories);
-            String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY,
@@ -309,6 +309,6 @@ public final class ConfigurationUtils {
                 throw newConfigurationException(type, tag, null, e);
             }
         }
-        throw new ElasticsearchParseException("No processor type exists with name [" + type + "]");
+        throw newConfigurationException(type, tag, null, "No processor type exists with name [" + type + "]");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index eaae1a3e881..02d3988c4a8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -145,8 +145,24 @@ public final class IngestDocument {
      * or if the field that is found at the provided path is not of the expected type.
      */
     public byte[] getFieldValueAsBytes(String path) {
-        Object object = getFieldValue(path, Object.class);
-        if (object instanceof byte[]) {
+        return getFieldValueAsBytes(path, false);
+    }
+
+    /**
+     * Returns the value contained in the document for the provided path as a byte array.
+     * If the path value is a string, a base64 decode operation will happen.
+     * If the path value is a byte array, it is just returned
+     * @param path The path within the document in dot-notation
+     * @param ignoreMissing The flag to determine whether to throw an exception when `path` is not found in the document.
+     * @return the byte array for the provided path if existing
+     * @throws IllegalArgumentException if the path is null, empty, invalid, if the field doesn't exist
+     * or if the field that is found at the provided path is not of the expected type.
+     */
+    public byte[] getFieldValueAsBytes(String path, boolean ignoreMissing) {
+        Object object = getFieldValue(path, Object.class, ignoreMissing);
+        if (object == null) {
+            return null;
+        } else if (object instanceof byte[]) {
             return (byte[]) object;
         } else if (object instanceof String) {
             return Base64.getDecoder().decode(object.toString());
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 40f401ac6b4..ca8a5df8450 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -21,10 +21,9 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -45,9 +44,8 @@ import java.util.Map;
 public final class IngestMetadata implements MetaData.Custom {
 
     public static final String TYPE = "ingest";
-    public static final IngestMetadata PROTO = new IngestMetadata();
     private static final ParseField PIPELINES_FIELD = new ParseField("pipeline");
-    private static final ObjectParser<List<PipelineConfiguration>, ParseFieldMatcherSupplier> INGEST_METADATA_PARSER = new ObjectParser<>(
+    private static final ObjectParser<List<PipelineConfiguration>, Void> INGEST_METADATA_PARSER = new ObjectParser<>(
             "ingest_metadata", ArrayList::new);
 
     static {
@@ -67,7 +65,7 @@ public final class IngestMetadata implements MetaData.Custom {
     }
 
     @Override
-    public String type() {
+    public String getWriteableName() {
         return TYPE;
     }
 
@@ -75,15 +73,14 @@ public final class IngestMetadata implements MetaData.Custom {
         return pipelines;
     }
 
-    @Override
-    public IngestMetadata readFrom(StreamInput in) throws IOException {
+    public IngestMetadata(StreamInput in) throws IOException {
         int size = in.readVInt();
         Map<String, PipelineConfiguration> pipelines = new HashMap<>(size);
         for (int i = 0; i < size; i++) {
-            PipelineConfiguration pipeline = PipelineConfiguration.readPipelineConfiguration(in);
+            PipelineConfiguration pipeline = PipelineConfiguration.readFrom(in);
             pipelines.put(pipeline.getId(), pipeline);
         }
-        return new IngestMetadata(pipelines);
+        this.pipelines = Collections.unmodifiableMap(pipelines);
     }
 
     @Override
@@ -94,10 +91,9 @@ public final class IngestMetadata implements MetaData.Custom {
         }
     }
 
-    @Override
-    public IngestMetadata fromXContent(XContentParser parser) throws IOException {
+    public static IngestMetadata fromXContent(XContentParser parser) throws IOException {
         Map<String, PipelineConfiguration> pipelines = new HashMap<>();
-        List<PipelineConfiguration> configs = INGEST_METADATA_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT);
+        List<PipelineConfiguration> configs = INGEST_METADATA_PARSER.parse(parser, null);
         for (PipelineConfiguration pipeline : configs) {
             pipelines.put(pipeline.getId(), pipeline);
         }
@@ -124,12 +120,11 @@ public final class IngestMetadata implements MetaData.Custom {
         return new IngestMetadataDiff((IngestMetadata) before, this);
     }
 
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
         return new IngestMetadataDiff(in);
     }
 
-    static class IngestMetadataDiff implements Diff<MetaData.Custom> {
+    static class IngestMetadataDiff implements NamedDiff<MetaData.Custom> {
 
         final Diff<Map<String, PipelineConfiguration>> pipelines;
 
@@ -137,8 +132,9 @@ public final class IngestMetadata implements MetaData.Custom {
             this.pipelines = DiffableUtils.diff(before.pipelines, after.pipelines, DiffableUtils.getStringKeySerializer());
         }
 
-        public IngestMetadataDiff(StreamInput in) throws IOException {
-            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), PipelineConfiguration.PROTOTYPE);
+        IngestMetadataDiff(StreamInput in) throws IOException {
+            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), PipelineConfiguration::readFrom,
+                PipelineConfiguration::readDiffFrom);
         }
 
         @Override
@@ -150,6 +146,11 @@ public final class IngestMetadata implements MetaData.Custom {
         public void writeTo(StreamOutput out) throws IOException {
             pipelines.writeTo(out);
         }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
index 6bb410c78ea..26d6737706b 100644
--- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
@@ -44,10 +44,7 @@ public class InternalTemplateService implements TemplateService {
         int mustacheEnd = template.indexOf("}}");
         if (mustacheStart != -1 && mustacheEnd != -1 && mustacheStart < mustacheEnd) {
             Script script = new Script(ScriptType.INLINE, "mustache", template, Collections.emptyMap());
-            CompiledScript compiledScript = scriptService.compile(
-                script,
-                ScriptContext.Standard.INGEST,
-                Collections.emptyMap());
+            CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST);
             return new Template() {
                 @Override
                 public String execute(Map<String, Object> model) {
@@ -73,7 +70,7 @@ public class InternalTemplateService implements TemplateService {
 
         private final String value;
 
-        public StringTemplate(String value) {
+        StringTemplate(String value) {
             this.value = value;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
index 9e15e445f12..3e874a0fd1d 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,49 +30,50 @@ import org.elasticsearch.common.xcontent.ContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Encapsulates a pipeline's id and configuration as a blob
  */
 public final class PipelineConfiguration extends AbstractDiffable<PipelineConfiguration> implements ToXContent {
 
-    static final PipelineConfiguration PROTOTYPE = new PipelineConfiguration(null, null);
-
-    public static PipelineConfiguration readPipelineConfiguration(StreamInput in) throws IOException {
-        return PROTOTYPE.readFrom(in);
-    }
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
     static {
         PARSER.declareString(Builder::setId, new ParseField("id"));
         PARSER.declareField((parser, builder, aVoid) -> {
             XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent());
             XContentHelper.copyCurrentStructure(contentBuilder.generator(), parser);
-            builder.setConfig(contentBuilder.bytes());
+            builder.setConfig(contentBuilder.bytes(), contentBuilder.contentType());
         }, new ParseField("config"), ObjectParser.ValueType.OBJECT);
+
     }
 
-    public static ContextParser<ParseFieldMatcherSupplier, PipelineConfiguration> getParser() {
-        return (p, c) -> PARSER.apply(p ,c).build();
+    public static ContextParser<Void, PipelineConfiguration> getParser() {
+        return (parser, context) -> PARSER.apply(parser, null).build();
     }
     private static class Builder {
 
         private String id;
         private BytesReference config;
+        private XContentType xContentType;
 
         void setId(String id) {
             this.id = id;
         }
 
-        void setConfig(BytesReference config) {
+        void setConfig(BytesReference config, XContentType xContentType) {
             this.config = config;
+            this.xContentType = xContentType;
         }
 
         PipelineConfiguration build() {
-            return new PipelineConfiguration(id, config);
+            return new PipelineConfiguration(id, config, xContentType);
         }
     }
 
@@ -80,10 +82,12 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig
     // and the way the map of maps config is read requires a deep copy (it removes instead of gets entries to check for unused options)
     // also the get pipeline api just directly returns this to the caller
     private final BytesReference config;
+    private final XContentType xContentType;
 
-    public PipelineConfiguration(String id, BytesReference config) {
-        this.id = id;
-        this.config = config;
+    public PipelineConfiguration(String id, BytesReference config, XContentType xContentType) {
+        this.id = Objects.requireNonNull(id);
+        this.config = Objects.requireNonNull(config);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     public String getId() {
@@ -91,7 +95,17 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig
     }
 
     public Map<String, Object> getConfigAsMap() {
-        return XContentHelper.convertToMap(config, true).v2();
+        return XContentHelper.convertToMap(config, true, xContentType).v2();
+    }
+
+    // pkg-private for tests
+    XContentType getXContentType() {
+        return xContentType;
+    }
+
+    // pkg-private for tests
+    BytesReference getConfig() {
+        return config;
     }
 
     @Override
@@ -103,15 +117,27 @@ public final class PipelineConfiguration extends AbstractDiffable<PipelineConfig
         return builder;
     }
 
-    @Override
-    public PipelineConfiguration readFrom(StreamInput in) throws IOException {
-        return new PipelineConfiguration(in.readString(), in.readBytesReference());
+    public static PipelineConfiguration readFrom(StreamInput in) throws IOException {
+        if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            return new PipelineConfiguration(in.readString(), in.readBytesReference(), XContentType.readFrom(in));
+        } else {
+            final String id = in.readString();
+            final BytesReference config = in.readBytesReference();
+            return new PipelineConfiguration(id, config, XContentFactory.xContentType(config));
+        }
+    }
+
+    public static Diff<PipelineConfiguration> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(PipelineConfiguration::readFrom, in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeBytesReference(config);
+        if (out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            xContentType.writeTo(out);
+        }
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index 1e938581b8c..d476d7c2bd8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -41,9 +41,11 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 
 public class PipelineStore extends AbstractComponent implements ClusterStateApplier {
 
@@ -111,17 +113,26 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
             return currentState;
         }
         Map<String, PipelineConfiguration> pipelines = currentIngestMetadata.getPipelines();
-        if (pipelines.containsKey(request.getId()) == false) {
-            throw new ResourceNotFoundException("pipeline [{}] is missing", request.getId());
-        } else {
-            pipelines = new HashMap<>(pipelines);
-            pipelines.remove(request.getId());
-            ClusterState.Builder newState = ClusterState.builder(currentState);
-            newState.metaData(MetaData.builder(currentState.getMetaData())
-                    .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines))
-                    .build());
-            return newState.build();
+        Set<String> toRemove = new HashSet<>();
+        for (String pipelineKey : pipelines.keySet()) {
+            if (Regex.simpleMatch(request.getId(), pipelineKey)) {
+                toRemove.add(pipelineKey);
+            }
         }
+        if (toRemove.isEmpty() && Regex.isMatchAllPattern(request.getId()) == false) {
+            throw new ResourceNotFoundException("pipeline [{}] is missing", request.getId());
+        } else if (toRemove.isEmpty()) {
+            return currentState;
+        }
+        final Map<String, PipelineConfiguration> pipelinesCopy = new HashMap<>(pipelines);
+        for (String key : toRemove) {
+            pipelinesCopy.remove(key);
+        }
+        ClusterState.Builder newState = ClusterState.builder(currentState);
+        newState.metaData(MetaData.builder(currentState.getMetaData())
+                .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelinesCopy))
+                .build());
+        return newState.build();
     }
 
     /**
@@ -151,7 +162,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
             throw new IllegalStateException("Ingest info is empty");
         }
 
-        Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2();
+        Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
         Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories);
         List<IllegalArgumentException> exceptions = new ArrayList<>();
         for (Processor processor : pipeline.flattenAllProcessors()) {
@@ -174,7 +185,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
             pipelines = new HashMap<>();
         }
 
-        pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource()));
+        pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), request.getSource(), request.getXContentType()));
         ClusterState.Builder newState = ClusterState.builder(currentState);
         newState.metaData(MetaData.builder(currentState.getMetaData())
             .putCustom(IngestMetadata.TYPE, new IngestMetadata(pipelines))
diff --git a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
index 59434be7d93..9467bf29923 100644
--- a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
@@ -28,6 +28,7 @@ import org.elasticsearch.monitor.jvm.JvmService;
 import org.elasticsearch.monitor.os.OsService;
 import org.elasticsearch.monitor.process.ProcessService;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.cluster.ClusterInfoService;
 
 import java.io.IOException;
 
@@ -39,13 +40,14 @@ public class MonitorService extends AbstractLifecycleComponent {
     private final JvmService jvmService;
     private final FsService fsService;
 
-    public MonitorService(Settings settings, NodeEnvironment nodeEnvironment, ThreadPool threadPool) throws IOException {
+    public MonitorService(Settings settings, NodeEnvironment nodeEnvironment, ThreadPool threadPool,
+                          ClusterInfoService clusterInfoService) throws IOException {
         super(settings);
         this.jvmGcMonitorService = new JvmGcMonitorService(settings, threadPool);
         this.osService = new OsService(settings);
         this.processService = new ProcessService(settings);
         this.jvmService = new JvmService(settings);
-        this.fsService = new FsService(settings, nodeEnvironment);
+        this.fsService = new FsService(settings, nodeEnvironment, clusterInfoService);
     }
 
     public OsService osService() {
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
index bf7dce9c0d5..33cb70a0d0b 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.monitor.fs;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -438,12 +440,20 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
     private final Path[] paths;
     private final IoStats ioStats;
     private final Path total;
+    private final DiskUsage leastDiskEstimate;
+    private final DiskUsage mostDiskEstimate;
 
     public FsInfo(long timestamp, IoStats ioStats, Path[] paths) {
+        this(timestamp, ioStats, paths, null, null);
+    }
+
+    public FsInfo(long timestamp, IoStats ioStats, Path[] paths, @Nullable DiskUsage leastUsage, @Nullable DiskUsage mostUsage) {
         this.timestamp = timestamp;
         this.ioStats = ioStats;
         this.paths = paths;
         this.total = total();
+        this.leastDiskEstimate = leastUsage;
+        this.mostDiskEstimate = mostUsage;
     }
 
     /**
@@ -457,6 +467,13 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
             paths[i] = new Path(in);
         }
         this.total = total();
+        if (in.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            this.leastDiskEstimate = in.readOptionalWriteable(DiskUsage::new);
+            this.mostDiskEstimate = in.readOptionalWriteable(DiskUsage::new);
+        } else {
+            this.leastDiskEstimate = null;
+            this.mostDiskEstimate = null;
+        }
     }
 
     @Override
@@ -467,12 +484,26 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
         for (Path path : paths) {
             path.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            out.writeOptionalWriteable(this.leastDiskEstimate);
+            out.writeOptionalWriteable(this.mostDiskEstimate);
+        }
     }
 
     public Path getTotal() {
         return total;
     }
 
+    @Nullable
+    public DiskUsage getLeastDiskEstimate() {
+        return this.leastDiskEstimate;
+    }
+
+    @Nullable
+    public DiskUsage getMostDiskEstimate() {
+        return this.mostDiskEstimate;
+    }
+
     private Path total() {
         Path res = new Path();
         Set<String> seenDevices = new HashSet<>(paths.length);
@@ -506,6 +537,27 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
         builder.field(Fields.TIMESTAMP, timestamp);
         builder.field(Fields.TOTAL);
         total().toXContent(builder, params);
+        if (leastDiskEstimate != null) {
+            builder.startObject(Fields.LEAST_ESTIMATE);
+            {
+                builder.field(Fields.PATH, leastDiskEstimate.getPath());
+                builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, leastDiskEstimate.getTotalBytes());
+                builder.byteSizeField(Fields.AVAILABLE_IN_BYTES, Fields.AVAILABLE, leastDiskEstimate.getFreeBytes());
+                builder.field(Fields.USAGE_PERCENTAGE, leastDiskEstimate.getUsedDiskAsPercentage());
+            }
+            builder.endObject();
+        }
+
+        if (mostDiskEstimate != null) {
+            builder.startObject(Fields.MOST_ESTIMATE);
+            {
+                builder.field(Fields.PATH, mostDiskEstimate.getPath());
+                builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, mostDiskEstimate.getTotalBytes());
+                builder.byteSizeField(Fields.AVAILABLE_IN_BYTES, Fields.AVAILABLE, mostDiskEstimate.getFreeBytes());
+                builder.field(Fields.USAGE_PERCENTAGE, mostDiskEstimate.getUsedDiskAsPercentage());
+            }
+            builder.endObject();
+        }
         builder.startArray(Fields.DATA);
         for (Path path : paths) {
             path.toXContent(builder, params);
@@ -525,6 +577,13 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
         static final String TIMESTAMP = "timestamp";
         static final String DATA = "data";
         static final String TOTAL = "total";
+        static final String TOTAL_IN_BYTES = "total_in_bytes";
         static final String IO_STATS = "io_stats";
+        static final String LEAST_ESTIMATE = "least_usage_estimate";
+        static final String MOST_ESTIMATE = "most_usage_estimate";
+        static final String USAGE_PERCENTAGE = "used_disk_percent";
+        static final String AVAILABLE = "available";
+        static final String AVAILABLE_IN_BYTES = "available_in_bytes";
+        static final String PATH = "path";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
index 4cdbed367c9..5897b84543e 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
@@ -22,6 +22,9 @@ package org.elasticsearch.monitor.fs;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.DiskUsage;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -48,7 +51,7 @@ public class FsProbe extends AbstractComponent {
         this.nodeEnv = nodeEnv;
     }
 
-    public FsInfo stats(FsInfo previous) throws IOException {
+    public FsInfo stats(FsInfo previous, @Nullable ClusterInfo clusterInfo) throws IOException {
         if (!nodeEnv.hasNodeFile()) {
             return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]);
         }
@@ -67,7 +70,13 @@ public class FsProbe extends AbstractComponent {
             }
             ioStats = ioStats(devicesNumbers, previous);
         }
-        return new FsInfo(System.currentTimeMillis(), ioStats, paths);
+        DiskUsage leastDiskEstimate = null;
+        DiskUsage mostDiskEstimate = null;
+        if (clusterInfo != null) {
+            leastDiskEstimate = clusterInfo.getNodeLeastAvailableDiskUsages().get(nodeEnv.nodeId());
+            mostDiskEstimate = clusterInfo.getNodeMostAvailableDiskUsages().get(nodeEnv.nodeId());
+        }
+        return new FsInfo(System.currentTimeMillis(), ioStats, paths, leastDiskEstimate, mostDiskEstimate);
     }
 
     final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers, final FsInfo previous) {
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index 96467b4d407..66534514b36 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.monitor.fs;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -27,6 +29,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.SingleObjectCache;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.cluster.ClusterInfoService;
 
 import java.io.IOException;
 
@@ -35,6 +38,7 @@ public class FsService extends AbstractComponent {
     private final FsProbe probe;
     private final TimeValue refreshInterval;
     private final SingleObjectCache<FsInfo> cache;
+    private final ClusterInfoService clusterInfoService;
 
     public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting(
@@ -43,21 +47,22 @@ public class FsService extends AbstractComponent {
             TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
-    public FsService(final Settings settings, final NodeEnvironment nodeEnvironment) {
+    public FsService(final Settings settings, final NodeEnvironment nodeEnvironment, ClusterInfoService clusterInfoService) {
         super(settings);
         this.probe = new FsProbe(settings, nodeEnvironment);
+        this.clusterInfoService = clusterInfoService;
         refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         logger.debug("using refresh_interval [{}]", refreshInterval);
-        cache = new FsInfoCache(refreshInterval, stats(probe, null, logger));
+        cache = new FsInfoCache(refreshInterval, stats(probe, null, logger, null));
     }
 
     public FsInfo stats() {
         return cache.getOrRefresh();
     }
 
-    private static FsInfo stats(FsProbe probe, FsInfo initialValue, Logger logger) {
+    private static FsInfo stats(FsProbe probe, FsInfo initialValue, Logger logger, @Nullable ClusterInfo clusterInfo) {
         try {
-            return probe.stats(initialValue);
+            return probe.stats(initialValue, clusterInfo);
         } catch (IOException e) {
             logger.debug("unexpected exception reading filesystem info", e);
             return null;
@@ -68,14 +73,14 @@ public class FsService extends AbstractComponent {
 
         private final FsInfo initialValue;
 
-        public FsInfoCache(TimeValue interval, FsInfo initialValue) {
+        FsInfoCache(TimeValue interval, FsInfo initialValue) {
             super(interval, initialValue);
             this.initialValue = initialValue;
         }
 
         @Override
         protected FsInfo refresh() {
-            return stats(probe, initialValue, logger);
+            return stats(probe, initialValue, logger, clusterInfoService.getClusterInfo());
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 3a19fe5bd00..f260d7430e2 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -71,7 +71,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
         final int infoThreshold;
         final int debugThreshold;
 
-        public GcOverheadThreshold(final int warnThreshold, final int infoThreshold, final int debugThreshold) {
+        GcOverheadThreshold(final int warnThreshold, final int infoThreshold, final int debugThreshold) {
             this.warnThreshold = warnThreshold;
             this.infoThreshold = infoThreshold;
             this.debugThreshold = debugThreshold;
@@ -355,7 +355,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
             final JvmStats currentJvmStats;
             final ByteSizeValue maxHeapUsed;
 
-            public SlowGcEvent(
+            SlowGcEvent(
                 final GarbageCollector currentGc,
                 final long collectionCount,
                 final TimeValue collectionTime,
@@ -380,7 +380,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
         private final Map<String, JvmGcMonitorService.GcThreshold> gcThresholds;
         final GcOverheadThreshold gcOverheadThreshold;
 
-        public JvmMonitor(final Map<String, GcThreshold> gcThresholds, final GcOverheadThreshold gcOverheadThreshold) {
+        JvmMonitor(final Map<String, GcThreshold> gcThresholds, final GcOverheadThreshold gcOverheadThreshold) {
             this.gcThresholds = Objects.requireNonNull(gcThresholds);
             this.gcOverheadThreshold = Objects.requireNonNull(gcOverheadThreshold);
         }
@@ -486,9 +486,9 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
             return System.nanoTime();
         }
 
-        abstract void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent);
+        abstract void onSlowGc(Threshold threshold, long seq, SlowGcEvent slowGcEvent);
 
-        abstract void onGcOverhead(final Threshold threshold, final long total, final long elapsed, final long seq);
+        abstract void onGcOverhead(Threshold threshold, long total, long elapsed, long seq);
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index 8d66ba41b30..0242e712cce 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -35,7 +35,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -191,6 +190,10 @@ public class OsProbe {
     // pattern for lines in /proc/self/cgroup
     private static final Pattern CONTROL_GROUP_PATTERN = Pattern.compile("\\d+:([^:,]+(?:,[^:,]+)?):(/.*)");
 
+    // this property is to support a hack to workaround an issue with Docker containers mounting the cgroups hierarchy inconsistently with
+    // respect to /proc/self/cgroup; for Docker containers this should be set to "/"
+    private static final String CONTROL_GROUPS_HIERARCHY_OVERRIDE = System.getProperty("es.cgroups.hierarchy.override");
+
     /**
      * A map of the control groups to which the Elasticsearch process belongs. Note that this is a map because the control groups can vary
      * from subsystem to subsystem. Additionally, this map can not be cached because a running process can be reclassified.
@@ -203,16 +206,22 @@ public class OsProbe {
         final Map<String, String> controllerMap = new HashMap<>();
         for (final String line : lines) {
             final Matcher matcher = CONTROL_GROUP_PATTERN.matcher(line);
-            // note that Matcher#matches must be invoked as
-            // matching is lazy; this can not happen in an assert
-            // as assertions might not be enabled
+            // Matcher#matches must be invoked as matching is lazy; this can not happen in an assert as assertions might not be enabled
             final boolean matches = matcher.matches();
             assert matches : line;
-            // at this point we have captured the subsystems and the
-            // control group
+            // at this point we have captured the subsystems and the control group
             final String[] controllers = matcher.group(1).split(",");
             for (final String controller : controllers) {
-                controllerMap.put(controller, matcher.group(2));
+                if (CONTROL_GROUPS_HIERARCHY_OVERRIDE != null) {
+                    /*
+                     * Docker violates the relationship between /proc/self/cgroup and the /sys/fs/cgroup hierarchy. It's possible that this
+                     * will be fixed in future versions of Docker with cgroup namespaces, but this requires modern kernels. Thus, we provide
+                     * an undocumented hack for overriding the control group path. Do not rely on this hack, it will be removed.
+                     */
+                    controllerMap.put(controller, CONTROL_GROUPS_HIERARCHY_OVERRIDE);
+                } else {
+                    controllerMap.put(controller, matcher.group(2));
+                }
             }
         }
         return controllerMap;
@@ -376,7 +385,7 @@ public class OsProbe {
      * @return {@code true} if the stats are available, otherwise {@code false}
      */
     @SuppressForbidden(reason = "access /proc/self/cgroup, /sys/fs/cgroup/cpu, and /sys/fs/cgroup/cpuacct")
-    protected boolean areCgroupStatsAvailable() {
+    boolean areCgroupStatsAvailable() {
         if (!Files.exists(PathUtils.get("/proc/self/cgroup"))) {
             return false;
         }
@@ -421,9 +430,7 @@ public class OsProbe {
                     cpuStat);
             }
         } catch (final IOException e) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("error reading control group stats", e);
-            }
+            logger.debug("error reading control group stats", e);
             return null;
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
index f37daddbb06..ef6270850a7 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
@@ -55,7 +55,7 @@ public class OsService extends AbstractComponent {
     }
 
     private class OsStatsCache extends SingleObjectCache<OsStats> {
-        public OsStatsCache(TimeValue interval, OsStats initValue) {
+        OsStatsCache(TimeValue interval, OsStats initValue) {
             super(interval, initValue);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
index 1cae0602ff2..7bebeb97586 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
@@ -54,7 +54,7 @@ public final class ProcessService extends AbstractComponent {
     }
 
     private class ProcessStatsCache extends SingleObjectCache<ProcessStats> {
-        public ProcessStatsCache(TimeValue interval, ProcessStats initValue) {
+        ProcessStatsCache(TimeValue interval, ProcessStats initValue) {
             super(interval, initValue);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
rename to core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
index 8fb86ebdac0..c61ffff7311 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java
@@ -17,15 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.node.internal;
-
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
+package org.elasticsearch.node;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -38,14 +30,24 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.UnaryOperator;
+
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
 
 import static org.elasticsearch.common.Strings.cleanPath;
 
 public class InternalSettingsPreparer {
 
     private static final String[] ALLOWED_SUFFIXES = {".yml", ".yaml", ".json"};
-    static final String PROPERTY_DEFAULTS_PREFIX = "default.";
-    static final Predicate<String> PROPERTY_DEFAULTS_PREDICATE = key -> key.startsWith(PROPERTY_DEFAULTS_PREFIX);
+    private static final String PROPERTY_DEFAULTS_PREFIX = "default.";
+    private static final Predicate<String> PROPERTY_DEFAULTS_PREDICATE = key -> key.startsWith(PROPERTY_DEFAULTS_PREFIX);
+    private static final UnaryOperator<String> STRIP_PROPERTY_DEFAULTS_PREFIX = key -> key.substring(PROPERTY_DEFAULTS_PREFIX.length());
 
     public static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
     public static final String TEXT_PROMPT_VALUE = "${prompt.text}";
@@ -55,7 +57,7 @@ public class InternalSettingsPreparer {
      */
     public static Settings prepareSettings(Settings input) {
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true, Collections.emptyMap());
+        initializeSettings(output, input, Collections.emptyMap());
         finalizeSettings(output, null);
         return output.build();
     }
@@ -86,9 +88,10 @@ public class InternalSettingsPreparer {
     public static Environment prepareEnvironment(Settings input, Terminal terminal, Map<String, String> properties) {
         // just create enough settings to build the environment, to get the config dir
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true, properties);
+        initializeSettings(output, input, properties);
         Environment environment = new Environment(output.build());
 
+        output = Settings.builder(); // start with a fresh output
         boolean settingsFileFound = false;
         Set<String> foundSuffixes = new HashSet<>();
         for (String allowedSuffix : ALLOWED_SUFFIXES) {
@@ -106,12 +109,12 @@ public class InternalSettingsPreparer {
             }
         }
         if (foundSuffixes.size() > 1) {
-            throw new SettingsException("multiple settings files found with suffixes: " + Strings.collectionToDelimitedString(foundSuffixes, ","));
+            throw new SettingsException("multiple settings files found with suffixes: "
+                + Strings.collectionToDelimitedString(foundSuffixes, ","));
         }
 
         // re-initialize settings now that the config file has been loaded
-        // TODO: only re-initialize if a config file was actually loaded
-        initializeSettings(output, input, false, properties);
+        initializeSettings(output, input, properties);
         finalizeSettings(output, terminal);
 
         environment = new Environment(output.build());
@@ -125,11 +128,11 @@ public class InternalSettingsPreparer {
      * Initializes the builder with the given input settings, and loads system properties settings if allowed.
      * If loadDefaults is true, system property default settings are loaded.
      */
-    private static void initializeSettings(Settings.Builder output, Settings input, boolean loadDefaults, Map<String, String> esSettings) {
+    private static void initializeSettings(Settings.Builder output, Settings input, Map<String, String> esSettings) {
         output.put(input);
-        if (loadDefaults) {
-            output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE, key -> key.substring(PROPERTY_DEFAULTS_PREFIX.length()));
-        }
+        output.putProperties(esSettings,
+            PROPERTY_DEFAULTS_PREDICATE.and(key -> output.get(STRIP_PROPERTY_DEFAULTS_PREFIX.apply(key)) == null),
+            STRIP_PROPERTY_DEFAULTS_PREFIX);
         output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE.negate(), Function.identity());
         output.replacePropertyPlaceholders();
     }
@@ -195,7 +198,9 @@ public class InternalSettingsPreparer {
 
     private static String promptForValue(String key, Terminal terminal, boolean secret) {
         if (terminal == null) {
-            throw new UnsupportedOperationException("found property [" + key + "] with value [" + (secret ? SECRET_PROMPT_VALUE : TEXT_PROMPT_VALUE) +"]. prompting for property values is only supported when running elasticsearch in the foreground");
+            throw new UnsupportedOperationException("found property [" + key + "] with value ["
+                + (secret ? SECRET_PROMPT_VALUE : TEXT_PROMPT_VALUE)
+                + "]. prompting for property values is only supported when running elasticsearch in the foreground");
         }
 
         if (secret) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index ec64c0f710a..9279451d470 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -22,6 +22,7 @@ package org.elasticsearch.node;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
@@ -47,7 +48,6 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
@@ -74,6 +74,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -83,7 +84,6 @@ import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.gateway.GatewayModule;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.gateway.MetaStateService;
-import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.indices.IndicesModule;
@@ -93,7 +93,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoverySettings;
@@ -101,8 +100,6 @@ import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
@@ -117,11 +114,10 @@ import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchExtRegistry;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.snapshots.SnapshotShardsService;
@@ -159,6 +155,8 @@ import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.util.stream.Collectors.toList;
+
 /**
  * A node represent a node within a cluster (<tt>cluster.name</tt>). The {@link #client()} can be used
  * in order to use a {@link Client} to perform actions/operations against the cluster.
@@ -183,7 +181,16 @@ public class Node implements Closeable {
     */
     public static final Setting<Boolean> NODE_LOCAL_STORAGE_SETTING = Setting.boolSetting("node.local_storage", true, Property.NodeScope);
     public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", Property.NodeScope);
-    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", Property.NodeScope);
+    public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", (settings) -> {
+        Map<String, String> settingsMap = settings.getAsMap();
+        for (Map.Entry<String, String> entry : settingsMap.entrySet()) {
+            String value = entry.getValue();
+            if (Character.isWhitespace(value.charAt(0)) || Character.isWhitespace(value.charAt(value.length() - 1))) {
+                throw new IllegalArgumentException("node.attr." + entry.getKey() + " cannot have leading or trailing whitespace " +
+                                                       "[" + value + "]");
+            }
+        }
+    }, Property.NodeScope);
     public static final Setting<String> BREAKER_TYPE_KEY = new Setting<>("indices.breaker.type", "hierarchy", (s) -> {
         switch (s) {
             case "hierarchy":
@@ -214,6 +221,7 @@ public class Node implements Closeable {
     private final PluginsService pluginsService;
     private final NodeClient client;
     private final Collection<LifecycleComponent> pluginLifecycleComponents;
+    private final LocalNodeFactory localNodeFactory;
 
     /**
      * Constructs a node with the given settings.
@@ -250,7 +258,6 @@ public class Node implements Closeable {
             } catch (IOException ex) {
                 throw new IllegalStateException("Failed to created node environment", ex);
             }
-
             final boolean hadPredefinedNodeName = NODE_NAME_SETTING.exists(tmpSettings);
             Logger logger = Loggers.getLogger(Node.class, tmpSettings);
             final String nodeId = nodeEnvironment.nodeId();
@@ -286,6 +293,8 @@ public class Node implements Closeable {
 
             this.pluginsService = new PluginsService(tmpSettings, environment.modulesFile(), environment.pluginsFile(), classpathPlugins);
             this.settings = pluginsService.updatedSettings();
+            localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId());
+
             // create the environment based on the finalized (processed) view of the settings
             // this is just to makes sure that people get the same settings, no matter where they ask them from
             this.environment = new Environment(this.settings);
@@ -320,12 +329,10 @@ public class Node implements Closeable {
             resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(settings,
                 getCustomNameResolvers(pluginsService.filterPlugins(DiscoveryPlugin.class)));
-            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
+            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool,
+                localNodeFactory::getNode);
             clusterService.addListener(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
-            final TribeService tribeService = new TribeService(settings, clusterService, nodeId,
-                s -> newTribeClientNode(s, classpathPlugins));
-            resourcesToClose.add(tribeService);
             final IngestService ingestService = new IngestService(settings, threadPool, this.environment,
                 scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class));
             final ClusterInfoService clusterInfoService = newClusterInfoService(settings, clusterService, threadPool, client);
@@ -335,22 +342,25 @@ public class Node implements Closeable {
             for (Module pluginModule : pluginsService.createGuiceModules()) {
                 modules.add(pluginModule);
             }
-            final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
+            final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool, clusterInfoService);
             modules.add(new NodeModule(this, monitorService));
             ClusterModule clusterModule = new ClusterModule(settings, clusterService,
                 pluginsService.filterPlugins(ClusterPlugin.class));
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
             modules.add(indicesModule);
-            SearchModule searchModule = new SearchModule(settings, false, pluginsService.filterPlugins(SearchPlugin.class));
-            ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
-                settingsModule.getClusterSettings(), threadPool, pluginsService.filterPlugins(ActionPlugin.class));
-            modules.add(actionModule);
-            modules.add(new GatewayModule());
-            modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class)));
+
+            SearchModule searchModule = new SearchModule(settings, false, pluginsService.filterPlugins(SearchPlugin.class), client);
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
+            ActionModule actionModule = new ActionModule(false, settings, clusterModule.getIndexNameExpressionResolver(),
+                    settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(),
+                    threadPool, pluginsService.filterPlugins(ActionPlugin.class), client, circuitBreakerService);
+            modules.add(actionModule);
+            modules.add(new GatewayModule());
+
+
             BigArrays bigArrays = createBigArrays(settings, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
@@ -359,51 +369,66 @@ public class Node implements Closeable {
                 indicesModule.getNamedWriteables().stream(),
                 searchModule.getNamedWriteables().stream(),
                 pluginsService.filterPlugins(Plugin.class).stream()
-                    .flatMap(p -> p.getNamedWriteables().stream()))
+                    .flatMap(p -> p.getNamedWriteables().stream()),
+                ClusterModule.getNamedWriteables().stream())
                 .flatMap(Function.identity()).collect(Collectors.toList());
             final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables);
-            final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment);
-            final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment,
-                settingsModule.getClusterSettings(), analysisModule.getAnalysisRegistry(), searchModule.getQueryParserRegistry(),
+            NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(
+                NetworkModule.getNamedXContents().stream(),
+                searchModule.getNamedXContents().stream(),
+                pluginsService.filterPlugins(Plugin.class).stream()
+                    .flatMap(p -> p.getNamedXContent().stream()),
+                ClusterModule.getNamedXWriteables().stream())
+                .flatMap(Function.identity()).collect(toList()));
+            final TribeService tribeService = new TribeService(settings, clusterService, nodeId, namedWriteableRegistry,
+                s -> newTribeClientNode(s, classpathPlugins));
+            resourcesToClose.add(tribeService);
+            modules.add(new RepositoriesModule(this.environment, pluginsService.filterPlugins(RepositoryPlugin.class), xContentRegistry));
+            final MetaStateService metaStateService = new MetaStateService(settings, nodeEnvironment, xContentRegistry);
+            final IndicesService indicesService = new IndicesService(settings, pluginsService, nodeEnvironment, xContentRegistry,
+                settingsModule.getClusterSettings(), analysisModule.getAnalysisRegistry(),
                 clusterModule.getIndexNameExpressionResolver(), indicesModule.getMapperRegistry(), namedWriteableRegistry,
                 threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptModule.getScriptService(),
                 clusterService, client, metaStateService);
 
             Collection<Object> pluginComponents = pluginsService.filterPlugins(Plugin.class).stream()
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
-                                                 scriptModule.getScriptService(), searchModule.getSearchRequestParsers()).stream())
+                                                 scriptModule.getScriptService(), xContentRegistry).stream())
                 .collect(Collectors.toList());
             Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
                 pluginsService.filterPlugins(Plugin.class).stream()
                 .map(Plugin::getCustomMetaDataUpgrader)
                 .collect(Collectors.toList());
-            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class), threadPool,
-                bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
+            final RestController restController = actionModule.getRestController();
+            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
+                    threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService,
+                    restController::dispatchRequest);
             final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = newTransportService(settings, transport, threadPool,
-                networkModule.getTransportInterceptor(), settingsModule.getClusterSettings());
+                networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
             final Consumer<Binder> httpBind;
+            final HttpServerTransport httpServerTransport;
             if (networkModule.isHttpEnabled()) {
-                HttpServerTransport httpServerTransport = networkModule.getHttpServerTransportSupplier().get();
-                HttpServer httpServer = new HttpServer(settings, httpServerTransport, actionModule.getRestController(), client,
-                    circuitBreakerService);
+                httpServerTransport = networkModule.getHttpServerTransportSupplier().get();
                 httpBind = b -> {
-                    b.bind(HttpServer.class).toInstance(httpServer);
                     b.bind(HttpServerTransport.class).toInstance(httpServerTransport);
                 };
             } else {
                 httpBind = b -> {
-                    b.bind(HttpServer.class).toProvider(Providers.of(null));
+                    b.bind(HttpServerTransport.class).toProvider(Providers.of(null));
                 };
+                httpServerTransport = null;
             }
-
             final DiscoveryModule discoveryModule = new DiscoveryModule(this.settings, threadPool, transportService,
-                networkService, clusterService, pluginsService.filterPlugins(DiscoveryPlugin.class));
+                namedWriteableRegistry, networkService, clusterService, pluginsService.filterPlugins(DiscoveryPlugin.class));
+            NodeService nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(),
+                transportService, indicesService, pluginsService, circuitBreakerService, scriptModule.getScriptService(),
+                httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter());
+
             modules.add(b -> {
-                    b.bind(IndicesQueriesRegistry.class).toInstance(searchModule.getQueryParserRegistry());
-                    b.bind(SearchRequestParsers.class).toInstance(searchModule.getSearchRequestParsers());
-                    b.bind(SearchExtRegistry.class).toInstance(searchModule.getSearchExtRegistry());
+                    b.bind(NodeService.class).toInstance(nodeService);
+                    b.bind(NamedXContentRegistry.class).toInstance(xContentRegistry);
                     b.bind(PluginsService.class).toInstance(pluginsService);
                     b.bind(Client.class).toInstance(client);
                     b.bind(NodeClient.class).toInstance(client);
@@ -423,16 +448,16 @@ public class Node implements Closeable {
                     b.bind(IndicesService.class).toInstance(indicesService);
                     b.bind(SearchService.class).toInstance(newSearchService(clusterService, indicesService,
                         threadPool, scriptModule.getScriptService(), bigArrays, searchModule.getFetchPhase()));
-                    b.bind(SearchTransportService.class).toInstance(new SearchTransportService(settings, transportService));
+                    b.bind(SearchTransportService.class).toInstance(new SearchTransportService(settings,
+                            settingsModule.getClusterSettings(), transportService));
                     b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings, bigArrays,
-                            scriptModule.getScriptService()));
+                            scriptModule.getScriptService(), searchModule.getSearchResponseListeners()));
                     b.bind(Transport.class).toInstance(transport);
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
-                    b.bind(AllocationCommandRegistry.class).toInstance(NetworkModule.getAllocationCommandRegistry());
                     b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
                     b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings,
-                        indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
+                        xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
                     b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
                     b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
                     {
@@ -456,8 +481,13 @@ public class Node implements Closeable {
                 .map(injector::getInstance).collect(Collectors.toList()));
             resourcesToClose.addAll(pluginLifecycleComponents);
             this.pluginLifecycleComponents = Collections.unmodifiableList(pluginLifecycleComponents);
-            client.initialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}));
+            client.initialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}),
+                    () -> clusterService.localNode().getId());
 
+            if (NetworkModule.HTTP_ENABLED.get(settings)) {
+                logger.debug("initializing HTTP handlers ...");
+                actionModule.initRestHandlers(() -> clusterService.state().nodes());
+            }
             logger.info("initialized");
 
             success = true;
@@ -484,8 +514,10 @@ public class Node implements Closeable {
     }
 
     protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool,
-                                                   TransportInterceptor interceptor, ClusterSettings clusterSettings) {
-        return new TransportService(settings, transport, threadPool, interceptor, clusterSettings);
+                                                   TransportInterceptor interceptor,
+                                                   Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                                   ClusterSettings clusterSettings) {
+        return new TransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
     }
 
     protected void processRecoverySettings(ClusterSettings clusterSettings, RecoverySettings recoverySettings) {
@@ -556,6 +588,7 @@ public class Node implements Closeable {
         injector.getInstance(ResourceWatcherService.class).start();
         injector.getInstance(GatewayService.class).start();
         Discovery discovery = injector.getInstance(Discovery.class);
+        clusterService.setDiscoverySettings(discovery.getDiscoverySettings());
         clusterService.addInitialStateBlock(discovery.getDiscoverySettings().getNoMasterBlock());
         clusterService.setClusterStatePublisher(discovery::publish);
 
@@ -570,17 +603,13 @@ public class Node implements Closeable {
         validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress(), pluginsService.filterPlugins(Plugin.class).stream()
             .flatMap(p -> p.getBootstrapChecks().stream()).collect(Collectors.toList()));
 
-        DiscoveryNode localNode = DiscoveryNode.createLocal(settings,
-            transportService.boundAddress().publishAddress(), injector.getInstance(NodeEnvironment.class).nodeId());
-
-        // TODO: need to find a cleaner way to start/construct a service with some initial parameters,
-        // playing nice with the life cycle interfaces
-        clusterService.setLocalNode(localNode);
-        transportService.setLocalNode(localNode);
         clusterService.addStateApplier(transportService.getTaskManager());
-
         clusterService.start();
-
+        assert localNodeFactory.getNode() != null;
+        assert transportService.getLocalNode().equals(localNodeFactory.getNode())
+            : "transportService has a different local node than the factory provided";
+        assert clusterService.localNode().equals(localNodeFactory.getNode())
+            : "clusterService has a different local node than the factory provided";
         // start after cluster service so the local disco is known
         discovery.start();
         transportService.acceptIncomingRequests();
@@ -589,8 +618,9 @@ public class Node implements Closeable {
         final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings);
         if (initialStateTimeout.millis() > 0) {
             final ThreadPool thread = injector.getInstance(ThreadPool.class);
-            ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, thread.getThreadContext());
-            if (observer.observedState().nodes().getMasterNodeId() == null) {
+            ClusterState clusterState = clusterService.state();
+            ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, null, logger, thread.getThreadContext());
+            if (clusterState.nodes().getMasterNodeId() == null) {
                 logger.debug("waiting to join the cluster. timeout [{}]", initialStateTimeout);
                 final CountDownLatch latch = new CountDownLatch(1);
                 observer.waitForNextChange(new ClusterStateObserver.Listener() {
@@ -618,12 +648,16 @@ public class Node implements Closeable {
             }
         }
 
+
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            injector.getInstance(HttpServer.class).start();
+            injector.getInstance(HttpServerTransport.class).start();
         }
 
         // start nodes now, after the http server, because it may take some time
         tribeService.startNodes();
+        // starts connecting to remote clusters if any cluster is configured
+        SearchTransportService searchTransportService = injector.getInstance(SearchTransportService.class);
+        searchTransportService.start();
 
         if (WRITE_PORTS_FIELD_SETTING.get(settings)) {
             if (NetworkModule.HTTP_ENABLED.get(settings)) {
@@ -649,7 +683,7 @@ public class Node implements Closeable {
         injector.getInstance(TribeService.class).stop();
         injector.getInstance(ResourceWatcherService.class).stop();
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            injector.getInstance(HttpServer.class).stop();
+            injector.getInstance(HttpServerTransport.class).stop();
         }
 
         injector.getInstance(SnapshotsService.class).stop();
@@ -667,6 +701,7 @@ public class Node implements Closeable {
         injector.getInstance(GatewayService.class).stop();
         injector.getInstance(SearchService.class).stop();
         injector.getInstance(TransportService.class).stop();
+        injector.getInstance(SearchTransportService.class).stop();
 
         pluginLifecycleComponents.forEach(LifecycleComponent::stop);
         // we should stop this last since it waits for resources to get released
@@ -699,7 +734,7 @@ public class Node implements Closeable {
         toClose.add(injector.getInstance(NodeService.class));
         toClose.add(() -> stopWatch.stop().start("http"));
         if (NetworkModule.HTTP_ENABLED.get(settings)) {
-            toClose.add(injector.getInstance(HttpServer.class));
+            toClose.add(injector.getInstance(HttpServerTransport.class));
         }
         toClose.add(() -> stopWatch.stop().start("snapshot_service"));
         toClose.add(injector.getInstance(SnapshotsService.class));
@@ -728,6 +763,8 @@ public class Node implements Closeable {
         toClose.add(injector.getInstance(SearchService.class));
         toClose.add(() -> stopWatch.stop().start("transport"));
         toClose.add(injector.getInstance(TransportService.class));
+        toClose.add(() -> stopWatch.stop().start("search_transport_service"));
+        toClose.add(injector.getInstance(SearchTransportService.class));
 
         for (LifecycleComponent plugin : pluginLifecycleComponents) {
             toClose.add(() -> stopWatch.stop().start("plugin(" + plugin.getClass().getName() + ")"));
@@ -879,4 +916,26 @@ public class Node implements Closeable {
                                                        ThreadPool threadPool, NodeClient client) {
         return new InternalClusterInfoService(settings, clusterService, threadPool, client);
     }
+
+    private static class LocalNodeFactory implements Function<BoundTransportAddress, DiscoveryNode> {
+        private final SetOnce<DiscoveryNode> localNode = new SetOnce<>();
+        private final String persistentNodeId;
+        private final Settings settings;
+
+        private LocalNodeFactory(Settings settings, String persistentNodeId) {
+            this.persistentNodeId = persistentNodeId;
+            this.settings = settings;
+        }
+
+        @Override
+        public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) {
+            localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId));
+            return localNode.get();
+        }
+
+        DiscoveryNode getNode() {
+            assert localNode.get() != null;
+            return localNode.get();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 6a8f8b90681..929e889503e 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -22,7 +22,6 @@ package org.elasticsearch.node;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.monitor.MonitorService;
-import org.elasticsearch.node.service.NodeService;
 
 public class NodeModule extends AbstractModule {
 
@@ -38,7 +37,6 @@ public class NodeModule extends AbstractModule {
     protected void configure() {
         bind(Node.class).toInstance(node);
         bind(MonitorService.class).toInstance(monitorService);
-        bind(NodeService.class).asEagerSingleton();
         bind(DiskThresholdMonitor.class).asEagerSingleton();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/NodeService.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/node/service/NodeService.java
rename to core/src/main/java/org/elasticsearch/node/NodeService.java
index 7d9a148b271..cb245487152 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeService.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.node.service;
+package org.elasticsearch.node;
 
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
@@ -27,11 +27,10 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.http.HttpServer;
+import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.ingest.IngestService;
@@ -55,17 +54,16 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final IngestService ingestService;
     private final SettingsFilter settingsFilter;
     private ScriptService scriptService;
+    private final HttpServerTransport httpServerTransport;
 
-    @Nullable
-    private final HttpServer httpServer;
 
     private final Discovery discovery;
 
-    @Inject
-    public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
+    NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
                        TransportService transportService, IndicesService indicesService, PluginsService pluginService,
-                       CircuitBreakerService circuitBreakerService, ScriptService scriptService, @Nullable HttpServer httpServer,
-                       IngestService ingestService, ClusterService clusterService, SettingsFilter settingsFilter) {
+                       CircuitBreakerService circuitBreakerService, ScriptService scriptService,
+                       @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService,
+                       SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
         this.monitorService = monitorService;
@@ -74,7 +72,7 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.discovery = discovery;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
-        this.httpServer = httpServer;
+        this.httpServerTransport = httpServerTransport;
         this.ingestService = ingestService;
         this.settingsFilter = settingsFilter;
         this.scriptService = scriptService;
@@ -91,7 +89,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 jvm ? monitorService.jvmService().info() : null,
                 threadPool ? this.threadPool.info() : null,
                 transport ? transportService.info() : null,
-                http ? (httpServer == null ? null : httpServer.info()) : null,
+                http ? (httpServerTransport == null ? null : httpServerTransport.info()) : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
                 ingest ? (ingestService == null ? null : ingestService.info()) : null,
                 indices ? indicesService.getTotalIndexingBufferBytes() : null
@@ -111,7 +109,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 threadPool ? this.threadPool.stats() : null,
                 fs ? monitorService.fsService().stats() : null,
                 transport ? transportService.stats() : null,
-                http ? (httpServer == null ? null : httpServer.stats()) : null,
+                http ? (httpServerTransport == null ? null : httpServerTransport.stats()) : null,
                 circuitBreaker ? circuitBreakerService.stats() : null,
                 script ? scriptService.stats() : null,
                 discoveryStats ? discovery.stats() : null,
diff --git a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
index 782ff19c34f..346bf491d61 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
@@ -25,14 +25,23 @@ import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 
 /**
@@ -63,7 +72,9 @@ public interface ActionPlugin {
     /**
      * Rest handlers added by this plugin.
      */
-    default List<Class<? extends RestHandler>> getRestHandlers() {
+    default List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+            IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
         return Collections.emptyList();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java b/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
index 37b97855084..61e87d83a18 100644
--- a/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.function.Supplier;
 
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
@@ -60,6 +61,7 @@ public interface DiscoveryPlugin {
      * @param hostsProvider Use to find configured hosts which should be pinged for initial discovery
      */
     default Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                               NamedWriteableRegistry namedWriteableRegistry,
                                                                ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         return Collections.emptyMap();
     }
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 8d79b0543f0..00e10162b29 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -27,15 +27,13 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -63,7 +61,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
@@ -103,7 +100,7 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
  * elasticsearch config directory, using the name of the plugin. If any files to be installed
  * already exist, they will be skipped.
  */
-class InstallPluginCommand extends SettingCommand {
+class InstallPluginCommand extends EnvironmentAwareCommand {
 
     private static final String PROPERTY_STAGING_ID = "es.plugins.staging";
 
@@ -189,18 +186,17 @@ class InstallPluginCommand extends SettingCommand {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         String pluginId = arguments.value(options);
         boolean isBatch = options.has(batchOption) || System.console() == null;
-        execute(terminal, pluginId, isBatch, settings);
+        execute(terminal, pluginId, isBatch, env);
     }
 
     // pkg private for testing
-    void execute(Terminal terminal, String pluginId, boolean isBatch, Map<String, String> settings) throws Exception {
+    void execute(Terminal terminal, String pluginId, boolean isBatch, Environment env) throws Exception {
         if (pluginId == null) {
             throw new UserException(ExitCodes.USAGE, "plugin id is required");
         }
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
         // TODO: remove this leniency!! is it needed anymore?
         if (Files.exists(env.pluginsFile()) == false) {
             terminal.println("Plugins directory [" + env.pluginsFile() + "] does not exist. Creating...");
@@ -292,7 +288,7 @@ class InstallPluginCommand extends SettingCommand {
         private int width = 50;
         private final boolean enabled;
 
-        public TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
+        TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
             super(is, expectedTotalSize);
             this.terminal = terminal;
             this.enabled = expectedTotalSize > 0;
@@ -419,6 +415,18 @@ class InstallPluginCommand extends SettingCommand {
     private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
         // read and validate the plugin descriptor
         PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
+
+        // checking for existing version of the plugin
+        final Path destination = env.pluginsFile().resolve(info.getName());
+        if (Files.exists(destination)) {
+            final String message = String.format(
+                Locale.ROOT,
+                "plugin directory [%s] already exists; if you need to update the plugin, uninstall it first using command 'remove %s'",
+                destination.toAbsolutePath(),
+                info.getName());
+            throw new UserException(ExitCodes.CONFIG, message);
+        }
+
         terminal.println(VERBOSE, info.toString());
 
         // don't let user install plugin as a module...
@@ -472,14 +480,7 @@ class InstallPluginCommand extends SettingCommand {
 
         try {
             PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
-
             final Path destination = env.pluginsFile().resolve(info.getName());
-            if (Files.exists(destination)) {
-                throw new UserException(
-                    ExitCodes.USAGE,
-                    "plugin directory " + destination.toAbsolutePath() +
-                        " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command");
-            }
 
             Path tmpBinDir = tmpRoot.resolve("bin");
             if (Files.exists(tmpBinDir)) {
diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
index bd2f853bac0..a674e7c6e24 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -20,11 +20,9 @@
 package org.elasticsearch.plugins;
 
 import joptsimple.OptionSet;
-import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
@@ -33,20 +31,18 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 /**
  * A command for the plugin cli to list plugins installed in elasticsearch.
  */
-class ListPluginsCommand extends SettingCommand {
+class ListPluginsCommand extends EnvironmentAwareCommand {
 
     ListPluginsCommand() {
         super("Lists installed elasticsearch plugins");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         if (Files.exists(env.pluginsFile()) == false) {
             throw new IOException("Plugins directory missing: " + env.pluginsFile());
         }
diff --git a/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
index 32d0e6058ea..33fab61c24a 100644
--- a/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/NetworkPlugin.java
@@ -27,6 +27,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -41,8 +43,13 @@ public interface NetworkPlugin {
     /**
      * Returns a list of {@link TransportInterceptor} instances that are used to intercept incoming and outgoing
      * transport (inter-node) requests. This must not return <code>null</code>
+     *
+     * @param namedWriteableRegistry registry of all named writeables registered
+     * @param threadContext a {@link ThreadContext} of the current nodes or clients {@link ThreadPool} that can be used to set additional
+     *                      headers in the interceptors
      */
-    default List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+    default List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                ThreadContext threadContext) {
         return Collections.emptyList();
     }
 
@@ -62,9 +69,11 @@ public interface NetworkPlugin {
      * See {@link org.elasticsearch.common.network.NetworkModule#HTTP_TYPE_SETTING} to configure a specific implementation.
      */
     default Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
-                                                               CircuitBreakerService circuitBreakerService,
-                                                               NamedWriteableRegistry namedWriteableRegistry,
-                                                               NetworkService networkService) {
+                                                                         CircuitBreakerService circuitBreakerService,
+                                                                         NamedWriteableRegistry namedWriteableRegistry,
+                                                                         NamedXContentRegistry xContentRegistry,
+                                                                         NetworkService networkService,
+                                                                         HttpServerTransport.Dispatcher dispatcher) {
         return Collections.emptyMap();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 32d902248a6..87c5ef9a8c6 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -19,12 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
@@ -39,6 +33,8 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.indices.analysis.AnalysisModule;
@@ -46,11 +42,15 @@ import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.function.UnaryOperator;
 
@@ -102,11 +102,10 @@ public abstract class Plugin implements Closeable {
      * @param threadPool A service to allow retrieving an executor to run an async action
      * @param resourceWatcherService A service to watch for changes to node local files
      * @param scriptService A service to allow running scripts on the local node
-     * @param searchRequestParsers Parsers for search requests which may be used to templatize search requests
      */
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
                                                ResourceWatcherService resourceWatcherService, ScriptService scriptService,
-                                               SearchRequestParsers searchRequestParsers) {
+                                               NamedXContentRegistry xContentRegistry) {
         return Collections.emptyList();
     }
 
@@ -126,6 +125,14 @@ public abstract class Plugin implements Closeable {
         return Collections.emptyList();
     }
 
+    /**
+     * Returns parsers for named objects this plugin will parse from {@link XContentParser#namedObject(Class, String, Object)}.
+     * @see NamedWriteableRegistry
+     */
+    public List<NamedXContentRegistry.Entry> getNamedXContent() {
+        return Collections.emptyList();
+    }
+
     /**
      * Called before a new index is created on a node. The given module can be used to register index-level
      * extensions.
diff --git a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
index 16e1f203bb3..64d9437fa24 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
@@ -37,7 +37,7 @@ abstract class ProgressInputStream extends FilterInputStream {
     private int currentPercent;
     private int count = 0;
 
-    public ProgressInputStream(InputStream is, int expectedTotalSize) {
+    ProgressInputStream(InputStream is, int expectedTotalSize) {
         super(is);
         this.expectedTotalSize = expectedTotalSize;
         this.currentPercent = 0;
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index 2eaa25ca39c..8a8faf3174e 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -19,32 +19,28 @@
 
 package org.elasticsearch.plugins;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cli.EnvironmentAwareCommand;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.env.Environment;
 
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
  * A command for the plugin cli to remove a plugin from elasticsearch.
  */
-class RemovePluginCommand extends SettingCommand {
+class RemovePluginCommand extends EnvironmentAwareCommand {
 
     private final OptionSpec<String> arguments;
 
@@ -54,14 +50,16 @@ class RemovePluginCommand extends SettingCommand {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         String arg = arguments.value(options);
-        execute(terminal, arg, settings);
+        execute(terminal, arg, env);
     }
 
     // pkg private for testing
-    void execute(Terminal terminal, String pluginName, Map<String, String> settings) throws Exception {
-        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+    void execute(Terminal terminal, String pluginName, Environment env) throws Exception {
+        if (pluginName == null) {
+            throw new UserException(ExitCodes.USAGE, "plugin name is required");
+        }
 
         terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "...");
 
diff --git a/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
index 9306ee37076..a3af52a9a4a 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
@@ -22,6 +22,7 @@ package org.elasticsearch.plugins;
 import java.util.Collections;
 import java.util.Map;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.Repository;
 
@@ -38,7 +39,7 @@ public interface RepositoryPlugin {
      * The key of the returned {@link Map} is the type name of the repository and
      * the value is a factory to construct the {@link Repository} interface.
      */
-    default Map<String, Repository.Factory> getRepositories(Environment env) {
+    default Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.emptyMap();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
index 364454de0c5..01685535a4e 100644
--- a/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/SearchPlugin.java
@@ -20,18 +20,21 @@
 package org.elasticsearch.plugins;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
 import org.elasticsearch.search.SearchExtBuilder;
-import org.elasticsearch.search.SearchExtParser;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -46,10 +49,13 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.suggest.Suggester;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -85,7 +91,7 @@ public interface SearchPlugin {
         return emptyList();
     }
     /**
-     * The new {@link SearchExtParser}s defined by this plugin.
+     * The new {@link SearchExtBuilder}s defined by this plugin.
      */
     default List<SearchExtSpec<?>> getSearchExts() {
         return emptyList();
@@ -99,8 +105,8 @@ public interface SearchPlugin {
     /**
      * The new {@link Suggester}s defined by this plugin.
      */
-    default Map<String, Suggester<?>> getSuggesters() {
-        return emptyMap();
+    default List<SuggesterSpec<?>> getSuggesters() {
+        return emptyList();
     }
     /**
      * The new {@link Query}s defined by this plugin.
@@ -120,6 +126,15 @@ public interface SearchPlugin {
     default List<PipelineAggregationSpec> getPipelineAggregations() {
         return emptyList();
     }
+    /**
+     * The new search response listeners in the form of {@link BiConsumer}s added by this plugin.
+     * The listeners are invoked on the coordinating node, at the very end of the search request.
+     * This provides a convenient location if you wish to inspect/modify the final response (took time, etc).
+     * The BiConsumers are passed the original {@link SearchRequest} and the final {@link SearchResponse}
+     */
+    default List<BiConsumer<SearchRequest, SearchResponse>> getSearchResponseListeners() {
+        return emptyList();
+    }
 
     /**
      * Specification of custom {@link ScoreFunction}.
@@ -134,6 +149,38 @@ public interface SearchPlugin {
         }
     }
 
+    /**
+     * Specification for a {@link Suggester}.
+     */
+    class SuggesterSpec<T extends SuggestionBuilder<T>> extends SearchExtensionSpec<T, CheckedFunction<XContentParser, T, IOException>> {
+        /**
+         * Specification of custom {@link Suggester}.
+         *
+         * @param name holds the names by which this suggester might be parsed. The {@link ParseField#getPreferredName()} is special as it
+         *        is the name by under which the reader is registered. So it is the name that the query should use as its
+         *        {@link NamedWriteable#getWriteableName()} too.
+         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the query suggester from xcontent
+         */
+        public SuggesterSpec(ParseField name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+
+        /**
+         * Specification of custom {@link Suggester}.
+         *
+         * @param name the name by which this suggester might be parsed or deserialized. Make sure that the query builder returns this name
+         *        for {@link NamedWriteable#getWriteableName()}.
+         * @param reader the reader registered for this suggester's builder. Typically a reference to a constructor that takes a
+         *        {@link StreamInput}
+         * @param parser the parser the reads the suggester builder from xcontent
+         */
+        public SuggesterSpec(String name, Writeable.Reader<T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
+            super(name, reader, parser);
+        }
+    }
+
     /**
      * Specification of custom {@link Query}.
      */
@@ -302,12 +349,13 @@ public interface SearchPlugin {
      * Specification for a {@link SearchExtBuilder} which represents an additional section that can be
      * parsed in a search request (within the ext element).
      */
-    class SearchExtSpec<T extends SearchExtBuilder> extends SearchExtensionSpec<T, SearchExtParser<T>> {
-        public SearchExtSpec(ParseField name, Writeable.Reader<? extends T> reader, SearchExtParser<T> parser) {
+    class SearchExtSpec<T extends SearchExtBuilder> extends SearchExtensionSpec<T, CheckedFunction<XContentParser, T, IOException>> {
+        public SearchExtSpec(ParseField name, Writeable.Reader<? extends T> reader,
+                CheckedFunction<XContentParser, T, IOException> parser) {
             super(name, reader, parser);
         }
 
-        public SearchExtSpec(String name, Writeable.Reader<? extends T> reader, SearchExtParser<T> parser) {
+        public SearchExtSpec(String name, Writeable.Reader<? extends T> reader, CheckedFunction<XContentParser, T, IOException> parser) {
             super(name, reader, parser);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
index 50ab90b4fe1..3073e1f0b0e 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -19,23 +19,22 @@
 
 package org.elasticsearch.repositories;
 
+import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
+import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.multibindings.MapBinder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.snapshots.SnapshotShardsService;
+import org.elasticsearch.snapshots.SnapshotsService;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
-import org.elasticsearch.common.inject.multibindings.MapBinder;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.fs.FsRepository;
-import org.elasticsearch.repositories.uri.URLRepository;
-import org.elasticsearch.snapshots.RestoreService;
-import org.elasticsearch.snapshots.SnapshotShardsService;
-import org.elasticsearch.snapshots.SnapshotsService;
-
 /**
  * Sets up classes for Snapshot/Restore.
  */
@@ -43,13 +42,12 @@ public class RepositoriesModule extends AbstractModule {
 
     private final Map<String, Repository.Factory> repositoryTypes;
 
-    public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins) {
+    public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins, NamedXContentRegistry namedXContentRegistry) {
         Map<String, Repository.Factory> factories = new HashMap<>();
-        factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env));
-        factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env));
+        factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry));
 
         for (RepositoryPlugin repoPlugin : repoPlugins) {
-            Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env);
+            Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry);
             for (Map.Entry<String, Repository.Factory> entry : newRepoTypes.entrySet()) {
                 if (factories.put(entry.getKey(), entry.getValue()) != null) {
                     throw new IllegalArgumentException("Repository type [" + entry.getKey() + "] is already registered");
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 300ba811a1b..7aea5d18afb 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -401,7 +401,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
 
         private final ActionListener<ClusterStateUpdateResponse> listener;
 
-        public VerifyingRegisterRepositoryListener(String name, final ActionListener<ClusterStateUpdateResponse> listener) {
+        VerifyingRegisterRepositoryListener(String name, final ActionListener<ClusterStateUpdateResponse> listener) {
             this.name = name;
             this.listener = listener;
         }
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index b1f534bf684..462b7ea1dab 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -115,16 +115,19 @@ public interface Repository extends LifecycleComponent {
      * @param failure       global failure reason or null
      * @param totalShards   total number of shards
      * @param shardFailures list of shard failures
+     * @param repositoryStateId the unique id identifying the state of the repository when the snapshot began
      * @return snapshot description
      */
-    SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures);
+    SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
+                                  List<SnapshotShardFailure> shardFailures, long repositoryStateId);
 
     /**
      * Deletes snapshot
      *
      * @param snapshotId snapshot id
+     * @param repositoryStateId the unique id identifying the state of the repository when the snapshot deletion began
      */
-    void deleteSnapshot(SnapshotId snapshotId);
+    void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId);
 
     /**
      * Returns snapshot throttle time in nanoseconds
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
index 4927e2b41b7..f823da0f2e5 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoryData.java
@@ -42,10 +42,22 @@ import java.util.stream.Collectors;
  * A class that represents the data in a repository, as captured in the
  * repository's index blob.
  */
-public final class RepositoryData implements ToXContent {
+public final class RepositoryData {
 
-    public static final RepositoryData EMPTY = new RepositoryData(Collections.emptyList(), Collections.emptyMap());
+    /**
+     * The generation value indicating the repository has no index generational files.
+     */
+    public static final long EMPTY_REPO_GEN = -1L;
+    /**
+     * An instance initialized for an empty repository.
+     */
+    public static final RepositoryData EMPTY =
+        new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), Collections.emptyMap(), Collections.emptyList());
 
+    /**
+     * The generational id of the index file from which the repository data was read.
+     */
+    private final long genId;
     /**
      * The ids of the snapshots in the repository.
      */
@@ -58,17 +70,31 @@ public final class RepositoryData implements ToXContent {
      * The snapshots that each index belongs to.
      */
     private final Map<IndexId, Set<SnapshotId>> indexSnapshots;
+    /**
+     * The snapshots that are no longer compatible with the current cluster ES version.
+     */
+    private final List<SnapshotId> incompatibleSnapshotIds;
 
-    public RepositoryData(List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots) {
+    public RepositoryData(long genId, List<SnapshotId> snapshotIds, Map<IndexId, Set<SnapshotId>> indexSnapshots,
+                          List<SnapshotId> incompatibleSnapshotIds) {
+        this.genId = genId;
         this.snapshotIds = Collections.unmodifiableList(snapshotIds);
         this.indices = Collections.unmodifiableMap(indexSnapshots.keySet()
                                                        .stream()
                                                        .collect(Collectors.toMap(IndexId::getName, Function.identity())));
         this.indexSnapshots = Collections.unmodifiableMap(indexSnapshots);
+        this.incompatibleSnapshotIds = Collections.unmodifiableList(incompatibleSnapshotIds);
     }
 
     protected RepositoryData copy() {
-        return new RepositoryData(snapshotIds, indexSnapshots);
+        return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
+    }
+
+    /**
+     * Gets the generational index file id from which this instance was read.
+     */
+    public long getGenId() {
+        return genId;
     }
 
     /**
@@ -78,6 +104,25 @@ public final class RepositoryData implements ToXContent {
         return snapshotIds;
     }
 
+    /**
+     * Returns an immutable collection of the snapshot ids in the repository that are incompatible with the
+     * current ES version.
+     */
+    public List<SnapshotId> getIncompatibleSnapshotIds() {
+        return incompatibleSnapshotIds;
+    }
+
+    /**
+     * Returns an immutable collection of all the snapshot ids in the repository, both active and
+     * incompatible snapshots.
+     */
+    public List<SnapshotId> getAllSnapshotIds() {
+        List<SnapshotId> allSnapshotIds = new ArrayList<>(snapshotIds.size() + incompatibleSnapshotIds.size());
+        allSnapshotIds.addAll(snapshotIds);
+        allSnapshotIds.addAll(incompatibleSnapshotIds);
+        return Collections.unmodifiableList(allSnapshotIds);
+    }
+
     /**
      * Returns an unmodifiable map of the index names to {@link IndexId} in the repository.
      */
@@ -91,7 +136,10 @@ public final class RepositoryData implements ToXContent {
      */
     public RepositoryData addSnapshot(final SnapshotId snapshotId, final List<IndexId> snapshottedIndices) {
         if (snapshotIds.contains(snapshotId)) {
-            throw new IllegalArgumentException("[" + snapshotId + "] already exists in the repository data");
+            // if the snapshot id already exists in the repository data, it means an old master
+            // that is blocked from the cluster is trying to finalize a snapshot concurrently with
+            // the new master, so we make the operation idempotent
+            return this;
         }
         List<SnapshotId> snapshots = new ArrayList<>(snapshotIds);
         snapshots.add(snapshotId);
@@ -110,14 +158,7 @@ public final class RepositoryData implements ToXContent {
                 allIndexSnapshots.put(indexId, ids);
             }
         }
-        return new RepositoryData(snapshots, allIndexSnapshots);
-    }
-
-    /**
-     * Initializes the indices in the repository metadata; returns a new instance.
-     */
-    public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
-        return new RepositoryData(snapshotIds, indexSnapshots);
+        return new RepositoryData(genId, snapshots, allIndexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
@@ -146,7 +187,21 @@ public final class RepositoryData implements ToXContent {
             indexSnapshots.put(indexId, set);
         }
 
-        return new RepositoryData(newSnapshotIds, indexSnapshots);
+        return new RepositoryData(genId, newSnapshotIds, indexSnapshots, incompatibleSnapshotIds);
+    }
+
+    /**
+     * Returns a new {@link RepositoryData} instance containing the same snapshot data as the
+     * invoking instance, with the given incompatible snapshots added to the new instance.
+     */
+    public RepositoryData addIncompatibleSnapshots(final List<SnapshotId> incompatibleSnapshotIds) {
+        List<SnapshotId> newSnapshotIds = new ArrayList<>(this.snapshotIds);
+        List<SnapshotId> newIncompatibleSnapshotIds = new ArrayList<>(this.incompatibleSnapshotIds);
+        for (SnapshotId snapshotId : incompatibleSnapshotIds) {
+            newSnapshotIds.remove(snapshotId);
+            newIncompatibleSnapshotIds.add(snapshotId);
+        }
+        return new RepositoryData(this.genId, newSnapshotIds, this.indexSnapshots, newIncompatibleSnapshotIds);
     }
 
     /**
@@ -160,6 +215,13 @@ public final class RepositoryData implements ToXContent {
         return snapshotIds;
     }
 
+    /**
+     * Initializes the indices in the repository metadata; returns a new instance.
+     */
+    public RepositoryData initIndices(final Map<IndexId, Set<SnapshotId>> indexSnapshots) {
+        return new RepositoryData(genId, snapshotIds, indexSnapshots, incompatibleSnapshotIds);
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (this == obj) {
@@ -171,12 +233,13 @@ public final class RepositoryData implements ToXContent {
         @SuppressWarnings("unchecked") RepositoryData that = (RepositoryData) obj;
         return snapshotIds.equals(that.snapshotIds)
                    && indices.equals(that.indices)
-                   && indexSnapshots.equals(that.indexSnapshots);
+                   && indexSnapshots.equals(that.indexSnapshots)
+                   && incompatibleSnapshotIds.equals(that.incompatibleSnapshotIds);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(snapshotIds, indices, indexSnapshots);
+        return Objects.hash(snapshotIds, indices, indexSnapshots, incompatibleSnapshotIds);
     }
 
     /**
@@ -225,11 +288,15 @@ public final class RepositoryData implements ToXContent {
     }
 
     private static final String SNAPSHOTS = "snapshots";
+    private static final String INCOMPATIBLE_SNAPSHOTS = "incompatible-snapshots";
     private static final String INDICES = "indices";
     private static final String INDEX_ID = "id";
 
-    @Override
-    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+    /**
+     * Writes the snapshots metadata and the related indices metadata to x-content, omitting the
+     * incompatible snapshots.
+     */
+    public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
         builder.startObject();
         // write the snapshots list
         builder.startArray(SNAPSHOTS);
@@ -256,7 +323,10 @@ public final class RepositoryData implements ToXContent {
         return builder;
     }
 
-    public static RepositoryData fromXContent(final XContentParser parser) throws IOException {
+    /**
+     * Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
+     */
+    public static RepositoryData snapshotsFromXContent(final XContentParser parser, long genId) throws IOException {
         List<SnapshotId> snapshots = new ArrayList<>();
         Map<IndexId, Set<SnapshotId>> indexSnapshots = new HashMap<>();
         if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
@@ -305,7 +375,51 @@ public final class RepositoryData implements ToXContent {
         } else {
             throw new ElasticsearchParseException("start object expected");
         }
-        return new RepositoryData(snapshots, indexSnapshots);
+        return new RepositoryData(genId, snapshots, indexSnapshots, Collections.emptyList());
+    }
+
+    /**
+     * Writes the incompatible snapshot ids to x-content.
+     */
+    public XContentBuilder incompatibleSnapshotsToXContent(final XContentBuilder builder, final ToXContent.Params params)
+        throws IOException {
+
+        builder.startObject();
+        // write the incompatible snapshots list
+        builder.startArray(INCOMPATIBLE_SNAPSHOTS);
+        for (final SnapshotId snapshot : getIncompatibleSnapshotIds()) {
+            snapshot.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Reads the incompatible snapshot ids from x-content, loading them into a new instance of {@link RepositoryData}
+     * that is created from the invoking instance, plus the incompatible snapshots that are read from x-content.
+     */
+    public RepositoryData incompatibleSnapshotsFromXContent(final XContentParser parser) throws IOException {
+        List<SnapshotId> incompatibleSnapshotIds = new ArrayList<>();
+        if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
+            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                String currentFieldName = parser.currentName();
+                if (INCOMPATIBLE_SNAPSHOTS.equals(currentFieldName)) {
+                    if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
+                        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                            incompatibleSnapshotIds.add(SnapshotId.fromXContent(parser));
+                        }
+                    } else {
+                        throw new ElasticsearchParseException("expected array for [" + currentFieldName + "]");
+                    }
+                } else {
+                    throw new ElasticsearchParseException("unknown field name  [" + currentFieldName + "]");
+                }
+            }
+        } else {
+            throw new ElasticsearchParseException("start object expected");
+        }
+        return new RepositoryData(this.genId, this.snapshotIds, this.indexSnapshots, incompatibleSnapshotIds);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
index 04900705e0a..2f25ec3b508 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
@@ -19,10 +19,10 @@
 package org.elasticsearch.repositories.blobstore;
 
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -40,9 +40,9 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
 
     protected final String blobNameFormat;
 
-    protected final FromXContentBuilder<T> reader;
+    protected final CheckedFunction<XContentParser, T, IOException> reader;
 
-    protected final ParseFieldMatcher parseFieldMatcher;
+    protected final NamedXContentRegistry namedXContentRegistry;
 
     // Serialization parameters to specify correct context for metadata serialization
     protected static final ToXContent.Params SNAPSHOT_ONLY_FORMAT_PARAMS;
@@ -60,12 +60,12 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
     /**
      * @param blobNameFormat format of the blobname in {@link String#format(Locale, String, Object...)} format
      * @param reader the prototype object that can deserialize objects with type T
-     * @param parseFieldMatcher parse field matcher
      */
-    protected BlobStoreFormat(String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher) {
+    protected BlobStoreFormat(String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+            NamedXContentRegistry namedXContentRegistry) {
         this.reader = reader;
         this.blobNameFormat = blobNameFormat;
-        this.parseFieldMatcher = parseFieldMatcher;
+        this.namedXContentRegistry = namedXContentRegistry;
     }
 
     /**
@@ -109,10 +109,9 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
     }
 
     protected T read(BytesReference bytes) throws IOException {
-        try (XContentParser parser = XContentHelper.createParser(bytes)) {
-            T obj = reader.fromXContent(parser, parseFieldMatcher);
+        try (XContentParser parser = XContentHelper.createParser(namedXContentRegistry, bytes)) {
+            T obj = reader.apply(parser);
             return obj;
-
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index aa487056cc1..ab4b63e0781 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -39,12 +39,12 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Numbers;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.blobstore.BlobContainer;
@@ -68,6 +68,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -127,8 +128,9 @@ import static java.util.Collections.unmodifiableMap;
  * <pre>
  * {@code
  *   STORE_ROOT
- *   |- index-N           - list of all snapshot name as JSON array, N is the generation of the file
+ *   |- index-N           - list of all snapshot ids and the indices belonging to each snapshot, N is the generation of the file
  *   |- index.latest      - contains the numeric value of the latest generation of the index file (i.e. N from above)
+ *   |- incompatible-snapshots - list of all snapshot ids that are no longer compatible with the current version of the cluster
  *   |- snap-20131010 - JSON serialized Snapshot for snapshot "20131010"
  *   |- meta-20131010.dat - JSON serialized MetaData for snapshot "20131010" (includes only global metadata)
  *   |- snap-20131011 - JSON serialized Snapshot for snapshot "20131011"
@@ -166,18 +168,20 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     protected final RepositoryMetaData metadata;
 
+    protected final NamedXContentRegistry namedXContentRegistry;
+
     private static final int BUFFER_SIZE = 4096;
 
     private static final String SNAPSHOT_PREFIX = "snap-";
 
     private static final String SNAPSHOT_CODEC = "snapshot";
 
-    static final String SNAPSHOTS_FILE = "index"; // package private for unit testing
-
     private static final String INDEX_FILE_PREFIX = "index-";
 
     private static final String INDEX_LATEST_BLOB = "index.latest";
 
+    private static final String INCOMPATIBLE_SNAPSHOTS_BLOB = "incompatible-snapshots";
+
     private static final String TESTS_FILE = "tests-";
 
     private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
@@ -186,15 +190,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String INDEX_METADATA_CODEC = "index-metadata";
 
-    protected static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
+    private static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
 
-    protected static final String SNAPSHOT_INDEX_PREFIX = "index-";
+    private static final String SNAPSHOT_INDEX_PREFIX = "index-";
 
-    protected static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
+    private static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
 
-    protected static final String SNAPSHOT_INDEX_CODEC = "snapshots";
+    private static final String SNAPSHOT_INDEX_CODEC = "snapshots";
 
-    protected static final String DATA_BLOB_PREFIX = "__";
+    private static final String DATA_BLOB_PREFIX = "__";
 
     private final RateLimiter snapshotRateLimiter;
 
@@ -212,8 +216,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final boolean readOnly;
 
-    private final ParseFieldMatcher parseFieldMatcher;
-
     private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat;
 
     private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshots> indexShardSnapshotsFormat;
@@ -224,25 +226,29 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * @param metadata       The metadata for this repository including name and settings
      * @param globalSettings Settings for the node this repository object is created on
      */
-    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings) {
+    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings, NamedXContentRegistry namedXContentRegistry) {
         super(globalSettings);
         this.metadata = metadata;
-        parseFieldMatcher = new ParseFieldMatcher(settings);
+        this.namedXContentRegistry = namedXContentRegistry;
         snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = metadata.settings().getAsBoolean("readonly", false);
-        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
-        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
 
+        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
+            BlobStoreIndexShardSnapshot::fromXContent, namedXContentRegistry, isCompress());
+        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT,
+            BlobStoreIndexShardSnapshots::fromXContent, namedXContentRegistry, isCompress());
     }
 
     @Override
     protected void doStart() {
         this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
-        ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(settings);
-        globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher, isCompress());
-        indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher, isCompress());
-        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher, isCompress());
+        globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT,
+            MetaData::fromXContent, namedXContentRegistry, isCompress());
+        indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT,
+            IndexMetaData::fromXContent, namedXContentRegistry, isCompress());
+        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT,
+            SnapshotInfo::fromXContent, namedXContentRegistry, isCompress());
     }
 
     @Override
@@ -300,7 +306,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         try {
             final String snapshotName = snapshotId.getName();
             // check if the snapshot name already exists in the repository
-            if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
+            final RepositoryData repositoryData = getRepositoryData();
+            if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                 throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with the same name already exists");
             }
             if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
@@ -323,7 +330,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public void deleteSnapshot(SnapshotId snapshotId) {
+    public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
         if (isReadOnly()) {
             throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
         }
@@ -351,7 +358,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         try {
             // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
             final RepositoryData updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
-            writeIndexGen(updatedRepositoryData);
+            writeIndexGen(updatedRepositoryData, repositoryStateId);
 
             // delete the snapshot file
             safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
@@ -364,7 +371,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 BlobPath indexPath = basePath().add("indices").add(indexId.getId());
                 BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
                 try {
-                    indexMetaDataFormat(snapshot.version()).delete(indexMetaDataBlobContainer, snapshotId.getUUID());
+                    indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
                 } catch (IOException ex) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]", snapshotId, index), ex);
                 }
@@ -412,7 +419,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         if (snapshotInfo != null) {
             // we know the version the snapshot was created with
             try {
-                snapshotFormat(snapshotInfo.version()).delete(snapshotsBlobContainer, blobId);
+                snapshotFormat.delete(snapshotsBlobContainer, blobId);
             } catch (IOException e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] Unable to delete snapshot file [{}]", snapshotInfo.snapshotId(), blobId), e);
             }
@@ -430,7 +437,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         if (snapshotInfo != null) {
             // we know the version the snapshot was created with
             try {
-                globalMetaDataFormat(snapshotInfo.version()).delete(snapshotsBlobContainer, blobId);
+                globalMetaDataFormat.delete(snapshotsBlobContainer, blobId);
             } catch (IOException e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] Unable to delete global metadata file [{}]", snapshotInfo.snapshotId(), blobId), e);
             }
@@ -453,7 +460,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                                          final long startTime,
                                          final String failure,
                                          final int totalShards,
-                                         final List<SnapshotShardFailure> shardFailures) {
+                                         final List<SnapshotShardFailure> shardFailures,
+                                         final long repositoryStateId) {
         try {
             SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
                                                               indices.stream().map(IndexId::getName).collect(Collectors.toList()),
@@ -466,7 +474,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             final RepositoryData repositoryData = getRepositoryData();
             List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
             if (!snapshotIds.contains(snapshotId)) {
-                writeIndexGen(repositoryData.addSnapshot(snapshotId, indices));
+                writeIndexGen(repositoryData.addSnapshot(snapshotId, indices), repositoryStateId);
             }
             return blobStoreSnapshot;
         } catch (IOException ex) {
@@ -474,10 +482,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    public List<SnapshotId> getSnapshots() {
-        return getRepositoryData().getSnapshotIds();
-    }
-
     @Override
     public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<IndexId> indices) throws IOException {
         return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
@@ -485,6 +489,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
+        if (getRepositoryData().getIncompatibleSnapshotIds().contains(snapshotId)) {
+            // an incompatible snapshot - cannot read its snapshot metadata file, just return
+            // a SnapshotInfo indicating its incompatible
+            return SnapshotInfo.incompatible(snapshotId);
+        }
+        return getSnapshotInfoInternal(snapshotId);
+    }
+
+    private SnapshotInfo getSnapshotInfoInternal(final SnapshotId snapshotId) {
         try {
             return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
         } catch (NoSuchFileException ex) {
@@ -507,7 +520,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             }
         }
         try {
-            metaData = globalMetaDataFormat(snapshotVersion).read(snapshotsBlobContainer, snapshotId.getUUID());
+            metaData = globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
         } catch (NoSuchFileException ex) {
             throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
         } catch (IOException ex) {
@@ -518,7 +531,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             BlobPath indexPath = basePath().add("indices").add(index.getId());
             BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
             try {
-                metaDataBuilder.put(indexMetaDataFormat(snapshotVersion).read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
+                metaDataBuilder.put(indexMetaDataFormat.read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
             } catch (ElasticsearchParseException | IOException ex) {
                 if (ignoreIndexErrors) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
@@ -548,27 +561,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    /**
-     * Returns appropriate global metadata format based on the provided version of the snapshot
-     */
-    private BlobStoreFormat<MetaData> globalMetaDataFormat(Version version) {
-        return globalMetaDataFormat;
-    }
-
-    /**
-     * Returns appropriate snapshot format based on the provided version of the snapshot
-     */
-    private BlobStoreFormat<SnapshotInfo> snapshotFormat(Version version) {
-        return snapshotFormat;
-    }
-
-    /**
-     * Returns appropriate index metadata format based on the provided version of the snapshot
-     */
-    private BlobStoreFormat<IndexMetaData> indexMetaDataFormat(Version version) {
-        return indexMetaDataFormat;
-    }
-
     @Override
     public long getSnapshotThrottleTimeInNanos() {
         return snapshotRateLimitingTimeInNanos.count();
@@ -625,10 +617,26 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             try (InputStream blob = snapshotsBlobContainer.readBlob(snapshotsIndexBlobName)) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 Streams.copy(blob, out);
-                try (XContentParser parser = XContentHelper.createParser(out.bytes())) {
-                    repositoryData = RepositoryData.fromXContent(parser);
+                // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                    repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
+                } catch (NotXContentException e) {
+                    logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length());
+                    throw e;
                 }
             }
+
+            // now load the incompatible snapshot ids, if they exist
+            try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
+                BytesStreamOutput out = new BytesStreamOutput();
+                Streams.copy(blob, out);
+                try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
+                    repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
+                }
+            } catch (NoSuchFileException e) {
+                logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely reason is that " +
+                             "there are no incompatible snapshots in the repository", metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
+            }
             return repositoryData;
         } catch (NoSuchFileException ex) {
             // repository doesn't have an index blob, its a new blank repo
@@ -652,36 +660,40 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         return snapshotsBlobContainer;
     }
 
-    protected void writeIndexGen(final RepositoryData repositoryData) throws IOException {
+    protected void writeIndexGen(final RepositoryData repositoryData, final long repositoryStateId) throws IOException {
         assert isReadOnly() == false; // can not write to a read only repository
+        final long currentGen = latestIndexBlobId();
+        if (repositoryStateId != SnapshotsInProgress.UNDEFINED_REPOSITORY_STATE_ID && currentGen != repositoryStateId) {
+            // the index file was updated by a concurrent operation, so we were operating on stale
+            // repository data
+            throw new RepositoryException(metadata.name(), "concurrent modification of the index-N file, expected current generation [" +
+                                              repositoryStateId + "], actual current generation [" + currentGen +
+                                              "] - possibly due to simultaneous snapshot deletion requests");
+        }
+        final long newGen = currentGen + 1;
         final BytesReference snapshotsBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
             try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
-                repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
                 builder.close();
             }
             snapshotsBytes = bStream.bytes();
         }
-        final long gen = latestIndexBlobId() + 1;
         // write the index file
-        writeAtomic(INDEX_FILE_PREFIX + Long.toString(gen), snapshotsBytes);
+        writeAtomic(INDEX_FILE_PREFIX + Long.toString(newGen), snapshotsBytes);
         // delete the N-2 index file if it exists, keep the previous one around as a backup
-        if (isReadOnly() == false && gen - 2 >= 0) {
-            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(gen - 2);
+        if (isReadOnly() == false && newGen - 2 >= 0) {
+            final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2);
             if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
                 snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
             }
-            // delete the old index file (non-generational) if it exists
-            if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
-                snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
-            }
         }
 
         // write the current generation to the index-latest file
         final BytesReference genBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
-            bStream.writeLong(gen);
+            bStream.writeLong(newGen);
             genBytes = bStream.bytes();
         }
         if (snapshotsBlobContainer.blobExists(INDEX_LATEST_BLOB)) {
@@ -690,6 +702,26 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         writeAtomic(INDEX_LATEST_BLOB, genBytes);
     }
 
+    /**
+     * Writes the incompatible snapshot ids list to the `incompatible-snapshots` blob in the repository.
+     *
+     * Package private for testing.
+     */
+    void writeIncompatibleSnapshots(RepositoryData repositoryData) throws IOException {
+        assert isReadOnly() == false; // can not write to a read only repository
+        final BytesReference bytes;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
+                repositoryData.incompatibleSnapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
+                builder.close();
+            }
+            bytes = bStream.bytes();
+        }
+        // write the incompatible snapshots blob
+        writeAtomic(INCOMPATIBLE_SNAPSHOTS_BLOB, bytes);
+    }
+
     /**
      * Get the latest snapshot index blob id.  Snapshot index blobs are named index-N, where N is
      * the next version number from when the index blob was written.  Each individual index-N blob is
@@ -700,24 +732,23 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      */
     long latestIndexBlobId() throws IOException {
         try {
-            // first, try listing the blobs and determining which index blob is the latest
-            return listBlobsToGetLatestIndexId();
-        } catch (UnsupportedOperationException e) {
-            // could not list the blobs because the repository does not support the operation,
-            // try reading from the index-latest file
+            // first, try reading the latest index generation from the index.latest blob
+            return readSnapshotIndexLatestBlob();
+        } catch (IOException ioe) {
+            // we could not find the index.latest blob, this can happen in two scenarios:
+            //  (1) its an empty repository
+            //  (2) when writing the index-latest blob, if the blob already exists,
+            //      we first delete it, then atomically write the new blob.  there is
+            //      a small window in time when the blob is deleted and the new one
+            //      written - if the node crashes during that time, we won't have an
+            //      index-latest blob
+            // lets try to list all index-N blobs to determine the last one, if listing the blobs
+            // is not a supported operation (which is the case for read-only repositories), then
+            // assume its an empty repository.
             try {
-                return readSnapshotIndexLatestBlob();
-            } catch (IOException ioe) {
-                // we likely could not find the blob, this can happen in two scenarios:
-                //  (1) its an empty repository
-                //  (2) when writing the index-latest blob, if the blob already exists,
-                //      we first delete it, then atomically write the new blob.  there is
-                //      a small window in time when the blob is deleted and the new one
-                //      written - if the node crashes during that time, we won't have an
-                //      index-latest blob
-                // in a read-only repository, we can't know which of the two scenarios it is,
-                // but we will assume (1) because we can't do anything about (2) anyway
-                return -1;
+                return listBlobsToGetLatestIndexId();
+            } catch (UnsupportedOperationException uoe) {
+                return RepositoryData.EMPTY_REPO_GEN;
             }
         }
     }
@@ -870,11 +901,11 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
         protected final Version version;
 
-        public Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
+        Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId) {
             this(snapshotId, version, indexId, shardId, shardId);
         }
 
-        public Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
+        Context(SnapshotId snapshotId, Version version, IndexId indexId, ShardId shardId, ShardId snapshotShardId) {
             this.snapshotId = snapshotId;
             this.version = version;
             this.shardId = shardId;
@@ -1081,7 +1112,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
          * @param indexId        the id of the index being snapshotted
          * @param snapshotStatus snapshot status to report progress
          */
-        public SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus) {
+        SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexShardSnapshotStatus snapshotStatus) {
             super(snapshotId, Version.CURRENT, indexId, shard.shardId());
             this.snapshotStatus = snapshotStatus;
             this.store = shard.store();
@@ -1284,7 +1315,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         private class AbortableInputStream extends FilterInputStream {
             private final String fileName;
 
-            public AbortableInputStream(InputStream delegate, String fileName) {
+            AbortableInputStream(InputStream delegate, String fileName) {
                 super(delegate);
                 this.fileName = fileName;
             }
@@ -1322,7 +1353,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 // we have a hash - check if our repo has a hash too otherwise we have
                 // to calculate it.
                 // we might have multiple parts even though the file is small... make sure we read all of it.
-                try (final InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
+                try (InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
                     BytesRefBuilder builder = new BytesRefBuilder();
                     Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length());
                     BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash
@@ -1340,7 +1371,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         private final BlobContainer container;
         private final BlobStoreIndexShardSnapshot.FileInfo info;
 
-        public PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
+        PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
             super(info.numberOfParts());
             this.info = info;
             this.container = container;
@@ -1370,7 +1401,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
          * @param snapshotShardId shard in the snapshot that data should be restored from
          * @param recoveryState   recovery state to report progress
          */
-        public RestoreContext(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
+        RestoreContext(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
             super(snapshotId, version, indexId, shard.shardId(), snapshotShardId);
             this.recoveryState = recoveryState;
             this.targetShard = shard;
@@ -1532,7 +1563,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                     stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreRateLimitingTimeInNanos::inc);
                 }
 
-                try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
+                try (IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
                     final byte[] buffer = new byte[BUFFER_SIZE];
                     int length;
                     while ((length = stream.read(buffer)) > 0) {
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 17fad25e610..0cb38d9976d 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -23,7 +23,7 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexFormatTooNewException;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.OutputStreamIndexOutput;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -33,10 +33,11 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.store.ByteArrayIndexInput;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.CorruptStateException;
 
@@ -73,8 +74,9 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
      * @param compress       true if the content should be compressed
      * @param xContentType   content type that should be used for write operations
      */
-    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher, boolean compress, XContentType xContentType) {
-        super(blobNameFormat, reader, parseFieldMatcher);
+    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                   NamedXContentRegistry namedXContentRegistry, boolean compress, XContentType xContentType) {
+        super(blobNameFormat, reader, namedXContentRegistry);
         this.xContentType = xContentType;
         this.compress = compress;
         this.codec = codec;
@@ -86,8 +88,9 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
      * @param reader         prototype object that can deserialize T from XContent
      * @param compress       true if the content should be compressed
      */
-    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, FromXContentBuilder<T> reader, ParseFieldMatcher parseFieldMatcher, boolean compress) {
-        this(codec, blobNameFormat, reader, parseFieldMatcher, compress, DEFAULT_X_CONTENT_TYPE);
+    public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader,
+                                   NamedXContentRegistry namedXContentRegistry, boolean compress) {
+        this(codec, blobNameFormat, reader, namedXContentRegistry, compress, DEFAULT_X_CONTENT_TYPE);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index c028913d343..b490a2e784d 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -72,8 +73,9 @@ public class FsRepository extends BlobStoreRepository {
     /**
      * Constructs a shared file system repository.
      */
-    public FsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public FsRepository(RepositoryMetaData metadata, Environment environment,
+                        NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
         String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         if (location.isEmpty()) {
             logger.warn("the repository location is missing, it should point to a shared file system location that is available on all master and data nodes");
diff --git a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
index 4b3505e97e3..bdc78c82dd5 100644
--- a/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
+++ b/core/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java
@@ -20,7 +20,6 @@ package org.elasticsearch.rest;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -58,28 +57,37 @@ public abstract class AbstractRestChannel implements RestChannel {
 
     @Override
     public XContentBuilder newBuilder() throws IOException {
-        return newBuilder(request.hasContent() ? request.content() : null, true);
+        return newBuilder(request.getXContentType(), true);
     }
 
     @Override
     public XContentBuilder newErrorBuilder() throws IOException {
         // Disable filtering when building error responses
-        return newBuilder(request.hasContent() ? request.content() : null, false);
+        return newBuilder(request.getXContentType(), false);
     }
 
+    /**
+     * Creates a new {@link XContentBuilder} for a response to be sent using this channel. The builder's type is determined by the following
+     * logic. If the request has a format parameter that will be used to attempt to map to an {@link XContentType}. If there is no format
+     * parameter, the HTTP Accept header is checked to see if it can be matched to a {@link XContentType}. If this first attempt to map
+     * fails, the request content type will be used if the value is not {@code null}; if the value is {@code null} the output format falls
+     * back to JSON.
+     */
     @Override
-    public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-        XContentType contentType = XContentType.fromMediaTypeOrFormat(format);
-        if (contentType == null) {
-            // try and guess it from the auto detect source
-            if (autoDetectSource != null) {
-                contentType = XContentFactory.xContentType(autoDetectSource);
+    public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException {
+        // try to determine the response content type from the media type or the format query string parameter, with the format parameter
+        // taking precedence over the Accept header
+        XContentType responseContentType = XContentType.fromMediaTypeOrFormat(format);
+        if (responseContentType == null) {
+            if (requestContentType != null) {
+                // if there was a parsed content-type for the incoming request use that since no format was specified using the query
+                // string parameter or the HTTP Accept header
+                responseContentType = requestContentType;
+            } else {
+                // default to JSON output when all else fails
+                responseContentType = XContentType.JSON;
             }
         }
-        if (contentType == null) {
-            // default to JSON
-            contentType = XContentType.JSON;
-        }
 
         Set<String> includes = Collections.emptySet();
         Set<String> excludes = Collections.emptySet();
@@ -89,7 +97,7 @@ public abstract class AbstractRestChannel implements RestChannel {
             excludes = filters.stream().filter(EXCLUDE_FILTER).map(f -> f.substring(1)).collect(toSet());
         }
 
-        XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), includes, excludes);
+        XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(responseContentType), bytesOutput(), includes, excludes);
         if (pretty) {
             builder.prettyPrint().lfAtEnd();
         }
@@ -125,5 +133,4 @@ public abstract class AbstractRestChannel implements RestChannel {
     public boolean detailedErrorsEnabled() {
         return detailedErrorsEnabled;
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index f1804ee01b1..81620ec8a7c 100644
--- a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest;
 import org.apache.lucene.search.spell.LevensteinDistance;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.CheckedConsumer;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
@@ -53,11 +53,9 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH
 
     public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX =
         Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope);
-    protected final ParseFieldMatcher parseFieldMatcher;
 
     protected BaseRestHandler(Settings settings) {
         super(settings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
     }
 
     @Override
@@ -128,14 +126,7 @@ public abstract class BaseRestHandler extends AbstractComponent implements RestH
      * the request against a channel.
      */
     @FunctionalInterface
-    protected interface RestChannelConsumer {
-        /**
-         * Executes a request against the given channel.
-         *
-         * @param channel the channel for sending the response
-         * @throws Exception if an exception occurred executing the request
-         */
-        void accept(RestChannel channel) throws Exception;
+    protected interface RestChannelConsumer extends CheckedConsumer<RestChannel, Exception> {
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index 7af8249bf2e..72ee7efc489 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -23,21 +23,30 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Collections;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE;
+import static org.elasticsearch.ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 
 public class BytesRestResponse extends RestResponse {
 
     public static final String TEXT_CONTENT_TYPE = "text/plain; charset=UTF-8";
 
+    private static final String STATUS = "status";
+
     private final RestStatus status;
     private final BytesReference content;
     private final String contentType;
@@ -89,9 +98,10 @@ public class BytesRestResponse extends RestResponse {
             this.content = BytesArray.EMPTY;
             this.contentType = TEXT_CONTENT_TYPE;
         } else {
-            XContentBuilder builder = convert(channel, status, e);
-            this.content = builder.bytes();
-            this.contentType = builder.contentType().mediaType();
+            try (XContentBuilder builder = build(channel, status, e)) {
+                this.content = builder.bytes();
+                this.contentType = builder.contentType().mediaType();
+            }
         }
         if (e instanceof ElasticsearchException) {
             copyHeaders(((ElasticsearchException) e));
@@ -115,57 +125,68 @@ public class BytesRestResponse extends RestResponse {
 
     private static final Logger SUPPRESSED_ERROR_LOGGER = ESLoggerFactory.getLogger("rest.suppressed");
 
-    private static XContentBuilder convert(RestChannel channel, RestStatus status, Exception e) throws IOException {
-        XContentBuilder builder = channel.newErrorBuilder().startObject();
-        if (e == null) {
-            builder.field("error", "unknown");
-        } else if (channel.detailedErrorsEnabled()) {
-            final ToXContent.Params params;
-            if (channel.request().paramAsBoolean("error_trace", !ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT)) {
-                params =  new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"), channel.request());
-            } else {
-                if (status.getStatus() < 500) {
-                    SUPPRESSED_ERROR_LOGGER.debug((Supplier<?>) () -> new ParameterizedMessage("path: {}, params: {}", channel.request().rawPath(), channel.request().params()), e);
-                } else {
-                    SUPPRESSED_ERROR_LOGGER.warn((Supplier<?>) () -> new ParameterizedMessage("path: {}, params: {}", channel.request().rawPath(), channel.request().params()), e);
-                }
-                params = channel.request();
-            }
-            builder.field("error");
-            builder.startObject();
-            final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
-            builder.field("root_cause");
-            builder.startArray();
-            for (ElasticsearchException rootCause : rootCauses){
-                builder.startObject();
-                rootCause.toXContent(builder, new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_CAUSE, "true"), params));
-                builder.endObject();
-            }
-            builder.endArray();
+    private static XContentBuilder build(RestChannel channel, RestStatus status, Exception e) throws IOException {
+        ToXContent.Params params = channel.request();
+        if (params.paramAsBoolean("error_trace", !REST_EXCEPTION_SKIP_STACK_TRACE_DEFAULT)) {
+            params =  new ToXContent.DelegatingMapParams(singletonMap(REST_EXCEPTION_SKIP_STACK_TRACE, "false"), params);
+        } else if (e != null) {
+            Supplier<?> messageSupplier = () -> new ParameterizedMessage("path: {}, params: {}",
+                    channel.request().rawPath(), channel.request().params());
 
-            ElasticsearchException.toXContent(builder, params, e);
-            builder.endObject();
-        } else {
-            builder.field("error", simpleMessage(e));
+            if (status.getStatus() < 500) {
+                SUPPRESSED_ERROR_LOGGER.debug(messageSupplier, e);
+            } else {
+                SUPPRESSED_ERROR_LOGGER.warn(messageSupplier, e);
+            }
         }
-        builder.field("status", status.getStatus());
+
+        XContentBuilder builder = channel.newErrorBuilder().startObject();
+        ElasticsearchException.generateFailureXContent(builder, params, e, channel.detailedErrorsEnabled());
+        builder.field(STATUS, status.getStatus());
         builder.endObject();
         return builder;
     }
 
-    /*
-     * Builds a simple error string from the message of the first ElasticsearchException
-     */
-    private static String simpleMessage(Throwable t) throws IOException {
-        int counter = 0;
-        Throwable next = t;
-        while (next != null && counter++ < 10) {
-            if (t instanceof ElasticsearchException) {
-                return next.getClass().getSimpleName() + "[" + next.getMessage() + "]";
+    static BytesRestResponse createSimpleErrorResponse(RestStatus status, String errorMessage) throws IOException {
+        return new BytesRestResponse(status, JsonXContent.contentBuilder().startObject()
+            .field("error", errorMessage)
+            .field("status", status.getStatus())
+            .endObject());
+    }
+
+    public static ElasticsearchStatusException errorFromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.nextToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+
+        ElasticsearchException exception = null;
+        RestStatus status = null;
+
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            }
+            if (STATUS.equals(currentFieldName)) {
+                if (token != XContentParser.Token.FIELD_NAME) {
+                    ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, token, parser::getTokenLocation);
+                    status = RestStatus.fromCode(parser.intValue());
+                }
+            } else {
+                exception = ElasticsearchException.failureFromXContent(parser);
             }
-            next = next.getCause();
         }
 
-        return "No ElasticsearchException found";
+        if (exception == null) {
+            throw new IllegalStateException("Failed to parse elasticsearch status exception: no exception was found");
+        }
+
+        ElasticsearchStatusException result = new ElasticsearchStatusException(exception.getMessage(), status, exception.getCause());
+        for (String header : exception.getHeaderKeys()) {
+            result.addHeader(header, exception.getHeader(header));
+        }
+        for (String metadata : exception.getMetadataKeys()) {
+            result.addMetadata(metadata, exception.getMetadata(metadata));
+        }
+        return result;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestChannel.java b/core/src/main/java/org/elasticsearch/rest/RestChannel.java
index 2a56313fd8a..8c8346f0ef4 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestChannel.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestChannel.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 
@@ -35,7 +35,7 @@ public interface RestChannel {
 
     XContentBuilder newErrorBuilder() throws IOException;
 
-    XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException;
+    XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException;
 
     BytesStreamOutput bytesOutput();
 
@@ -47,5 +47,4 @@ public interface RestChannel {
     boolean detailedErrorsEnabled();
 
     void sendResponse(RestResponse response);
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index c701e8ff0ee..060e90c5e9d 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -19,23 +19,39 @@
 
 package org.elasticsearch.rest;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
+import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
+import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
+import static org.elasticsearch.rest.RestStatus.NOT_ACCEPTABLE;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestController extends AbstractComponent {
@@ -48,16 +64,29 @@ public class RestController extends AbstractComponent {
 
     private final UnaryOperator<RestHandler> handlerWrapper;
 
+    private final NodeClient client;
+
+    private final CircuitBreakerService circuitBreakerService;
+
     /** Rest headers that are copied to internal requests made during a rest request. */
     private final Set<String> headersToCopy;
 
-    public RestController(Settings settings, Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper) {
+    private final boolean isContentTypeRequired;
+
+    private final DeprecationLogger deprecationLogger;
+
+    public RestController(Settings settings, Set<String> headersToCopy, UnaryOperator<RestHandler> handlerWrapper,
+                          NodeClient client, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.headersToCopy = headersToCopy;
         if (handlerWrapper == null) {
             handlerWrapper = h -> h; // passthrough if no wrapper set
         }
         this.handlerWrapper = handlerWrapper;
+        this.client = client;
+        this.circuitBreakerService = circuitBreakerService;
+        this.isContentTypeRequired = HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.get(settings);
+        this.deprecationLogger = new DeprecationLogger(logger);
     }
 
     /**
@@ -137,35 +166,111 @@ public class RestController extends AbstractComponent {
         return (handler != null) ? handler.canTripCircuitBreaker() : true;
     }
 
-    public void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception {
-        if (!checkRequestParameters(request, channel)) {
+    public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+        if (request.rawPath().equals("/favicon.ico")) {
+            handleFavicon(request, channel);
             return;
         }
-        try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
-            for (String key : headersToCopy) {
-                String httpHeader = request.header(key);
-                if (httpHeader != null) {
-                    threadContext.putHeader(key, httpHeader);
-                }
-            }
-
+        RestChannel responseChannel = channel;
+        try {
+            final int contentLength = request.hasContent() ? request.content().length() : 0;
+            assert contentLength >= 0 : "content length was negative, how is that possible?";
             final RestHandler handler = getHandler(request);
 
-            if (handler == null) {
-                if (request.method() == RestRequest.Method.OPTIONS) {
-                    // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
-                    channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                } else {
-                    final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]";
-                    channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg));
-                }
+            if (contentLength > 0 && hasContentTypeOrCanAutoDetect(request, handler) == false) {
+                sendContentTypeErrorMessage(request, responseChannel);
             } else {
-                final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler));
-                wrappedHandler.handleRequest(request, channel, client);
+                if (canTripCircuitBreaker(request)) {
+                    inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
+                } else {
+                    inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength);
+                }
+                // iff we could reserve bytes for the request we need to send the response also over this channel
+                responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
+                dispatchRequest(request, responseChannel, client, threadContext, handler);
+            }
+        } catch (Exception e) {
+            try {
+                responseChannel.sendResponse(new BytesRestResponse(channel, e));
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.error((Supplier<?>) () ->
+                    new ParameterizedMessage("failed to send failure response for uri [{}]", request.uri()), inner);
             }
         }
     }
 
+    void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext,
+                         final RestHandler handler) throws Exception {
+        if (checkRequestParameters(request, channel) == false) {
+            channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(BAD_REQUEST, "error traces in responses are disabled."));
+        } else {
+            try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+                for (String key : headersToCopy) {
+                    String httpHeader = request.header(key);
+                    if (httpHeader != null) {
+                        threadContext.putHeader(key, httpHeader);
+                    }
+                }
+
+                if (handler == null) {
+                    if (request.method() == RestRequest.Method.OPTIONS) {
+                        // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
+
+                        channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+                    } else {
+                        final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]";
+                        channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg));
+                    }
+                } else {
+                    final RestHandler wrappedHandler = Objects.requireNonNull(handlerWrapper.apply(handler));
+                    wrappedHandler.handleRequest(request, channel, client);
+                }
+            }
+        }
+    }
+
+    /**
+     * If a request contains content, this method will return {@code true} if the {@code Content-Type} header is present, matches an
+     * {@link XContentType} or the request is plain text, and content type is required. If content type is not required then this method
+     * returns true unless a content type could not be inferred from the body and the rest handler does not support plain text
+     */
+    private boolean hasContentTypeOrCanAutoDetect(final RestRequest restRequest, final RestHandler restHandler) {
+        if (restRequest.getXContentType() == null) {
+            if (restHandler != null && restHandler.supportsPlainText()) {
+                // content type of null with a handler that supports plain text gets through for now. Once we remove plain text this can
+                // be removed!
+                deprecationLogger.deprecated("Plain text request bodies are deprecated. Use request parameters or body " +
+                    "in a supported format.");
+            } else if (isContentTypeRequired) {
+                return false;
+            } else {
+                deprecationLogger.deprecated("Content type detection for rest requests is deprecated. Specify the content type using " +
+                    "the [Content-Type] header.");
+                XContentType xContentType = XContentFactory.xContentType(restRequest.content());
+                if (xContentType == null) {
+                    return false;
+                } else {
+                    restRequest.setXContentType(xContentType);
+                }
+            }
+        }
+        return true;
+    }
+
+    private void sendContentTypeErrorMessage(RestRequest restRequest, RestChannel channel) throws IOException {
+        final List<String> contentTypeHeader = restRequest.getAllHeaderValues("Content-Type");
+        final String errorMessage;
+        if (contentTypeHeader == null) {
+            errorMessage = "Content-Type header is missing";
+        } else {
+            errorMessage = "Content-Type header [" +
+                Strings.collectionToCommaDelimitedString(restRequest.getAllHeaderValues("Content-Type")) + "] is not supported";
+        }
+
+        channel.sendResponse(BytesRestResponse.createSimpleErrorResponse(NOT_ACCEPTABLE, errorMessage));
+    }
+
     /**
      * Checks the request parameters against enabled settings for error trace support
      * @return true if the request does not have any parameters that conflict with system settings
@@ -174,15 +279,6 @@ public class RestController extends AbstractComponent {
         // error_trace cannot be used when we disable detailed errors
         // we consume the error_trace parameter first to ensure that it is always consumed
         if (request.paramAsBoolean("error_trace", false) && channel.detailedErrorsEnabled() == false) {
-            try {
-                XContentBuilder builder = channel.newErrorBuilder();
-                builder.startObject().field("error", "error traces in responses are disabled.").endObject().string();
-                RestResponse response = new BytesRestResponse(BAD_REQUEST, builder);
-                response.addHeader("Content-Type", "application/json");
-                channel.sendResponse(response);
-            } catch (IOException e) {
-                logger.warn("Failed to send response", e);
-            }
             return false;
         }
 
@@ -223,4 +319,84 @@ public class RestController extends AbstractComponent {
         // my_index/my_type/http%3A%2F%2Fwww.google.com
         return request.rawPath();
     }
+
+    void handleFavicon(RestRequest request, RestChannel channel) {
+        if (request.method() == RestRequest.Method.GET) {
+            try {
+                try (InputStream stream = getClass().getResourceAsStream("/config/favicon.ico")) {
+                    ByteArrayOutputStream out = new ByteArrayOutputStream();
+                    Streams.copy(stream, out);
+                    BytesRestResponse restResponse = new BytesRestResponse(RestStatus.OK, "image/x-icon", out.toByteArray());
+                    channel.sendResponse(restResponse);
+                }
+            } catch (IOException e) {
+                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+        } else {
+            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+        }
+    }
+
+    private static final class ResourceHandlingHttpChannel implements RestChannel {
+        private final RestChannel delegate;
+        private final CircuitBreakerService circuitBreakerService;
+        private final int contentLength;
+        private final AtomicBoolean closed = new AtomicBoolean();
+
+        ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength) {
+            this.delegate = delegate;
+            this.circuitBreakerService = circuitBreakerService;
+            this.contentLength = contentLength;
+        }
+
+        @Override
+        public XContentBuilder newBuilder() throws IOException {
+            return delegate.newBuilder();
+        }
+
+        @Override
+        public XContentBuilder newErrorBuilder() throws IOException {
+            return delegate.newErrorBuilder();
+        }
+
+        @Override
+        public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException {
+            return delegate.newBuilder(xContentType, useFiltering);
+        }
+
+        @Override
+        public BytesStreamOutput bytesOutput() {
+            return delegate.bytesOutput();
+        }
+
+        @Override
+        public RestRequest request() {
+            return delegate.request();
+        }
+
+        @Override
+        public boolean detailedErrorsEnabled() {
+            return delegate.detailedErrorsEnabled();
+        }
+
+        @Override
+        public void sendResponse(RestResponse response) {
+            close();
+            delegate.sendResponse(response);
+        }
+
+        private void close() {
+            // attempt to close once atomically
+            if (closed.compareAndSet(false, true) == false) {
+                throw new IllegalStateException("Channel is already closed");
+            }
+            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength);
+        }
+
+    }
+
+    private static CircuitBreaker inFlightRequestsBreaker(CircuitBreakerService circuitBreakerService) {
+        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
+        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
index 393e425baf9..740b98ae737 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -28,7 +28,6 @@ public interface RestHandler {
 
     /**
      * Handles a rest request.
-     *
      * @param request The request to handle
      * @param channel The channel to write the request response to
      * @param client A client to use to make internal requests on behalf of the original request
@@ -38,4 +37,13 @@ public interface RestHandler {
     default boolean canTripCircuitBreaker() {
         return true;
     }
+
+    /**
+     * Indicates if a RestHandler supports plain text bodies
+     * @deprecated use request parameters or bodies that can be parsed with XContent!
+     */
+    @Deprecated
+    default boolean supportsPlainText() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestRequest.java b/core/src/main/java/org/elasticsearch/rest/RestRequest.java
index ad6a367a7b7..7d41faf12e0 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestRequest.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.CheckedConsumer;
@@ -26,19 +27,26 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.net.SocketAddress;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
@@ -46,11 +54,26 @@ import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 
 public abstract class RestRequest implements ToXContent.Params {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestRequest.class));
+    // tchar pattern as defined by RFC7230 section 3.2.6
+    private static final Pattern TCHAR_PATTERN = Pattern.compile("[a-zA-z0-9!#$%&'*+\\-.\\^_`|~]+");
+
+    private final NamedXContentRegistry xContentRegistry;
     private final Map<String, String> params;
+    private final Map<String, List<String>> headers;
     private final String rawPath;
     private final Set<String> consumedParams = new HashSet<>();
+    private final SetOnce<XContentType> xContentType = new SetOnce<>();
 
-    public RestRequest(String uri) {
+    /**
+     * Creates a new RestRequest
+     * @param xContentRegistry the xContentRegistry to use when parsing XContent
+     * @param uri the URI of the request that potentially contains request parameters
+     * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case
+     *                insensitive
+     */
+    public RestRequest(NamedXContentRegistry xContentRegistry, String uri, Map<String, List<String>> headers) {
+        this.xContentRegistry = xContentRegistry;
         final Map<String, String> params = new HashMap<>();
         int pathEndPos = uri.indexOf('?');
         if (pathEndPos < 0) {
@@ -60,11 +83,32 @@ public abstract class RestRequest implements ToXContent.Params {
             RestUtils.decodeQueryString(uri, pathEndPos + 1, params);
         }
         this.params = params;
+        this.headers = Collections.unmodifiableMap(headers);
+        final List<String> contentType = getAllHeaderValues("Content-Type");
+        final XContentType xContentType = parseContentType(contentType);
+        if (xContentType != null) {
+            this.xContentType.set(xContentType);
+        }
     }
 
-    public RestRequest(Map<String, String> params, String path) {
+    /**
+     * Creates a new RestRequest
+     * @param xContentRegistry the xContentRegistry to use when parsing XContent
+     * @param params the parameters of the request
+     * @param path the path of the request. This should not contain request parameters
+     * @param headers a map of the headers. This map should implement a Case-Insensitive hashing for keys as HTTP header names are case
+     *                insensitive
+     */
+    public RestRequest(NamedXContentRegistry xContentRegistry, Map<String, String> params, String path, Map<String, List<String>> headers) {
+        this.xContentRegistry = xContentRegistry;
         this.params = params;
         this.rawPath = path;
+        this.headers = Collections.unmodifiableMap(headers);
+        final List<String> contentType = getAllHeaderValues("Content-Type");
+        final XContentType xContentType = parseContentType(contentType);
+        if (xContentType != null) {
+            this.xContentType.set(xContentType);
+        }
     }
 
     public enum Method {
@@ -96,9 +140,53 @@ public abstract class RestRequest implements ToXContent.Params {
 
     public abstract BytesReference content();
 
-    public abstract String header(String name);
+    /**
+     * Get the value of the header or {@code null} if not found. This method only retrieves the first header value if multiple values are
+     * sent. Use of {@link #getAllHeaderValues(String)} should be preferred
+     */
+    public final String header(String name) {
+        List<String> values = headers.get(name);
+        if (values != null && values.isEmpty() == false) {
+            return values.get(0);
+        }
+        return null;
+    }
 
-    public abstract Iterable<Map.Entry<String, String>> headers();
+    /**
+     * Get all values for the header or {@code null} if the header was not found
+     */
+    public final List<String> getAllHeaderValues(String name) {
+        List<String> values = headers.get(name);
+        if (values != null) {
+            return Collections.unmodifiableList(values);
+        }
+        return null;
+    }
+
+    /**
+     * Get all of the headers and values associated with the headers. Modifications of this map are not supported.
+     */
+    public final Map<String, List<String>> getHeaders() {
+        return headers;
+    }
+
+    /**
+     * The {@link XContentType} that was parsed from the {@code Content-Type} header. This value will be {@code null} in the case of
+     * a request without a valid {@code Content-Type} header, a request without content ({@link #hasContent()}, or a plain text request
+     */
+    @Nullable
+    public final XContentType getXContentType() {
+        return xContentType.get();
+    }
+
+    /**
+     * Sets the {@link XContentType}
+     * @deprecated this is only used to allow BWC with content-type detection
+     */
+    @Deprecated
+    final void setXContentType(XContentType xContentType) {
+        this.xContentType.set(xContentType);
+    }
 
     @Nullable
     public SocketAddress getRemoteAddress() {
@@ -196,7 +284,13 @@ public abstract class RestRequest implements ToXContent.Params {
 
     @Override
     public boolean paramAsBoolean(String key, boolean defaultValue) {
-        return Booleans.parseBoolean(param(key), defaultValue);
+        String rawParam = param(key);
+        // Treat empty string as true because that allows the presence of the url parameter to mean "turn this on"
+        if (rawParam != null && rawParam.length() == 0) {
+            return true;
+        } else {
+            return Booleans.parseBoolean(rawParam, defaultValue);
+        }
     }
 
     @Override
@@ -228,6 +322,13 @@ public abstract class RestRequest implements ToXContent.Params {
         return params;
     }
 
+    /**
+     * Get the {@link NamedXContentRegistry} that should be used to create parsers from this request.
+     */
+    public NamedXContentRegistry getXContentRegistry() {
+        return xContentRegistry;
+    }
+
     /**
      * A parser for the contents of this request if there is a body, otherwise throws an {@link ElasticsearchParseException}. Use
      * {@link #applyContentParser(CheckedConsumer)} if you want to gracefully handle when the request doesn't have any contents. Use
@@ -237,8 +338,10 @@ public abstract class RestRequest implements ToXContent.Params {
         BytesReference content = content();
         if (content.length() == 0) {
             throw new ElasticsearchParseException("Body required");
+        } else if (xContentType.get() == null) {
+            throw new IllegalStateException("unknown content type");
         }
-        return XContentFactory.xContent(content).createParser(content);
+        return xContentType.get().xContent().createParser(xContentRegistry, content);
     }
 
     /**
@@ -266,11 +369,12 @@ public abstract class RestRequest implements ToXContent.Params {
      * if you need to handle the absence request content gracefully.
      */
     public final XContentParser contentOrSourceParamParser() throws IOException {
-        BytesReference content = contentOrSourceParam();
+        Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+        BytesReference content = tuple.v2();
         if (content.length() == 0) {
             throw new ElasticsearchParseException("Body required");
         }
-        return XContentFactory.xContent(content).createParser(content);
+        return tuple.v1().xContent().createParser(xContentRegistry, content);
     }
 
     /**
@@ -279,9 +383,11 @@ public abstract class RestRequest implements ToXContent.Params {
      * back to the user when there isn't request content.
      */
     public final void withContentOrSourceParamParserOrNull(CheckedConsumer<XContentParser, IOException> withParser) throws IOException {
-        BytesReference content = contentOrSourceParam();
+        Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+        BytesReference content = tuple.v2();
+        XContentType xContentType = tuple.v1();
         if (content.length() > 0) {
-            try (XContentParser parser = XContentFactory.xContent(content).createParser(content)) {
+            try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) {
                 withParser.accept(parser);
             }
         } else {
@@ -293,7 +399,66 @@ public abstract class RestRequest implements ToXContent.Params {
      * Get the content of the request or the contents of the {@code source} param. Prefer {@link #contentOrSourceParamParser()} or
      * {@link #withContentOrSourceParamParserOrNull(CheckedConsumer)} if you need a parser.
      */
-    public final BytesReference contentOrSourceParam() {
+    public final Tuple<XContentType, BytesReference> contentOrSourceParam() {
+        if (hasContent()) {
+            if (xContentType.get() == null) {
+                throw new IllegalStateException("unknown content type");
+            }
+            return new Tuple<>(xContentType.get(), content());
+        }
+
+        String source = param("source");
+        String typeParam = param("source_content_type");
+        if (source != null) {
+            BytesArray bytes = new BytesArray(source);
+            final XContentType xContentType;
+            if (typeParam != null) {
+                xContentType = parseContentType(Collections.singletonList(typeParam));
+            } else {
+                DEPRECATION_LOGGER.deprecated("Deprecated use of the [source] parameter without the [source_content_type] parameter. Use " +
+                    "the [source_content_type] parameter to specify the content type of the source such as [application/json]");
+                xContentType = XContentFactory.xContentType(bytes);
+            }
+
+            if (xContentType == null) {
+                throw new IllegalStateException("could not determine source content type");
+            }
+            return new Tuple<>(xContentType, bytes);
+        }
+        return new Tuple<>(XContentType.JSON, BytesArray.EMPTY);
+    }
+
+    /**
+     * Call a consumer with the parser for the contents of this request if it has contents, otherwise with a parser for the {@code source}
+     * parameter if there is one, otherwise with {@code null}. Use {@link #contentOrSourceParamParser()} if you should throw an exception
+     * back to the user when there isn't request content. This version allows for plain text content
+     */
+    @Deprecated
+    public final void withContentOrSourceParamParserOrNullLenient(CheckedConsumer<XContentParser, IOException> withParser)
+        throws IOException {
+        if (hasContent() && xContentType.get() == null) {
+            withParser.accept(null);
+        }
+
+        Tuple<XContentType, BytesReference> tuple = contentOrSourceParam();
+        BytesReference content = tuple.v2();
+        XContentType xContentType = tuple.v1();
+        if (content.length() > 0) {
+            try (XContentParser parser = xContentType.xContent().createParser(xContentRegistry, content)) {
+                withParser.accept(parser);
+            }
+        } else {
+            withParser.accept(null);
+        }
+    }
+
+    /**
+     * Get the content of the request or the contents of the {@code source} param without the xcontent type. This is useful the request can
+     * accept non xcontent values.
+     * @deprecated we should only take xcontent
+     */
+    @Deprecated
+    public final BytesReference getContentOrSourceParamOnly() {
         if (hasContent()) {
             return content();
         }
@@ -303,4 +468,29 @@ public abstract class RestRequest implements ToXContent.Params {
         }
         return BytesArray.EMPTY;
     }
+
+    /**
+     * Parses the given content type string for the media type. This method currently ignores parameters.
+     */
+    // TODO stop ignoring parameters such as charset...
+    private static XContentType parseContentType(List<String> header) {
+        if (header == null || header.isEmpty()) {
+            return null;
+        } else if (header.size() > 1) {
+            throw new IllegalArgumentException("only one Content-Type header should be provided");
+        }
+
+        String rawContentType = header.get(0);
+        final String[] elements = rawContentType.split("[ \t]*;");
+        if (elements.length > 0) {
+            final String[] splitMediaType = elements[0].split("/");
+            if (splitMediaType.length == 2 && TCHAR_PATTERN.matcher(splitMediaType[0]).matches()
+                && TCHAR_PATTERN.matcher(splitMediaType[1].trim()).matches()) {
+                return XContentType.fromMediaType(elements[0]);
+            } else {
+                throw new IllegalArgumentException("invalid Content-Type header [" + rawContentType + "]");
+            }
+        }
+        throw new IllegalArgumentException("empty Content-Type header");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
index 0cb1a248b17..74836b1dc2d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestActions.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContent.Params;
@@ -36,7 +35,6 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
@@ -195,9 +193,8 @@ public class RestActions {
         return queryBuilder;
     }
 
-    public static QueryBuilder getQueryContent(XContentParser requestParser, IndicesQueriesRegistry indicesQueriesRegistry,
-                                               ParseFieldMatcher parseFieldMatcher) {
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, requestParser, parseFieldMatcher);
+    public static QueryBuilder getQueryContent(XContentParser requestParser) {
+        QueryParseContext context = new QueryParseContext(requestParser);
         return context.parseTopLevelQueryBuilder();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
index 8a4a91b2493..1393f429372 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestFieldStatsAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -44,8 +43,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestFieldStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestFieldStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_field_stats", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
index 210ccb2e227..006b6d71db4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestMainAction.java
@@ -23,8 +23,6 @@ import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.MainRequest;
 import org.elasticsearch.action.main.MainResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,8 +38,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 
 public class RestMainAction extends BaseRestHandler {
-
-    @Inject
     public RestMainAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
index f147d6bc00d..f17399743b4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestStatusToXContentListener.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action;
 
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -30,7 +30,7 @@ import java.util.function.Function;
 /**
  * Content listener that extracts that {@link RestStatus} from the response.
  */
-public class RestStatusToXContentListener<Response extends StatusToXContent> extends RestResponseListener<Response> {
+public class RestStatusToXContentListener<Response extends StatusToXContentObject> extends RestToXContentListener<Response> {
     private final Function<Response, String> extractLocation;
 
     /**
@@ -52,15 +52,10 @@ public class RestStatusToXContentListener<Response extends StatusToXContent> ext
     }
 
     @Override
-    public final RestResponse buildResponse(Response response) throws Exception {
-        return buildResponse(response, channel.newBuilder());
-    }
-
-    public final RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
-        builder.startObject();
+    public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+        assert response.isFragment() == false; //would be nice if we could make default methods final
         response.toXContent(builder, channel.request());
-        builder.endObject();
-        BytesRestResponse restResponse = new BytesRestResponse(response.status(), builder);
+        RestResponse restResponse = new BytesRestResponse(response.status(), builder);
         if (RestStatus.CREATED == restResponse.status()) {
             String location = extractLocation.apply(response);
             if (location != null) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
index 9e5c4f40659..06fdd021a32 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/RestToXContentListener.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action;
 
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -30,7 +31,7 @@ import org.elasticsearch.rest.RestStatus;
  * A REST based action listener that assumes the response is of type {@link ToXContent} and automatically
  * builds an XContent based response (wrapping the toXContent in startObject/endObject).
  */
-public class RestToXContentListener<Response extends ToXContent> extends RestResponseListener<Response> {
+public class RestToXContentListener<Response extends ToXContentObject> extends RestResponseListener<Response> {
 
     public RestToXContentListener(RestChannel channel) {
         super(channel);
@@ -41,10 +42,9 @@ public class RestToXContentListener<Response extends ToXContent> extends RestRes
         return buildResponse(response, channel.newBuilder());
     }
 
-    public final RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
-        builder.startObject();
+    public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+        assert response.isFragment() == false; //would be nice if we could make default methods final
         response.toXContent(builder, channel.request());
-        builder.endObject();
         return new BytesRestResponse(getStatus(response), builder);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
index b6a4e743d13..28631501b7f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCancelTasksAction.java
@@ -21,9 +21,8 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -31,18 +30,18 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.listTasksResponseListener;
 
 
 public class RestCancelTasksAction extends BaseRestHandler {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestCancelTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestCancelTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(POST, "/_tasks/_cancel", this);
         controller.registerHandler(POST, "/_tasks/{task_id}/_cancel", this);
     }
@@ -61,7 +60,7 @@ public class RestCancelTasksAction extends BaseRestHandler {
         cancelTasksRequest.setActions(actions);
         cancelTasksRequest.setParentTaskId(parentTaskId);
         return channel ->
-            client.admin().cluster().cancelTasks(cancelTasksRequest, listTasksResponseListener(clusterService, groupBy, channel));
+            client.admin().cluster().cancelTasks(cancelTasksRequest, listTasksResponseListener(nodesInCluster, groupBy, channel));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
index 2f60c448e7c..27a777229d1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterAllocationExplainAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplai
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,8 +42,6 @@ import java.io.IOException;
  * Class handling cluster allocation explanation at the REST level
  */
 public class RestClusterAllocationExplainAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterAllocationExplainAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/allocation/explain", this);
@@ -70,10 +67,12 @@ public class RestClusterAllocationExplainAction extends BaseRestHandler {
         try {
             req.includeYesDecisions(request.paramAsBoolean("include_yes_decisions", false));
             req.includeDiskInfo(request.paramAsBoolean("include_disk_info", false));
+            final boolean humanReadable = request.paramAsBoolean("human", false);
             return channel ->
                     client.admin().cluster().allocationExplain(req, new RestBuilderListener<ClusterAllocationExplainResponse>(channel) {
                 @Override
                 public RestResponse buildResponse(ClusterAllocationExplainResponse response, XContentBuilder builder) throws Exception {
+                    builder.humanReadable(humanReadable);
                     response.getExplanation().toXContent(builder, ToXContent.EMPTY_PARAMS);
                     return new BytesRestResponse(RestStatus.OK, builder);
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
index e55603a19bd..5c3be8f4347 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
@@ -46,7 +45,6 @@ public class RestClusterGetSettingsAction extends BaseRestHandler {
     private final ClusterSettings clusterSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestClusterGetSettingsAction(Settings settings, RestController controller, ClusterSettings clusterSettings,
             SettingsFilter settingsFilter) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
index ad51c9f1e64..d8dd34d4a28 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterHealthAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -40,8 +39,6 @@ import java.util.Set;
 import static org.elasticsearch.client.Requests.clusterHealthRequest;
 
 public class RestClusterHealthAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterHealthAction(Settings settings, RestController controller) {
         super(settings);
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
index 31eef941a8b..08ed23c7fb4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java
@@ -24,21 +24,15 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -51,10 +45,10 @@ import java.util.HashSet;
 import java.util.Set;
 
 public class RestClusterRerouteAction extends BaseRestHandler {
-    private static final ObjectParser<ClusterRerouteRequest, ParseContext> PARSER = new ObjectParser<>("cluster_reroute");
+    private static final ObjectParser<ClusterRerouteRequest, Void> PARSER = new ObjectParser<>("cluster_reroute");
     static {
-        PARSER.declareField((p, v, c) -> v.commands(AllocationCommands.fromXContent(p, c.getParseFieldMatcher(), c.registry)),
-                new ParseField("commands"), ValueType.OBJECT_ARRAY);
+        PARSER.declareField((p, v, c) -> v.commands(AllocationCommands.fromXContent(p)), new ParseField("commands"),
+                ValueType.OBJECT_ARRAY);
         PARSER.declareBoolean(ClusterRerouteRequest::dryRun, new ParseField("dry_run"));
     }
 
@@ -62,20 +56,16 @@ public class RestClusterRerouteAction extends BaseRestHandler {
             .arrayToCommaDelimitedString(EnumSet.complementOf(EnumSet.of(ClusterState.Metric.METADATA)).toArray());
 
     private final SettingsFilter settingsFilter;
-    private final AllocationCommandRegistry registry;
 
-    @Inject
-    public RestClusterRerouteAction(Settings settings, RestController controller, SettingsFilter settingsFilter,
-            AllocationCommandRegistry registry) {
+    public RestClusterRerouteAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         this.settingsFilter = settingsFilter;
-        this.registry = registry;
         controller.registerHandler(RestRequest.Method.POST, "/_cluster/reroute", this);
     }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        ClusterRerouteRequest clusterRerouteRequest = createRequest(request, registry, parseFieldMatcher);
+        ClusterRerouteRequest clusterRerouteRequest = createRequest(request);
 
         // by default, return everything but metadata
         final String metric = request.param("metric");
@@ -113,31 +103,14 @@ public class RestClusterRerouteAction extends BaseRestHandler {
         return RESPONSE_PARAMS;
     }
 
-    public static ClusterRerouteRequest createRequest(RestRequest request, AllocationCommandRegistry registry,
-                                                      ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static ClusterRerouteRequest createRequest(RestRequest request) throws IOException {
         ClusterRerouteRequest clusterRerouteRequest = Requests.clusterRerouteRequest();
         clusterRerouteRequest.dryRun(request.paramAsBoolean("dry_run", clusterRerouteRequest.dryRun()));
         clusterRerouteRequest.explain(request.paramAsBoolean("explain", clusterRerouteRequest.explain()));
         clusterRerouteRequest.timeout(request.paramAsTime("timeout", clusterRerouteRequest.timeout()));
         clusterRerouteRequest.setRetryFailed(request.paramAsBoolean("retry_failed", clusterRerouteRequest.isRetryFailed()));
         clusterRerouteRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterRerouteRequest.masterNodeTimeout()));
-        request.applyContentParser(parser -> PARSER.parse(parser, clusterRerouteRequest, new ParseContext(registry, parseFieldMatcher)));
+        request.applyContentParser(parser -> PARSER.parse(parser, clusterRerouteRequest, null));
         return clusterRerouteRequest;
     }
-
-    private static class ParseContext implements ParseFieldMatcherSupplier {
-        private final AllocationCommandRegistry registry;
-        private final ParseFieldMatcher parseFieldMatcher;
-
-        private ParseContext(AllocationCommandRegistry registry, ParseFieldMatcher parseFieldMatcher) {
-            this.registry = registry;
-            this.parseFieldMatcher = parseFieldMatcher;
-        }
-
-        @Override
-        public ParseFieldMatcher getParseFieldMatcher() {
-            return parseFieldMatcher;
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
index 541fb96f65b..90673dccd68 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterSearchShardsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestClusterSearchShardsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterSearchShardsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_search_shards", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
index 54a55a0ba06..fe99d544d46 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -48,7 +47,6 @@ public class RestClusterStateAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestClusterStateAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/state", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
index d1c7346eb0e..e58cf72ffca 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -31,8 +30,6 @@ import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener;
 import java.io.IOException;
 
 public class RestClusterStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/stats", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
index 1213b03c467..02b62b8e4bf 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterUpdateSettingsAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequ
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -37,8 +36,6 @@ import java.util.Map;
 import java.util.Set;
 
 public class RestClusterUpdateSettingsAction extends BaseRestHandler {
-
-    @Inject
     public RestClusterUpdateSettingsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_cluster/settings", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
index 6bd3223003f..bd0d7e2a9d7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestCreateSnapshotAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -38,8 +37,6 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
  * Creates a new snapshot
  */
 public class RestCreateSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestCreateSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}/{snapshot}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
index 780368dfd34..2019e04be55 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteRepositoryAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
  * Unregisters a repository
  */
 public class RestDeleteRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
index 29ad89acda6..a11d47278a8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
  * Deletes a snapshot
  */
 public class RestDeleteSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}/{snapshot}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
index 31a0c70b67b..102ae6bd571 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteStoredScriptAction.java
@@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -33,26 +32,34 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestDeleteStoredScriptAction extends BaseRestHandler {
 
-    @Inject
     public RestDeleteStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
-    }
-
-    protected RestDeleteStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
         super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(DELETE, "/_scripts/{lang}/{id}", this);
-        }
-    }
 
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
+        // Note {lang} is actually {id} in the first handler.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(DELETE, "/_scripts/{lang}", this);
+        controller.registerHandler(DELETE, "/_scripts/{lang}/{id}", this);
     }
 
     @Override
-    public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
-        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(getScriptLang(request), request.param("id"));
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        String lang = request.param("lang");
+
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (id == null) {
+            id = lang;
+            lang = null;
+        }
+
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated");
+        }
+
+        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id, lang);
         return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
index 6e5d06f3740..1af138400e3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetRepositoriesAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -50,7 +49,6 @@ public class RestGetRepositoriesAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestGetRepositoriesAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
index 515ce2d68d0..508953cdde8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetSnapshotsAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -38,8 +37,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * Returns information about snapshot
  */
 public class RestGetSnapshotsAction extends BaseRestHandler {
-
-    @Inject
     public RestGetSnapshotsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
index a2d11e75a67..0c3857ffb58 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetStoredScriptAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -31,6 +31,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.script.StoredScriptSource;
 
 import java.io.IOException;
 
@@ -38,52 +39,76 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestGetStoredScriptAction extends BaseRestHandler {
 
-    @Inject
+    public static final ParseField _ID_PARSE_FIELD = new ParseField("_id");
+
+    public static final ParseField FOUND_PARSE_FIELD = new ParseField("found");
+
     public RestGetStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
-    }
-
-    protected RestGetStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
         super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(GET, "/_scripts/{lang}/{id}", this);
-        }
-    }
 
-    protected String getScriptFieldName() {
-        return Fields.SCRIPT;
-    }
-
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
+        // Note {lang} is actually {id} in the first handler.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(GET, "/_scripts/{lang}", this);
+        controller.registerHandler(GET, "/_scripts/{lang}/{id}", this);
     }
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
-        final GetStoredScriptRequest getRequest = new GetStoredScriptRequest(getScriptLang(request), request.param("id"));
+        String id;
+        String lang;
+
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (request.param("id") == null) {
+            id = request.param("lang");;
+            lang = null;
+        } else {
+            id = request.param("id");
+            lang = request.param("lang");
+        }
+
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated");
+        }
+
+        GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id, lang);
+
         return channel -> client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetStoredScriptResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
-                builder.field(Fields.LANG, getRequest.lang());
-                builder.field(Fields._ID, getRequest.id());
-                boolean found = response.getStoredScript() != null;
-                builder.field(Fields.FOUND, found);
-                RestStatus status = RestStatus.NOT_FOUND;
-                if (found) {
-                    builder.field(getScriptFieldName(), response.getStoredScript());
-                    status = RestStatus.OK;
+                builder.field(_ID_PARSE_FIELD.getPreferredName(), id);
+
+                if (lang != null) {
+                    builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), lang);
                 }
+
+                StoredScriptSource source = response.getSource();
+                boolean found = source != null;
+                builder.field(FOUND_PARSE_FIELD.getPreferredName(), found);
+
+                if (found) {
+                    if (lang == null) {
+                        builder.startObject(StoredScriptSource.SCRIPT_PARSE_FIELD.getPreferredName());
+                        builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), source.getLang());
+                        builder.field(StoredScriptSource.CODE_PARSE_FIELD.getPreferredName(), source.getCode());
+
+                        if (source.getOptions().isEmpty() == false) {
+                            builder.field(StoredScriptSource.OPTIONS_PARSE_FIELD.getPreferredName(), source.getOptions());
+                        }
+
+                        builder.endObject();
+                    } else {
+                        builder.field(StoredScriptSource.SCRIPT_PARSE_FIELD.getPreferredName(), source.getCode());
+                    }
+                }
+
                 builder.endObject();
-                return new BytesRestResponse(status, builder);
+
+                return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
             }
         });
     }
-
-    private static final class Fields {
-        private static final String SCRIPT = "script";
-        private static final String LANG = "lang";
-        private static final String _ID = "_id";
-        private static final String FOUND = "found";
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
index df0945d99b8..e013970553f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetTaskAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,7 +34,6 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestGetTaskAction extends BaseRestHandler {
-    @Inject
     public RestGetTaskAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_tasks/{taskId}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
index 66a3c9cf6d6..4177386eff9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestListTasksAction.java
@@ -23,9 +23,8 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -41,18 +40,18 @@ import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 
 public class RestListTasksAction extends BaseRestHandler {
 
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestListTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestListTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(GET, "/_tasks", this);
     }
 
@@ -60,7 +59,8 @@ public class RestListTasksAction extends BaseRestHandler {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ListTasksRequest listTasksRequest = generateListTasksRequest(request);
         final String groupBy = request.param("group_by", "nodes");
-        return channel -> client.admin().cluster().listTasks(listTasksRequest, listTasksResponseListener(clusterService, groupBy, channel));
+        return channel -> client.admin().cluster().listTasks(listTasksRequest,
+                listTasksResponseListener(nodesInCluster, groupBy, channel));
     }
 
     public static ListTasksRequest generateListTasksRequest(RestRequest request) {
@@ -85,15 +85,15 @@ public class RestListTasksAction extends BaseRestHandler {
      * Standard listener for extensions of {@link ListTasksResponse} that supports {@code group_by=nodes}.
      */
     public static <T extends ListTasksResponse> ActionListener<T> listTasksResponseListener(
-        ClusterService clusterService,
-        String groupBy,
-        final RestChannel channel) {
+                Supplier<DiscoveryNodes> nodesInCluster,
+                String groupBy,
+                final RestChannel channel) {
         if ("nodes".equals(groupBy)) {
             return new RestBuilderListener<T>(channel) {
                 @Override
                 public RestResponse buildResponse(T response, XContentBuilder builder) throws Exception {
                     builder.startObject();
-                    response.toXContentGroupedByNode(builder, channel.request(), clusterService.state().nodes());
+                    response.toXContentGroupedByNode(builder, channel.request(), nodesInCluster.get());
                     builder.endObject();
                     return new BytesRestResponse(RestStatus.OK, builder);
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
index 3cfb80a46f0..48ad588688d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsReq
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -39,8 +38,6 @@ import java.io.IOException;
 
 
 public class RestNodesHotThreadsAction extends BaseRestHandler {
-
-    @Inject
     public RestNodesHotThreadsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/hotthreads", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
index d4da4fc931d..dfe3b08697c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesInfoAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.set.Sets;
@@ -51,7 +50,6 @@ public class RestNodesInfoAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestNodesInfoAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
         super(settings);
         controller.registerHandler(GET, "/_nodes", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
index dce22c0139d..f42a8fcf077 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -43,8 +42,6 @@ import java.util.function.Consumer;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodesStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestNodesStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_nodes/stats", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
index e3441d00d06..29b2b72895b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPendingClusterTasksAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -31,8 +30,6 @@ import org.elasticsearch.rest.action.RestToXContentListener;
 import java.io.IOException;
 
 public class RestPendingClusterTasksAction extends BaseRestHandler {
-
-    @Inject
     public RestPendingClusterTasksAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/pending_tasks", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
index 794ccf410b9..afd2fd851ae 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutRepositoryAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -39,8 +38,6 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
  * Registers repositories
  */
 public class RestPutRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestPutRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
index aec998e6f9a..358c3656ced 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPutStoredScriptAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -34,27 +34,38 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestPutStoredScriptAction extends BaseRestHandler {
 
-    @Inject
     public RestPutStoredScriptAction(Settings settings, RestController controller) {
-        this(settings, controller, true);
-    }
-
-    protected RestPutStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
         super(settings);
-        if (registerDefaultHandlers) {
-            controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
-            controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
-        }
-    }
 
-    protected String getScriptLang(RestRequest request) {
-        return request.param("lang");
+        // Note {lang} is actually {id} in the first two handlers.  It appears
+        // parameters as part of the path must be of the same ordering relative
+        // to name or they will not work as expected.
+        controller.registerHandler(POST, "/_scripts/{lang}", this);
+        controller.registerHandler(PUT, "/_scripts/{lang}", this);
+        controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
+        controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
     }
 
     @Override
-    public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
-        PutStoredScriptRequest putRequest = new PutStoredScriptRequest(getScriptLang(request), request.param("id"));
-        putRequest.script(request.content());
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        String lang = request.param("lang");
+
+        // In the case where only {lang} is not null, we make it {id} because of
+        // name ordering issues in the handlers' paths.
+        if (id == null) {
+            id = lang;
+            lang = null;
+        }
+
+        BytesReference content = request.content();
+
+        if (lang != null) {
+            deprecationLogger.deprecated(
+                "specifying lang [" + lang + "] as part of the url path is deprecated, use request content instead");
+        }
+
+        PutStoredScriptRequest putRequest = new PutStoredScriptRequest(id, lang, content, request.getXContentType());
         return channel -> client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
index a6d0cad7091..3948921c12d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRestoreSnapshotAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
  * Restores a snapshot
  */
 public class RestRestoreSnapshotAction extends BaseRestHandler {
-
-    @Inject
     public RestRestoreSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/{snapshot}/_restore", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
index b517e32a3da..c5122641c1a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestSnapshotsStatusAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -38,8 +37,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * Returns status of currently running snapshot
  */
 public class RestSnapshotsStatusAction extends BaseRestHandler {
-
-    @Inject
     public RestSnapshotsStatusAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}/_status", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
index 84427524466..a9fb44eae64 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestVerifyRepositoryAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -34,8 +33,6 @@ import static org.elasticsearch.client.Requests.verifyRepositoryRequest;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestVerifyRepositoryAction extends BaseRestHandler {
-
-    @Inject
     public RestVerifyRepositoryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/_verify", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java
index 59aafde5930..c2699b1e435 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAliasesExistAction.java
@@ -27,7 +27,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -41,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestAliasesExistAction extends BaseRestHandler {
-
-    @Inject
     public RestAliasesExistAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(HEAD, "/_alias/{name}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
index c32010642e0..52ba58e4c38 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeAction.java
@@ -21,8 +21,6 @@ package org.elasticsearch.rest.action.admin.indices;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -50,7 +48,6 @@ public class RestAnalyzeAction extends BaseRestHandler {
         public static final ParseField ATTRIBUTES = new ParseField("attributes");
     }
 
-    @Inject
     public RestAnalyzeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_analyze", this);
@@ -65,7 +62,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
         AnalyzeRequest analyzeRequest = new AnalyzeRequest(request.param("index"));
 
         try (XContentParser parser = request.contentOrSourceParamParser()) {
-            buildFromContent(parser, analyzeRequest, parseFieldMatcher);
+            buildFromContent(parser, analyzeRequest);
         } catch (IOException e) {
             throw new IllegalArgumentException("Failed to parse request body", e);
         }
@@ -73,7 +70,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
         return channel -> client.admin().indices().analyze(analyzeRequest, new RestToXContentListener<>(channel));
     }
 
-    static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest, ParseFieldMatcher parseFieldMatcher)
+    static void buildFromContent(XContentParser parser, AnalyzeRequest analyzeRequest)
             throws IOException {
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new IllegalArgumentException("Malformed content, must start with an object");
@@ -83,9 +80,9 @@ public class RestAnalyzeAction extends BaseRestHandler {
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.VALUE_STRING) {
+                } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                     analyzeRequest.text(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.TEXT) && token == XContentParser.Token.START_ARRAY) {
+                } else if (Fields.TEXT.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) {
                     List<String> texts = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token.isValue() == false) {
@@ -94,11 +91,11 @@ public class RestAnalyzeAction extends BaseRestHandler {
                         texts.add(parser.text());
                     }
                     analyzeRequest.text(texts.toArray(new String[texts.size()]));
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.ANALYZER) && token == XContentParser.Token.VALUE_STRING) {
+                } else if (Fields.ANALYZER.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                     analyzeRequest.analyzer(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.FIELD) && token == XContentParser.Token.VALUE_STRING) {
+                } else if (Fields.FIELD.match(currentFieldName) && token == XContentParser.Token.VALUE_STRING) {
                     analyzeRequest.field(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKENIZER)) {
+                } else if (Fields.TOKENIZER.match(currentFieldName)) {
                     if (token == XContentParser.Token.VALUE_STRING) {
                         analyzeRequest.tokenizer(parser.text());
                     } else if (token == XContentParser.Token.START_OBJECT) {
@@ -106,7 +103,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
                     } else {
                         throw new IllegalArgumentException(currentFieldName + " should be tokenizer's name or setting");
                     }
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.TOKEN_FILTERS)
+                } else if (Fields.TOKEN_FILTERS.match(currentFieldName)
                         && token == XContentParser.Token.START_ARRAY) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -118,7 +115,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
                                     + " array element should contain filter's name or setting");
                         }
                     }
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.CHAR_FILTERS)
+                } else if (Fields.CHAR_FILTERS.match(currentFieldName)
                         && token == XContentParser.Token.START_ARRAY) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -130,13 +127,13 @@ public class RestAnalyzeAction extends BaseRestHandler {
                                     + " array element should contain char filter's name or setting");
                         }
                     }
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.EXPLAIN)) {
+                } else if (Fields.EXPLAIN.match(currentFieldName)) {
                     if (parser.isBooleanValue()) {
                         analyzeRequest.explain(parser.booleanValue());
                     } else {
                         throw new IllegalArgumentException(currentFieldName + " must be either 'true' or 'false'");
                     }
-                } else if (parseFieldMatcher.match(currentFieldName, Fields.ATTRIBUTES) && token == XContentParser.Token.START_ARRAY) {
+                } else if (Fields.ATTRIBUTES.match(currentFieldName) && token == XContentParser.Token.START_ARRAY) {
                     List<String> attributes = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token.isValue() == false) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
index 942da1f092e..6654deb76fb 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestClearIndicesCacheAction.java
@@ -24,9 +24,7 @@ import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRespo
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -45,8 +43,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestClearIndicesCacheAction extends BaseRestHandler {
-
-    @Inject
     public RestClearIndicesCacheAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_cache/clear", this);
@@ -61,7 +57,7 @@ public class RestClearIndicesCacheAction extends BaseRestHandler {
         ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(
                 Strings.splitStringByCommaToArray(request.param("index")));
         clearIndicesCacheRequest.indicesOptions(IndicesOptions.fromRequest(request, clearIndicesCacheRequest.indicesOptions()));
-        fromRequest(request, clearIndicesCacheRequest, parseFieldMatcher);
+        fromRequest(request, clearIndicesCacheRequest);
         return channel ->
                 client.admin().indices().clearCache(clearIndicesCacheRequest, new RestBuilderListener<ClearIndicesCacheResponse>(channel) {
             @Override
@@ -79,23 +75,22 @@ public class RestClearIndicesCacheAction extends BaseRestHandler {
         return false;
     }
 
-    public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest,
-            ParseFieldMatcher parseFieldMatcher) {
+    public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest) {
 
         for (Map.Entry<String, String> entry : request.params().entrySet()) {
-            if (parseFieldMatcher.match(entry.getKey(), Fields.QUERY)) {
+            if (Fields.QUERY.match(entry.getKey())) {
                 clearIndicesCacheRequest.queryCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.queryCache()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.REQUEST_CACHE)) {
+            if (Fields.REQUEST_CACHE.match(entry.getKey())) {
                 clearIndicesCacheRequest.requestCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.requestCache()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.FIELD_DATA)) {
+            if (Fields.FIELD_DATA.match(entry.getKey())) {
                 clearIndicesCacheRequest.fieldDataCache(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.fieldDataCache()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.RECYCLER)) {
+            if (Fields.RECYCLER.match(entry.getKey())) {
                 clearIndicesCacheRequest.recycler(request.paramAsBoolean(entry.getKey(), clearIndicesCacheRequest.recycler()));
             }
-            if (parseFieldMatcher.match(entry.getKey(), Fields.FIELDS)) {
+            if (Fields.FIELDS.match(entry.getKey())) {
                 clearIndicesCacheRequest.fields(request.paramAsStringArray(entry.getKey(), clearIndicesCacheRequest.fields()));
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
index 68929fda4e1..2e0a46747ca 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCloseIndexAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -33,8 +32,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestCloseIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestCloseIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_close", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
index d1e6919f4c0..8c314efee63 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -34,8 +33,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestCreateIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestCreateIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}", this);
@@ -45,7 +42,7 @@ public class RestCreateIndexAction extends BaseRestHandler {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
         if (request.hasContent()) {
-            createIndexRequest.source(request.content());
+            createIndexRequest.source(request.content(), request.getXContentType());
         }
         createIndexRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false));
         createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
index 94ffc7e989b..6ca806bcffe 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -33,8 +32,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestDeleteIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
index 3d721d27482..e9f315e8aee 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteIndexTemplateAction.java
@@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -30,8 +29,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestDeleteIndexTemplateAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_template/{name}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
index efab741a227..ff084e023a8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestFlushAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -42,8 +41,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestFlushAction extends BaseRestHandler {
-
-    @Inject
     public RestFlushAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_flush", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
index 75e54a2b60f..32820edda0d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -41,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestForceMergeAction extends BaseRestHandler {
-
-    @Inject
     public RestForceMergeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_forcemerge", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
index 531f36aa4b3..c6e13729f6f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -46,8 +46,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetAliasesAction extends BaseRestHandler {
-
-    @Inject
     public RestGetAliasesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_alias/{name}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
index ac51978a399..fe6eb9a552f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRespon
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -44,8 +43,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetFieldMappingAction extends BaseRestHandler {
-
-    @Inject
     public RestGetFieldMappingAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mapping/field/{fields}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
index 0d1752d7368..98241b9dd0b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequ
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -39,8 +38,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetIndexTemplateAction extends BaseRestHandler {
-
-    @Inject
     public RestGetIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
index b7cf07945bc..eaa772583a4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
@@ -28,7 +29,6 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
@@ -53,7 +53,6 @@ public class RestGetIndicesAction extends BaseRestHandler {
     private final IndexScopedSettings indexScopedSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestGetIndicesAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings,
             SettingsFilter settingsFilter) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
index 684a54b7f7e..66e23fd5f47 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -27,7 +28,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -45,8 +45,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetMappingAction extends BaseRestHandler {
-
-    @Inject
     public RestGetMappingAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_mapping", this);
@@ -72,8 +70,10 @@ public class RestGetMappingAction extends BaseRestHandler {
                     if (indices.length != 0 && types.length != 0) {
                         return new BytesRestResponse(OK, builder.startObject().endObject());
                     } else if (indices.length != 0) {
+                        builder.close();
                         return new BytesRestResponse(channel, new IndexNotFoundException(indices[0]));
                     } else if (types.length != 0) {
+                        builder.close();
                         return new BytesRestResponse(channel, new TypeMissingException("_all", types[0]));
                     } else {
                         return new BytesRestResponse(OK, builder.startObject().endObject());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
index 10f0564de84..fcad131a359 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetSettingsAction.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
@@ -47,7 +47,6 @@ public class RestGetSettingsAction extends BaseRestHandler {
     private final IndexScopedSettings indexScopedSettings;
     private final SettingsFilter settingsFilter;
 
-    @Inject
     public RestGetSettingsAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings,
             final SettingsFilter settingsFilter) {
         super(settings);
@@ -76,7 +75,7 @@ public class RestGetSettingsAction extends BaseRestHandler {
                 builder.startObject();
                 for (ObjectObjectCursor<String, Settings> cursor : getSettingsResponse.getIndexToSettings()) {
                     // no settings, jump over it to shorten the response data
-                    if (cursor.value.getAsMap().isEmpty()) {
+                    if (cursor.value.isEmpty()) {
                         continue;
                     }
                     builder.startObject(cursor.key);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java
index bea821e4850..6711f8d3a7b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestHeadIndexTemplateAction.java
@@ -22,7 +22,6 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequ
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -39,8 +38,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestHeadIndexTemplateAction extends BaseRestHandler {
-
-    @Inject
     public RestHeadIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
index d169ce42825..2068faa83e7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexDeleteAliasesAction.java
@@ -22,7 +22,6 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -34,8 +33,6 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestIndexDeleteAliasesAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexDeleteAliasesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/{index}/_alias/{name}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
index 9b6e80df98a..2c68c488610 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java
@@ -22,9 +22,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -38,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexPutAliasAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexPutAliasAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/{index}/_alias/{name}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
index 84eeda9e1e7..58add2b4ea8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesAliasesAction.java
@@ -23,9 +23,6 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -39,7 +36,7 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestIndicesAliasesAction extends BaseRestHandler {
-    static final ObjectParser<IndicesAliasesRequest, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("aliases");
+    static final ObjectParser<IndicesAliasesRequest, Void> PARSER = new ObjectParser<>("aliases");
     static {
         PARSER.declareObjectArray((request, actions) -> {
             for (AliasActions action: actions) {
@@ -48,7 +45,6 @@ public class RestIndicesAliasesAction extends BaseRestHandler {
         }, AliasActions.PARSER, new ParseField("actions"));
     }
 
-    @Inject
     public RestIndicesAliasesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_aliases", this);
@@ -60,7 +56,7 @@ public class RestIndicesAliasesAction extends BaseRestHandler {
         indicesAliasesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", indicesAliasesRequest.masterNodeTimeout()));
         indicesAliasesRequest.timeout(request.paramAsTime("timeout", indicesAliasesRequest.timeout()));
         try (XContentParser parser = request.contentParser()) {
-            PARSER.parse(parser, indicesAliasesRequest, () -> ParseFieldMatcher.STRICT);
+            PARSER.parse(parser, indicesAliasesRequest, null);
         }
         if (indicesAliasesRequest.getAliasActions().isEmpty()) {
             throw new IllegalArgumentException("No action specified");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java
index ed7e656aa0a..525ac6c7e0a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesExistsAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -41,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestIndicesExistsAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesExistsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(HEAD, "/{index}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
index 556bb5b1d1c..6852b8527c7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesSegmentsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -41,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestIndicesSegmentsAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesSegmentsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_segments", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
index cf1ef9ce5ee..c00e6efffb0 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesShardStoresAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -44,8 +43,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
  * Rest action for {@link IndicesShardStoresAction}
  */
 public class RestIndicesShardStoresAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesShardStoresAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_shard_stores", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
index 5fddbe0dff9..ef91ad6accd 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -48,8 +47,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestIndicesStatsAction extends BaseRestHandler {
-
-    @Inject
     public RestIndicesStatsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_stats", this);
@@ -58,7 +55,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
         controller.registerHandler(GET, "/{index}/_stats/{metric}", this);
     }
 
-    static Map<String, Consumer<IndicesStatsRequest>> METRICS;
+    static final Map<String, Consumer<IndicesStatsRequest>> METRICS;
 
     static {
         final Map<String, Consumer<IndicesStatsRequest>> metrics = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
index 23331c7d4a6..4bde6fdb722 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestOpenIndexAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -34,8 +33,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestOpenIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestOpenIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_open", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
index fd8a56b4df1..b376f3dab30 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.admin.indices;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -39,7 +38,6 @@ public class RestPutIndexTemplateAction extends BaseRestHandler {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(RestPutIndexTemplateAction.class));
 
-    @Inject
     public RestPutIndexTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_template/{name}", this);
@@ -59,7 +57,7 @@ public class RestPutIndexTemplateAction extends BaseRestHandler {
         putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
         putRequest.create(request.paramAsBoolean("create", false));
         putRequest.cause(request.param("cause", ""));
-        putRequest.source(request.content());
+        putRequest.source(request.content(), request.getXContentType());
         return channel -> client.admin().indices().putTemplate(putRequest, new AcknowledgedRestListener<>(channel));
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
index 0fa394e6f3d..06fe6792645 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
@@ -20,11 +20,9 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -38,9 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestPutMappingAction extends BaseRestHandler {
-
-
-    @Inject
     public RestPutMappingAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(PUT, "/{index}/_mapping/", this);
@@ -69,11 +64,11 @@ public class RestPutMappingAction extends BaseRestHandler {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index")));
         putMappingRequest.type(request.param("type"));
-        putMappingRequest.source(request.content().utf8ToString());
+        putMappingRequest.source(request.content(), request.getXContentType());
         putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false));
         putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout()));
         putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout()));
         putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions()));
-        return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<PutMappingResponse>(channel));
+        return channel -> client.admin().indices().putMapping(putMappingRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
index c20c70c8c25..ab1295e9c60 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRecoveryAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -43,8 +42,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
  * REST handler to report on index recoveries.
  */
 public class RestRecoveryAction extends BaseRestHandler {
-
-    @Inject
     public RestRecoveryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_recovery", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
index 96afc014e97..c213f3be18b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRefreshAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -42,8 +41,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestRefreshAction extends BaseRestHandler {
-
-    @Inject
     public RestRefreshAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_refresh", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index ec42b426401..59068f277d6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -22,8 +22,6 @@ package org.elasticsearch.rest.action.admin.indices;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -33,8 +31,6 @@ import org.elasticsearch.rest.action.RestToXContentListener;
 import java.io.IOException;
 
 public class RestRolloverIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestRolloverIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
@@ -44,7 +40,7 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index"));
-        request.applyContentParser(parser -> RolloverRequest.PARSER.parse(parser, rolloverIndexRequest, () -> ParseFieldMatcher.EMPTY));
+        request.applyContentParser(parser -> RolloverRequest.PARSER.parse(parser, rolloverIndexRequest, null));
         rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false));
         rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout()));
         rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
index b533c8e5e7a..1b63eb7f6d3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -23,8 +23,6 @@ import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
 import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,8 +33,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestShrinkIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestShrinkIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_shrink/{target}", this);
@@ -52,7 +48,7 @@ public class RestShrinkIndexAction extends BaseRestHandler {
             throw new IllegalArgumentException("no source index");
         }
         ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index"));
-        request.applyContentParser(parser -> ShrinkRequest.PARSER.parse(parser, shrinkIndexRequest, () -> ParseFieldMatcher.EMPTY));
+        request.applyContentParser(parser -> ShrinkRequest.PARSER.parse(parser, shrinkIndexRequest, null));
         shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout()));
         shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout()));
         shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards")));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
index f645c32be71..8a311331729 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,8 +39,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSyncedFlushAction extends BaseRestHandler {
-
-    @Inject
     public RestSyncedFlushAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_flush/synced", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java
index 799ba8b1c62..dcded02eb0d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestTypesExistsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -43,8 +42,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
  * Rest api for checking if a type exists.
  */
 public class RestTypesExistsAction extends BaseRestHandler {
-
-    @Inject
     public RestTypesExistsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerWithDeprecatedHandler(
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
index 4c6730d5b15..47037460e2b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpdateSettingsAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -50,7 +49,6 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
             "ignore_unavailable",
             "allow_no_indices"));
 
-    @Inject
     public RestUpdateSettingsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_settings", this);
@@ -68,7 +66,9 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
         Settings.Builder updateSettings = Settings.builder();
         String bodySettingsStr = request.content().utf8ToString();
         if (Strings.hasText(bodySettingsStr)) {
-            Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build();
+            Settings buildSettings = Settings.builder()
+                .loadFromSource(bodySettingsStr, request.getXContentType())
+                .build();
             for (Map.Entry<String, String> entry : buildSettings.getAsMap().entrySet()) {
                 String key = entry.getKey();
                 String value = entry.getValue();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
index 9882b5bea3d..9437ad5eada 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestUpgradeAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -47,8 +46,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestUpgradeAction extends BaseRestHandler {
-
-    @Inject
     public RestUpgradeAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_upgrade", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
index 8b68a3b6168..d2f0f5ec3d2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestValidateQueryAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
@@ -27,12 +26,8 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
@@ -50,11 +45,7 @@ import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader;
 
 public class RestValidateQueryAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestValidateQueryAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestValidateQueryAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_validate/query", this);
         controller.registerHandler(POST, "/_validate/query", this);
@@ -62,7 +53,6 @@ public class RestValidateQueryAction extends BaseRestHandler {
         controller.registerHandler(POST, "/{index}/_validate/query", this);
         controller.registerHandler(GET, "/{index}/{type}/_validate/query", this);
         controller.registerHandler(POST, "/{index}/{type}/_validate/query", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
@@ -77,7 +67,7 @@ public class RestValidateQueryAction extends BaseRestHandler {
         try {
             request.withContentOrSourceParamParserOrNull(parser -> {
                 if (parser != null) {
-                    validateQueryRequest.query(RestActions.getQueryContent(parser, indicesQueriesRegistry, parseFieldMatcher));
+                    validateQueryRequest.query(RestActions.getQueryContent(parser));
                 } else if (request.hasParam("q")) {
                     validateQueryRequest.query(RestActions.urlParamsToQueryBuilder(request));
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
index 7289e9c76e7..806a89166d5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
@@ -43,11 +43,11 @@ public abstract class AbstractCatAction extends BaseRestHandler {
         super(settings);
     }
 
-    protected abstract RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client);
+    protected abstract RestChannelConsumer doCatRequest(RestRequest request, NodeClient client);
 
     protected abstract void documentation(StringBuilder sb);
 
-    protected abstract Table getTableWithHeader(final RestRequest request);
+    protected abstract Table getTableWithHeader(RestRequest request);
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index 981b573f0dc..120678d2e82 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -19,13 +19,13 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -37,15 +37,12 @@ import java.util.List;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestAliasAction extends AbstractCatAction {
-
-    @Inject
     public RestAliasAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/aliases", this);
         controller.registerHandler(GET, "/_cat/aliases/{alias}", this);
     }
 
-
     @Override
     protected RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
index 86ff1c44801..0077297cbf3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.ObjectIntScatterMap;
+
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -31,7 +32,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.rest.RestController;
@@ -44,8 +44,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 
 public class RestAllocationAction extends AbstractCatAction {
-
-    @Inject
     public RestAllocationAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/allocation", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
index c95b35a989c..7442a7d85ee 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
@@ -29,7 +29,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
-import java.util.Set;
+import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
@@ -40,7 +40,7 @@ public class RestCatAction extends BaseRestHandler {
     private final String HELP;
 
     @Inject
-    public RestCatAction(Settings settings, RestController controller, Set<AbstractCatAction> catActions) {
+    public RestCatAction(Settings settings, RestController controller, List<AbstractCatAction> catActions) {
         super(settings);
         controller.registerHandler(GET, "/_cat", this);
         StringBuilder sb = new StringBuilder();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index 22346ad2772..00eb89a9c02 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -25,10 +25,8 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -41,15 +39,10 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestCountAction extends AbstractCatAction {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestCountAction(Settings settings, RestController restController, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestCountAction(Settings settings, RestController restController) {
         super(settings);
         restController.registerHandler(GET, "/_cat/count", this);
         restController.registerHandler(GET, "/_cat/count/{index}", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
@@ -72,7 +65,7 @@ public class RestCountAction extends AbstractCatAction {
                         searchSourceBuilder.query(queryBuilder);
                     }
                 } else {
-                    searchSourceBuilder.query(RestActions.getQueryContent(parser, indicesQueriesRegistry, parseFieldMatcher));
+                    searchSourceBuilder.query(RestActions.getQueryContent(parser));
                 }
             });
         } catch (IOException e) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
index 4ad4d8aca27..4156ea46192 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.rest.RestController;
@@ -39,8 +39,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * Cat API class to display information about the size of fielddata fields per node
  */
 public class RestFielddataAction extends AbstractCatAction {
-
-    @Inject
     public RestFielddataAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/fielddata", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
index fc6462bcaa6..2bc1c11b9a1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -35,8 +34,6 @@ import java.util.Locale;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestHealthAction extends AbstractCatAction {
-
-    @Inject
     public RestHealthAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/health", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index e56347f16be..46bbc4652cf 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -37,7 +37,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestController;
@@ -60,7 +59,6 @@ public class RestIndicesAction extends AbstractCatAction {
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
-    @Inject
     public RestIndicesAction(Settings settings, RestController controller, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -260,6 +258,9 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("refresh.time", "sibling:pri;alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
         table.addCell("pri.refresh.time", "default:false;text-align:right;desc:time spent in refreshes");
 
+        table.addCell("refresh.listeners", "sibling:pri;alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners");
+        table.addCell("pri.refresh.listeners", "default:false;text-align:right;desc:number of pending refresh listeners");
+
         table.addCell("search.fetch_current", "sibling:pri;alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("pri.search.fetch_current", "default:false;text-align:right;desc:current fetch phase ops");
 
@@ -475,6 +476,9 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotalTime());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotalTime());
 
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getListeners());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getListeners());
+
             table.addCell(indexStats == null ? null : indexStats.getTotal().getSearch().getTotal().getFetchCurrent());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSearch().getTotal().getFetchCurrent());
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
index be2aeaafd4e..db5c1149cc0 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -35,8 +34,6 @@ import org.elasticsearch.rest.action.RestResponseListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestMasterAction extends AbstractCatAction {
-
-    @Inject
     public RestMasterAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/master", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
index a5f33d916fd..6d9c5bd57e5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
@@ -29,9 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -43,8 +41,6 @@ import java.util.Map;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodeAttrsAction extends AbstractCatAction {
-
-    @Inject
     public RestNodeAttrsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/nodeattrs", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index b632448192d..faa07e7ddaa 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -32,7 +32,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -67,8 +66,6 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestNodesAction extends AbstractCatAction {
-
-    @Inject
     public RestNodesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/nodes", this);
@@ -193,6 +190,8 @@ public class RestNodesAction extends AbstractCatAction {
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
+        table.addCell("refresh.listeners", "alias:rli,refreshListeners;default:false;text-align:right;"
+                + "desc:number of pending refresh listeners");
 
         table.addCell("script.compilations", "alias:scrcc,scriptCompilations;default:false;text-align:right;desc:script compilations");
         table.addCell("script.cache_evictions",
@@ -346,6 +345,7 @@ public class RestNodesAction extends AbstractCatAction {
             RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
             table.addCell(refreshStats == null ? null : refreshStats.getTotal());
             table.addCell(refreshStats == null ? null : refreshStats.getTotalTime());
+            table.addCell(refreshStats == null ? null : refreshStats.getListeners());
 
             ScriptStats scriptStats = stats == null ? null : stats.getScriptStats();
             table.addCell(scriptStats == null ? null : scriptStats.getCompilations());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
index aa92164506b..a9f044c1446 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -34,7 +33,6 @@ import org.elasticsearch.rest.action.RestResponseListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestPendingClusterTasksAction extends AbstractCatAction {
-    @Inject
     public RestPendingClusterTasksAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/pending_tasks", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
index b729c4879f3..7851c15b32f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.PluginInfo;
 import org.elasticsearch.rest.RestController;
@@ -40,8 +39,6 @@ import org.elasticsearch.rest.action.RestResponseListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestPluginsAction extends AbstractCatAction {
-
-    @Inject
     public RestPluginsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/plugins", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 612c2aaf427..e2e831f890d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.indices.recovery.RecoveryState;
@@ -49,9 +48,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * be specified to limit output to a particular index or indices.
  */
 public class RestRecoveryAction extends AbstractCatAction {
-
-    @Inject
-    public RestRecoveryAction(Settings settings, RestController restController, RestController controller) {
+    public RestRecoveryAction(Settings settings, RestController restController) {
         super(settings);
         restController.registerHandler(GET, "/_cat/recovery", this);
         restController.registerHandler(GET, "/_cat/recovery/{index}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
index 71aab69ad49..631c0305049 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRe
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -37,7 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * Cat API class to display information about snapshot repositories
  */
 public class RestRepositoriesAction extends AbstractCatAction {
-    @Inject
     public RestRepositoriesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/repositories", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
index 6b665d957d2..48983ab836b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
@@ -30,7 +30,6 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.engine.Segment;
 import org.elasticsearch.rest.RestController;
@@ -45,8 +44,6 @@ import java.util.Map;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestSegmentsAction extends AbstractCatAction {
-
-    @Inject
     public RestSegmentsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/segments", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index f8337fd25c4..8ce4ec0f8dc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.engine.CommitStats;
@@ -47,8 +46,6 @@ import java.util.Locale;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestShardsAction extends AbstractCatAction {
-
-    @Inject
     public RestShardsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/shards", this);
@@ -144,6 +141,7 @@ public class RestShardsAction extends AbstractCatAction {
 
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
+        table.addCell("refresh.listeners", "alias:rli,refreshListeners;default:false;text-align:right;desc:number of pending refresh listeners");
 
         table.addCell("search.fetch_current", "alias:sfc,searchFetchCurrent;default:false;text-align:right;desc:current fetch phase ops");
         table.addCell("search.fetch_time", "alias:sfti,searchFetchTime;default:false;text-align:right;desc:time spent in fetch phase");
@@ -194,7 +192,7 @@ public class RestShardsAction extends AbstractCatAction {
             IndexMetaData indexMeta = state.getState().getMetaData().getIndexSafe(shard.index());
             boolean usesShadowReplicas = false;
             if (indexMeta != null) {
-                usesShadowReplicas = IndexMetaData.isIndexUsingShadowReplicas(indexMeta.getSettings());
+                usesShadowReplicas = indexMeta.isIndexUsingShadowReplicas();
             }
             if (shard.primary()) {
                 table.addCell("p");
@@ -290,6 +288,7 @@ public class RestShardsAction extends AbstractCatAction {
 
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal());
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime());
+            table.addCell(commonStats == null ? null : commonStats.getRefresh().getListeners());
 
             table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchCurrent());
             table.addCell(commonStats == null ? null : commonStats.getSearch().getTotal().getFetchTime());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index ead3696281a..54337e5e143 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestController;
@@ -44,7 +43,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  * Cat API class to display information about snapshots
  */
 public class RestSnapshotAction extends AbstractCatAction {
-    @Inject
     public RestSnapshotAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/snapshots", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
index 0ab2c86453e..5ebf36c63a1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTable.java
@@ -461,7 +461,7 @@ public class RestTable {
         private final String column;
         private final boolean reverse;
 
-        public ColumnOrderElement(String column, boolean reverse) {
+        ColumnOrderElement(String column, boolean reverse) {
             this.column = column;
             this.reverse = reverse;
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index 2c490debea0..5a49feb9c4b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -24,12 +24,9 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -44,18 +41,18 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.generateListTasksRequest;
 
 public class RestTasksAction extends AbstractCatAction {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestTasksAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
         controller.registerHandler(GET, "/_cat/tasks", this);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
     }
 
     @Override
@@ -155,7 +152,7 @@ public class RestTasksAction extends AbstractCatAction {
     }
 
     private void buildGroups(Table table, boolean fullId, boolean detailed, List<TaskGroup> taskGroups) {
-        DiscoveryNodes discoveryNodes = clusterService.state().nodes();
+        DiscoveryNodes discoveryNodes = nodesInCluster.get();
         List<TaskGroup> sortedGroups = new ArrayList<>(taskGroups);
         sortedGroups.sort((o1, o2) -> Long.compare(o1.getTaskInfo().getStartTime(), o2.getTaskInfo().getStartTime()));
         for (TaskGroup taskGroup : sortedGroups) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
index 486fbb93d76..b624a9cea18 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTemplatesAction.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.rest.action.cat;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
@@ -37,7 +37,6 @@ import org.elasticsearch.rest.action.RestResponseListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestTemplatesAction extends AbstractCatAction {
-    @Inject
     public RestTemplatesAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/templates", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index 183a8eefabd..111e190638d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -31,10 +31,8 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -53,8 +51,6 @@ import java.util.TreeMap;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestThreadPoolAction extends AbstractCatAction {
-
-    @Inject
     public RestThreadPoolAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_cat/thread_pool", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
index ff603aceefb..8fdf2792a46 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java
@@ -27,7 +27,6 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -61,7 +60,6 @@ public class RestBulkAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestBulkAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -95,7 +93,7 @@ public class RestBulkAction extends BaseRestHandler {
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
         bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields,
-            defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex);
+            defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType());
 
         return channel -> client.bulk(bulkRequest, new RestBuilderListener<BulkResponse>(channel) {
             @Override
@@ -108,9 +106,7 @@ public class RestBulkAction extends BaseRestHandler {
                 builder.field(Fields.ERRORS, response.hasFailures());
                 builder.startArray(Fields.ITEMS);
                 for (BulkItemResponse itemResponse : response) {
-                    builder.startObject();
                     itemResponse.toXContent(builder, request);
-                    builder.endObject();
                 }
                 builder.endArray();
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
index 2983879961c..5de436b38c3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestCountAction.java
@@ -19,18 +19,14 @@
 
 package org.elasticsearch.rest.action.document;
 
-import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
@@ -48,11 +44,7 @@ import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHead
 import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER;
 
 public class RestCountAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestCountAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestCountAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/_count", this);
         controller.registerHandler(GET, "/_count", this);
@@ -60,7 +52,6 @@ public class RestCountAction extends BaseRestHandler {
         controller.registerHandler(GET, "/{index}/_count", this);
         controller.registerHandler(POST, "/{index}/{type}/_count", this);
         controller.registerHandler(GET, "/{index}/{type}/_count", this);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
@@ -76,7 +67,7 @@ public class RestCountAction extends BaseRestHandler {
                     searchSourceBuilder.query(queryBuilder);
                 }
             } else {
-                searchSourceBuilder.query(RestActions.getQueryContent(parser, indicesQueriesRegistry, parseFieldMatcher));
+                searchSourceBuilder.query(RestActions.getQueryContent(parser));
             }
         });
         countRequest.routing(request.param("routing"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
index 80e4efa4ecb..cd1084e4d26 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestDeleteAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.document;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -36,8 +35,6 @@ import java.io.IOException;
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestDeleteAction extends BaseRestHandler {
-
-    @Inject
     public RestDeleteAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(DELETE, "/{index}/{type}/{id}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
index 7206e6b9d5e..fc131aee3f6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java
@@ -23,17 +23,14 @@ import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.io.IOException;
@@ -43,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetAction extends BaseRestHandler {
-
-    @Inject
     public RestGetAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}", this);
@@ -76,17 +71,10 @@ public class RestGetAction extends BaseRestHandler {
 
         getRequest.fetchSourceContext(FetchSourceContext.parseFromRestRequest(request));
 
-        return channel -> client.get(getRequest, new RestBuilderListener<GetResponse>(channel) {
+        return channel -> client.get(getRequest, new RestToXContentListener<GetResponse>(channel) {
             @Override
-            public RestResponse buildResponse(GetResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                response.toXContent(builder, request);
-                builder.endObject();
-                if (!response.isExists()) {
-                    return new BytesRestResponse(NOT_FOUND, builder);
-                } else {
-                    return new BytesRestResponse(OK, builder);
-                }
+            protected RestStatus getStatus(GetResponse response) {
+                return response.isExists() ? OK : NOT_FOUND;
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
index ed36cc3c89d..83d424ed74d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestGetSourceAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -41,8 +40,6 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetSourceAction extends BaseRestHandler {
-
-    @Inject
     public RestGetSourceAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_source", this);
@@ -69,7 +66,7 @@ public class RestGetSourceAction extends BaseRestHandler {
                 client.get(getRequest, new RestResponseListener<GetResponse>(channel) {
                     @Override
                     public RestResponse buildResponse(GetResponse response) throws Exception {
-                        XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false);
+                        XContentBuilder builder = channel.newBuilder(request.getXContentType(), false);
                         if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing
                             return new BytesRestResponse(NOT_FOUND, builder);
                         } else {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java
index 0adb4fcfd28..cc90b2cfa0f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestHeadAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -48,8 +47,6 @@ public abstract class RestHeadAction extends BaseRestHandler {
      * Handler to check for document existence.
      */
     public static class Document extends RestHeadAction {
-
-        @Inject
         public Document(Settings settings, RestController controller) {
             super(settings, false);
             controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
@@ -60,8 +57,6 @@ public abstract class RestHeadAction extends BaseRestHandler {
      * Handler to check for document source existence (may be disabled in the mapping).
      */
     public static class Source extends RestHeadAction {
-
-        @Inject
         public Source(Settings settings, RestController controller) {
             super(settings, true);
             controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
index 3880ec6ca9e..ddaf2268756 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.document;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -32,13 +31,12 @@ import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
 public class RestIndexAction extends BaseRestHandler {
-
-    @Inject
     public RestIndexAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/{index}/{type}", this); // auto id creation
@@ -67,7 +65,7 @@ public class RestIndexAction extends BaseRestHandler {
         indexRequest.routing(request.param("routing"));
         indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
         indexRequest.setPipeline(request.param("pipeline"));
-        indexRequest.source(request.content());
+        indexRequest.source(request.content(), request.getXContentType());
         indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT));
         indexRequest.setRefreshPolicy(request.param("refresh"));
         indexRequest.version(RestActions.parseVersion(request));
@@ -82,7 +80,14 @@ public class RestIndexAction extends BaseRestHandler {
         }
 
         return channel ->
-            client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing())));
+            client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> {
+                try {
+                    return r.getLocation(indexRequest.routing());
+                } catch (URISyntaxException ex) {
+                    logger.warn("Location string is not a valid URI.", ex);
+                    return null;
+                }
+            }));
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
index 3cb52063368..cba8fafbda3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiGetAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.document;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,7 +39,6 @@ public class RestMultiGetAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestMultiGetAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mget", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
index 2b7359f9a1b..605c8654f50 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestMultiTermVectorsAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +36,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestMultiTermVectorsAction extends BaseRestHandler {
-
-    @Inject
     public RestMultiTermVectorsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_mtermvectors", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
index d71b66bed5c..a649e5eff47 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestTermVectorsAction.java
@@ -22,9 +22,7 @@ package org.elasticsearch.rest.action.document;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -45,8 +43,6 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
  * TermVectorsRequest.
  */
 public class RestTermVectorsAction extends BaseRestHandler {
-
-    @Inject
     public RestTermVectorsAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_termvectors", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
index ade0200215a..f9593072daa 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -37,6 +36,7 @@ import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -44,7 +44,6 @@ public class RestUpdateAction extends BaseRestHandler {
     private static final DeprecationLogger DEPRECATION_LOGGER =
         new DeprecationLogger(Loggers.getLogger(RestUpdateAction.class));
 
-    @Inject
     public RestUpdateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_update", this);
@@ -99,6 +98,13 @@ public class RestUpdateAction extends BaseRestHandler {
         });
 
         return channel ->
-            client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(updateRequest.routing())));
+            client.update(updateRequest, new RestStatusToXContentListener<>(channel, r -> {
+                try {
+                    return r.getLocation(updateRequest.routing());
+                } catch (URISyntaxException ex) {
+                    logger.warn("Location string is not a valid URI.", ex);
+                    return null;
+                }
+            }));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
index f9430ea0de4..b776d2475ce 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -31,8 +30,6 @@ import org.elasticsearch.rest.action.AcknowledgedRestListener;
 import java.io.IOException;
 
 public class RestDeletePipelineAction extends BaseRestHandler {
-
-    @Inject
     public RestDeletePipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_ingest/pipeline/{id}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
index 7124bb35b31..c8facf7b4cc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest.action.ingest;
 import org.elasticsearch.action.ingest.GetPipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -32,8 +31,6 @@ import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import java.io.IOException;
 
 public class RestGetPipelineAction extends BaseRestHandler {
-
-    @Inject
     public RestGetPipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
index 08bd65f6892..2496c9b4a24 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
@@ -21,20 +21,19 @@ package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.AcknowledgedRestListener;
-import org.elasticsearch.rest.action.RestActions;
 
 import java.io.IOException;
 
 
 public class RestPutPipelineAction extends BaseRestHandler {
-
-    @Inject
     public RestPutPipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_ingest/pipeline/{id}", this);
@@ -42,7 +41,8 @@ public class RestPutPipelineAction extends BaseRestHandler {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), restRequest.contentOrSourceParam());
+        Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam();
+        PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), sourceTuple.v2(), sourceTuple.v1());
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
         return channel -> client.admin().cluster().putPipeline(request, new AcknowledgedRestListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
index eb29be1994f..9dbe1808a8c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
@@ -21,19 +21,18 @@ package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 
 import java.io.IOException;
 
 public class RestSimulatePipelineAction extends BaseRestHandler {
-
-    @Inject
     public RestSimulatePipelineAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_ingest/pipeline/{id}/_simulate", this);
@@ -44,7 +43,8 @@ public class RestSimulatePipelineAction extends BaseRestHandler {
 
     @Override
     public RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
-        SimulatePipelineRequest request = new SimulatePipelineRequest(restRequest.contentOrSourceParam());
+        Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam();
+        SimulatePipelineRequest request = new SimulatePipelineRequest(sourceTuple.v2(), sourceTuple.v1());
         request.setId(restRequest.param("id"));
         request.setVerbose(restRequest.paramAsBoolean("verbose", false));
         return channel -> client.admin().cluster().simulatePipeline(request, new RestToXContentListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
index 675ba25d5d7..47252f5a102 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
@@ -20,13 +20,10 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -39,8 +36,6 @@ import java.util.Arrays;
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
 public class RestClearScrollAction extends BaseRestHandler {
-
-    @Inject
     public RestClearScrollAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -53,26 +48,34 @@ public class RestClearScrollAction extends BaseRestHandler {
         String scrollIds = request.param("scroll_id");
         ClearScrollRequest clearRequest = new ClearScrollRequest();
         clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
-        BytesReference body = request.contentOrSourceParam();
-        if (body.length() > 0) {
-            if (XContentFactory.xContentType(body) == null) {
-                scrollIds = body.utf8ToString();
-                clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
+        request.withContentOrSourceParamParserOrNullLenient((xContentParser -> {
+            if (xContentParser == null) {
+                if (request.hasContent()) {
+                    // TODO: why do we accept this plain text value? maybe we can just use the scroll params?
+                    BytesReference body = request.content();
+                    String bodyScrollIds = body.utf8ToString();
+                    clearRequest.setScrollIds(Arrays.asList(splitScrollIds(bodyScrollIds)));
+                }
             } else {
                 // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value
                 clearRequest.setScrollIds(null);
-                try (XContentParser parser = request.contentOrSourceParamParser()) {
-                    buildFromContent(parser, clearRequest);
+                try {
+                    buildFromContent(xContentParser, clearRequest);
                 } catch (IOException e) {
                     throw new IllegalArgumentException("Failed to parse request body", e);
                 }
             }
-        }
+        }));
 
-        return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<ClearScrollResponse>(channel));
+        return channel -> client.clearScroll(clearRequest, new RestStatusToXContentListener<>(channel));
     }
 
-    public static String[] splitScrollIds(String scrollIds) {
+    @Override
+    public boolean supportsPlainText() {
+        return true;
+    }
+
+    private static String[] splitScrollIds(String scrollIds) {
         if (scrollIds == null) {
             return Strings.EMPTY_ARRAY;
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
index 7d31ed7cad5..7339718c28b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestExplainAction.java
@@ -24,12 +24,10 @@ import org.elasticsearch.action.explain.ExplainRequest;
 import org.elasticsearch.action.explain.ExplainResponse;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
@@ -50,13 +48,8 @@ import static org.elasticsearch.rest.RestStatus.OK;
  * Rest action for computing a score explanation for specific documents.
  */
 public class RestExplainAction extends BaseRestHandler {
-
-    private final IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestExplainAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+    public RestExplainAction(Settings settings, RestController controller) {
         super(settings);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
         controller.registerHandler(GET, "/{index}/{type}/{id}/_explain", this);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_explain", this);
     }
@@ -70,7 +63,7 @@ public class RestExplainAction extends BaseRestHandler {
         String queryString = request.param("q");
         request.withContentOrSourceParamParserOrNull(parser -> {
             if (parser != null) {
-                explainRequest.query(RestActions.getQueryContent(parser, indicesQueriesRegistry, parseFieldMatcher));
+                explainRequest.query(RestActions.getQueryContent(parser));
             } else if (queryString != null) {
                 QueryBuilder query = RestActions.urlParamsToQueryBuilder(request);
                 explainRequest.query(query);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 6dfd587f848..a8039586186 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -24,27 +24,25 @@ import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.function.BiConsumer;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -53,12 +51,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestMultiSearchAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
-    private final SearchRequestParsers searchRequestParsers;
 
-    @Inject
-    public RestMultiSearchAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestMultiSearchAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
 
         controller.registerHandler(GET, "/_msearch", this);
         controller.registerHandler(POST, "/_msearch", this);
@@ -72,17 +67,14 @@ public class RestMultiSearchAction extends BaseRestHandler {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, searchRequestParsers, parseFieldMatcher);
+        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex);
         return channel -> client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
     }
 
     /**
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchRequest}
      */
-    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex,
-                                                  SearchRequestParsers searchRequestParsers,
-                                                  ParseFieldMatcher parseFieldMatcher) throws IOException {
-
+    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
         MultiSearchRequest multiRequest = new MultiSearchRequest();
         if (restRequest.hasParam("max_concurrent_searches")) {
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
@@ -90,10 +82,8 @@ public class RestMultiSearchAction extends BaseRestHandler {
 
         parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> {
             try {
-                final QueryParseContext queryParseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser,
-                        parseFieldMatcher);
-                searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext,
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers));
+                final QueryParseContext queryParseContext = new QueryParseContext(parser);
+                searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext));
                 multiRequest.add(searchRequest);
             } catch (IOException e) {
                 throw new ElasticsearchParseException("Exception when parsing search request", e);
@@ -104,19 +94,20 @@ public class RestMultiSearchAction extends BaseRestHandler {
     }
 
     /**
-     * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer.
+     * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer.
      */
     public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex,
-                                             BiConsumer<SearchRequest, XContentParser> consumer) throws IOException {
+            BiConsumer<SearchRequest, XContentParser> consumer) throws IOException {
 
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] types = Strings.splitStringByCommaToArray(request.param("type"));
         String searchType = request.param("search_type");
         String routing = request.param("routing");
 
-        final BytesReference data = request.contentOrSourceParam();
+        final Tuple<XContentType, BytesReference> sourceTuple = request.contentOrSourceParam();
+        final XContent xContent = sourceTuple.v1().xContent();
+        final BytesReference data = sourceTuple.v2();
 
-        XContent xContent = XContentFactory.xContent(data);
         int from = 0;
         int length = data.length();
         byte marker = xContent.streamSeparator();
@@ -153,7 +144,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
 
             // now parse the action
             if (nextMarker - from > 0) {
-                try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
+                try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), data.slice(from, nextMarker - from))) {
                     Map<String, Object> source = parser.map();
                     for (Map.Entry<String, Object> entry : source.entrySet()) {
                         Object value = entry.getValue();
@@ -167,7 +158,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
                         } else if ("search_type".equals(entry.getKey()) || "searchType".equals(entry.getKey())) {
                             searchRequest.searchType(nodeStringValue(value, null));
                         } else if ("request_cache".equals(entry.getKey()) || "requestCache".equals(entry.getKey())) {
-                            searchRequest.requestCache(lenientNodeBooleanValue(value));
+                            searchRequest.requestCache(nodeBooleanValue(value, entry.getKey()));
                         } else if ("preference".equals(entry.getKey())) {
                             searchRequest.preference(nodeStringValue(value, null));
                         } else if ("routing".equals(entry.getKey())) {
@@ -187,7 +178,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
                 break;
             }
             BytesReference bytes = data.slice(from, nextMarker - from);
-            try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) {
+            try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) {
                 consumer.accept(searchRequest, parser);
             }
             // move pointers
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 425acbf41e3..28f4b99e53f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -23,9 +23,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -36,7 +34,6 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.search.Scroll;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -54,13 +51,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
 
 public class RestSearchAction extends BaseRestHandler {
-
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public RestSearchAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestSearchAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
         controller.registerHandler(GET, "/_search", this);
         controller.registerHandler(POST, "/_search", this);
         controller.registerHandler(GET, "/{index}/_search", this);
@@ -73,7 +65,7 @@ public class RestSearchAction extends BaseRestHandler {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
         request.withContentOrSourceParamParserOrNull(parser ->
-            parseSearchRequest(searchRequest, request, searchRequestParsers, parseFieldMatcher, parser));
+            parseSearchRequest(searchRequest, request, parser));
 
         return channel -> client.search(searchRequest, new RestStatusToXContentListener<>(channel));
     }
@@ -84,25 +76,24 @@ public class RestSearchAction extends BaseRestHandler {
      * @param requestContentParser body of the request to read. This method does not attempt to read the body from the {@code request}
      *        parameter
      */
-    public static void parseSearchRequest(SearchRequest searchRequest, RestRequest request, SearchRequestParsers searchRequestParsers,
-                                          ParseFieldMatcher parseFieldMatcher, XContentParser requestContentParser) throws IOException {
+    public static void parseSearchRequest(SearchRequest searchRequest, RestRequest request,
+                                          XContentParser requestContentParser) throws IOException {
 
         if (searchRequest.source() == null) {
             searchRequest.source(new SearchSourceBuilder());
         }
         searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
         if (requestContentParser != null) {
-            QueryParseContext context = new QueryParseContext(searchRequestParsers.queryParsers, requestContentParser, parseFieldMatcher);
-            searchRequest.source().parseXContent(context, searchRequestParsers.aggParsers, searchRequestParsers.suggesters,
-                    searchRequestParsers.searchExtParsers);
+            QueryParseContext context = new QueryParseContext(requestContentParser);
+            searchRequest.source().parseXContent(context);
         }
 
         // do not allow 'query_and_fetch' or 'dfs_query_and_fetch' search types
         // from the REST layer. these modes are an internal optimization and should
         // not be specified explicitly by the user.
         String searchType = request.param("search_type");
-        if (SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.QUERY_AND_FETCH) ||
-                SearchType.fromString(searchType, parseFieldMatcher).equals(SearchType.DFS_QUERY_AND_FETCH)) {
+        if (SearchType.fromString(searchType).equals(SearchType.QUERY_AND_FETCH) ||
+                "dfs_query_and_fetch".equals(searchType)) {
             throw new IllegalArgumentException("Unsupported search type [" + searchType + "]");
         } else {
             searchRequest.searchType(searchType);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
index 5a8e5f8efe5..2a60fc6317a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
@@ -22,10 +22,8 @@ package org.elasticsearch.rest.action.search;
 import org.elasticsearch.action.search.SearchScrollRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -40,8 +38,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSearchScrollAction extends BaseRestHandler {
-
-    @Inject
     public RestSearchScrollAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -61,25 +57,33 @@ public class RestSearchScrollAction extends BaseRestHandler {
             searchScrollRequest.scroll(new Scroll(parseTimeValue(scroll, null, "scroll")));
         }
 
-        BytesReference body = request.contentOrSourceParam();
-        if (body.length() > 0) {
-            if (XContentFactory.xContentType(body) == null) {
-                if (scrollId == null) {
-                    scrollId = body.utf8ToString();
-                    searchScrollRequest.scrollId(scrollId);
+        request.withContentOrSourceParamParserOrNull(xContentParser -> {
+            if (xContentParser == null) {
+                if (request.hasContent()) {
+                    // TODO: why do we accept this plain text value? maybe we can just use the scroll params?
+                    BytesReference body = request.getContentOrSourceParamOnly();
+                    if (scrollId == null) {
+                        String bodyScrollId = body.utf8ToString();
+                        searchScrollRequest.scrollId(bodyScrollId);
+                    }
                 }
             } else {
                 // NOTE: if rest request with xcontent body has request parameters, these parameters override xcontent values
-                try (XContentParser parser = request.contentOrSourceParamParser()) {
-                    buildFromContent(parser, searchScrollRequest);
+                try {
+                    buildFromContent(xContentParser, searchScrollRequest);
                 } catch (IOException e) {
                     throw new IllegalArgumentException("Failed to parse request body", e);
                 }
             }
-        }
+        });
         return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel));
     }
 
+    @Override
+    public boolean supportsPlainText() {
+        return true;
+    }
+
     public static void buildFromContent(XContentParser parser, SearchScrollRequest searchScrollRequest) throws IOException {
         if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
             throw new IllegalArgumentException("Malformed content, must start with an object");
diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java
index fac9227db8f..3514671ea57 100644
--- a/core/src/main/java/org/elasticsearch/script/Script.java
+++ b/core/src/main/java/org/elasticsearch/script/Script.java
@@ -19,23 +19,23 @@
 
 package org.elasticsearch.script;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -45,11 +45,57 @@ import java.util.Map;
 import java.util.Objects;
 
 /**
- * Script represents used-defined input that can be used to
+ * {@link Script} represents used-defined input that can be used to
  * compile and execute a script from the {@link ScriptService}
  * based on the {@link ScriptType}.
+ *
+ * There are three types of scripts specified by {@link ScriptType}.
+ *
+ * The following describes the expected parameters for each type of script:
+ *
+ * <ul>
+ * <li> {@link ScriptType#INLINE}
+ * <ul>
+ * <li> {@link Script#lang}     - specifies the language, defaults to {@link Script#DEFAULT_SCRIPT_LANG}
+ * <li> {@link Script#idOrCode} - specifies the code to be compiled, must not be {@code null}
+ * <li> {@link Script#options}  - specifies the compiler options for this script; must not be {@code null},
+ *                                use an empty {@link Map} to specify no options
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * <li> {@link ScriptType#STORED}
+ * <ul>
+ * <li> {@link Script#lang}     - the language will be specified when storing the script, so this should
+ *                                be {@code null}; however, this can be specified to look up a stored
+ *                                script as part of the deprecated API
+ * <li> {@link Script#idOrCode} - specifies the id of the stored script to be looked up, must not be {@code null}
+ * <li> {@link Script#options}  - compiler options will be specified when a stored script is stored,
+ *                                so they have no meaning here and must be {@code null}
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * <li> {@link ScriptType#FILE}
+ * <ul>
+ * <li> {@link Script#lang}     - specifies the language for look up, defaults to {@link Script#DEFAULT_SCRIPT_LANG}
+ * <li> {@link Script#idOrCode} - specifies the id of the file script to be looked up, must not be {@code null}
+ * <li> {@link Script#options}  - compiler options will be specified when a file script is loaded,
+ *                                so they have no meaning here and must be {@code null}
+ * <li> {@link Script#params}   - {@link Map} of user-defined parameters; must not be {@code null},
+ *                                use an empty {@link Map} to specify no params
+ * </ul>
+ * </ul>
  */
-public final class Script implements ToXContent, Writeable {
+public final class Script implements ToXContentObject, Writeable {
+
+    /**
+     * Standard logger necessary for allocation of the deprecation logger.
+     */
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(ScriptMetaData.class);
+
+    /**
+     * Deprecation logger necessary for namespace changes related to stored scripts.
+     */
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);
 
     /**
      * The name of the of the default scripting language.
@@ -199,19 +245,66 @@ public final class Script implements ToXContent, Writeable {
                         "or [" + ScriptType.FILE.getParseField().getPreferredName() + "] script");
             }
 
-            if (idOrCode == null) {
-                throw new IllegalArgumentException("must specify an id or code for a script");
+            if (type == ScriptType.INLINE) {
+                if (lang == null) {
+                    lang = defaultLang;
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <id> for an [" + ScriptType.INLINE.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.size() > 1 || options.size() == 1 && options.get(CONTENT_TYPE_OPTION) == null) {
+                    options.remove(CONTENT_TYPE_OPTION);
+
+                    throw new IllegalArgumentException("illegal compiler options [" + options + "] specified");
+                }
+            } else if (type == ScriptType.STORED) {
+                // Only issue this deprecation warning if we aren't using a template.  Templates during
+                // this deprecation phase must always specify the default template language or they would
+                // possibly pick up a script in a different language as defined by the user under the new
+                // namespace unintentionally.
+                if (lang != null && lang.equals(DEFAULT_TEMPLATE_LANG) == false) {
+                    DEPRECATION_LOGGER.deprecated("specifying the field [" + LANG_PARSE_FIELD.getPreferredName() + "] " +
+                        "for executing " + ScriptType.STORED + " scripts is deprecated; use only the field " +
+                        "[" + ScriptType.STORED.getParseField().getPreferredName() + "] to specify an <id>");
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <code> for an [" + ScriptType.STORED.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.isEmpty()) {
+                    options = null;
+                } else {
+                    throw new IllegalArgumentException("field [" + OPTIONS_PARSE_FIELD.getPreferredName() + "] " +
+                        "cannot be specified using a [" + ScriptType.STORED.getParseField().getPreferredName() + "] script");
+                }
+            } else if (type == ScriptType.FILE) {
+                if (lang == null) {
+                    lang = defaultLang;
+                }
+
+                if (idOrCode == null) {
+                    throw new IllegalArgumentException(
+                        "must specify <code> for an [" + ScriptType.FILE.getParseField().getPreferredName() + "] script");
+                }
+
+                if (options.isEmpty()) {
+                    options = null;
+                } else {
+                    throw new IllegalArgumentException("field [" + OPTIONS_PARSE_FIELD.getPreferredName() + "] " +
+                        "cannot be specified using a [" + ScriptType.FILE.getParseField().getPreferredName() + "] script");
+                }
             }
 
-            if (options.size() > 1 || options.size() == 1 && options.get(CONTENT_TYPE_OPTION) == null) {
-                throw new IllegalArgumentException("illegal compiler options [" + options + "] specified");
-            }
-
-            return new Script(type, this.lang == null ? defaultLang : this.lang, idOrCode, options, params);
+            return new Script(type, lang, idOrCode, options, params);
         }
     }
 
-    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("script", Builder::new);
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("script", Builder::new);
 
     static {
         // Defines the fields necessary to parse a Script as XContent using an ObjectParser.
@@ -224,19 +317,11 @@ public final class Script implements ToXContent, Writeable {
     }
 
     /**
-     * Convenience method to call {@link Script#parse(XContentParser, ParseFieldMatcher, String)}
+     * Convenience method to call {@link Script#parse(XContentParser, String)}
      * using the default scripting language.
      */
-    public static Script parse(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
-        return parse(parser, matcher, DEFAULT_SCRIPT_LANG);
-    }
-
-    /**
-     * Convenience method to call {@link Script#parse(XContentParser, ParseFieldMatcher, String)} using the
-     * {@link ParseFieldMatcher} and scripting language provided by the {@link QueryParseContext}.
-     */
-    public static Script parse(XContentParser parser, QueryParseContext context) throws IOException {
-        return parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+    public static Script parse(XContentParser parser) throws IOException {
+        return parse(parser, DEFAULT_SCRIPT_LANG);
     }
 
     /**
@@ -300,13 +385,13 @@ public final class Script implements ToXContent, Writeable {
      * }
      *
      * @param parser       The {@link XContentParser} to be used.
-     * @param matcher      The {@link ParseFieldMatcher} to be used.
      * @param defaultLang  The default language to use if no language is specified.  The default language isn't necessarily
      *                     the one defined by {@link Script#DEFAULT_SCRIPT_LANG} due to backwards compatiblity requirements
      *                     related to stored queries using previously default languauges.
+     *
      * @return             The parsed {@link Script}.
      */
-    public static Script parse(XContentParser parser, ParseFieldMatcher matcher, String defaultLang) throws IOException {
+    public static Script parse(XContentParser parser, String defaultLang) throws IOException {
         Objects.requireNonNull(defaultLang);
 
         Token token = parser.currentToken();
@@ -319,7 +404,7 @@ public final class Script implements ToXContent, Writeable {
             return new Script(ScriptType.INLINE, defaultLang, parser.text(), Collections.emptyMap());
         }
 
-        return PARSER.apply(parser, () -> matcher).build(defaultLang);
+        return PARSER.apply(parser, null).build(defaultLang);
     }
 
     private final ScriptType type;
@@ -339,34 +424,57 @@ public final class Script implements ToXContent, Writeable {
     /**
      * Constructor for a script that does not need to use compiler options.
      * @param type     The {@link ScriptType}.
-     * @param lang     The lang for this {@link Script}.
+     * @param lang     The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *                 {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *                 be specified to access scripts stored as part of the stored scripts deprecated API.
      * @param idOrCode The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
      *                 The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
      * @param params   The user-defined params to be bound for script execution.
      */
     public Script(ScriptType type, String lang, String idOrCode, Map<String, Object> params) {
-        this(type, lang, idOrCode, Collections.emptyMap(), params);
+        this(type, lang, idOrCode, type == ScriptType.INLINE ? Collections.emptyMap() : null, params);
     }
 
     /**
      * Constructor for a script that requires the use of compiler options.
      * @param type     The {@link ScriptType}.
-     * @param lang     The lang for this {@link Script}.
+     * @param lang     The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *                 {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *                 be specified to access scripts stored as part of the stored scripts deprecated API.
      * @param idOrCode The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
      *                 The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
-     * @param options  The options to be passed to the compiler for use at compile-time.
+     * @param options  The map of compiler options for this {@link Script} if the {@link ScriptType}
+     *                 is {@link ScriptType#INLINE}, {@code null} otherwise.
      * @param params   The user-defined params to be bound for script execution.
      */
     public Script(ScriptType type, String lang, String idOrCode, Map<String, String> options, Map<String, Object> params) {
-        this.idOrCode = Objects.requireNonNull(idOrCode);
         this.type = Objects.requireNonNull(type);
-        this.lang = Objects.requireNonNull(lang);
-        this.options = Collections.unmodifiableMap(Objects.requireNonNull(options));
+        this.idOrCode = Objects.requireNonNull(idOrCode);
         this.params = Collections.unmodifiableMap(Objects.requireNonNull(params));
 
-        if (type != ScriptType.INLINE && !options.isEmpty()) {
-            throw new IllegalArgumentException(
-                "Compiler options [" + options + "] cannot be specified at runtime for [" + type + "] scripts.");
+        if (type == ScriptType.INLINE) {
+            this.lang = Objects.requireNonNull(lang);
+            this.options = Collections.unmodifiableMap(Objects.requireNonNull(options));
+        } else if (type == ScriptType.STORED) {
+            this.lang = lang;
+
+            if (options != null) {
+                throw new IllegalStateException(
+                    "options must be null for [" + ScriptType.STORED.getParseField().getPreferredName() + "] scripts");
+            }
+
+            this.options = null;
+        } else if (type == ScriptType.FILE) {
+            this.lang = Objects.requireNonNull(lang);
+
+            if (options != null) {
+                throw new IllegalStateException(
+                    "options must be null for [" + ScriptType.FILE.getParseField().getPreferredName() + "] scripts");
+            }
+
+            this.options = null;
+        } else {
+            throw new IllegalStateException("unknown script type [" + type.getName() + "]");
         }
     }
 
@@ -374,82 +482,125 @@ public final class Script implements ToXContent, Writeable {
      * Creates a {@link Script} read from an input stream.
      */
     public Script(StreamInput in) throws IOException {
-        // Version 5.1+ requires all Script members to be non-null and supports the potential
-        // for more options than just XContentType.  Reorders the read in contents to be in
-        // same order as the constructor.
-        if (in.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+        // Version 5.3 allows lang to be an optional parameter for stored scripts and expects
+        // options to be null for stored and file scripts.
+        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
             this.type = ScriptType.readFrom(in);
-            this.lang = in.readString();
+            this.lang = in.readOptionalString();
             this.idOrCode = in.readString();
             @SuppressWarnings("unchecked")
             Map<String, String> options = (Map)in.readMap();
             this.options = options;
             this.params = in.readMap();
-            // Prior to version 5.1 the script members are read in certain cases as optional and given
-            // default values when necessary.  Also the only option supported is for XContentType.
+        // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
+        // for more options than just XContentType.  Reorders the read in contents to be in
+        // same order as the constructor.
+        } else if (in.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+            this.type = ScriptType.readFrom(in);
+            this.lang = in.readString();
+
+            this.idOrCode = in.readString();
+            @SuppressWarnings("unchecked")
+            Map<String, String> options = (Map)in.readMap();
+
+            if (this.type != ScriptType.INLINE && options.isEmpty()) {
+                this.options = null;
+            } else {
+                this.options = options;
+            }
+
+            this.params = in.readMap();
+        // Prior to version 5.1 the script members are read in certain cases as optional and given
+        // default values when necessary.  Also the only option supported is for XContentType.
         } else {
-            String idOrCode = in.readString();
-            ScriptType type;
+            this.idOrCode = in.readString();
 
             if (in.readBoolean()) {
-                type = ScriptType.readFrom(in);
+                this.type = ScriptType.readFrom(in);
             } else {
-                type = DEFAULT_SCRIPT_TYPE;
+                this.type = DEFAULT_SCRIPT_TYPE;
             }
 
             String lang = in.readOptionalString();
 
             if (lang == null) {
-                lang = DEFAULT_SCRIPT_LANG;
+                this.lang = DEFAULT_SCRIPT_LANG;
+            } else {
+                this.lang = lang;
             }
 
             Map<String, Object> params = in.readMap();
 
             if (params == null) {
-                params = new HashMap<>();
+                this.params = new HashMap<>();
+            } else {
+                this.params = params;
             }
 
-            Map<String, String> options = new HashMap<>();
-
             if (in.readBoolean()) {
+                this.options = new HashMap<>();
                 XContentType contentType = XContentType.readFrom(in);
-                options.put(CONTENT_TYPE_OPTION, contentType.mediaType());
+                this.options.put(CONTENT_TYPE_OPTION, contentType.mediaType());
+            } else if (type == ScriptType.INLINE) {
+                options = new HashMap<>();
+            } else {
+                this.options = null;
             }
-
-            this.type = type;
-            this.lang = lang;
-            this.idOrCode = idOrCode;
-            this.options = options;
-            this.params = params;
         }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        // Version 5.1+ requires all Script members to be non-null and supports the potential
-        // for more options than just XContentType.  Reorders the written out contents to be in
-        // same order as the constructor.
-        if (out.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+        // Version 5.3+ allows lang to be an optional parameter for stored scripts and expects
+        // options to be null for stored and file scripts.
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
             type.writeTo(out);
-            out.writeString(lang);
+            out.writeOptionalString(lang);
             out.writeString(idOrCode);
             @SuppressWarnings("unchecked")
             Map<String, Object> options = (Map)this.options;
             out.writeMap(options);
             out.writeMap(params);
-            // Prior to version 5.1 the Script members were possibly written as optional or null, though this is no longer
-            // necessary since Script members cannot be null anymore, and there is no case where a null value wasn't equivalent
-            // to it's default value when actually compiling/executing a script.  Meaning, there are no backwards compatibility issues,
-            // and now there's enforced consistency.  Also the only supported compiler option was XContentType.
+        // Version 5.1 to 5.3 (exclusive) requires all Script members to be non-null and supports the potential
+        // for more options than just XContentType.  Reorders the written out contents to be in
+        // same order as the constructor.
+        } else if (out.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+            type.writeTo(out);
+
+            if (lang == null) {
+                out.writeString("");
+            } else {
+                out.writeString(lang);
+            }
+
+            out.writeString(idOrCode);
+            @SuppressWarnings("unchecked")
+            Map<String, Object> options = (Map)this.options;
+
+            if (options == null) {
+                out.writeMap(new HashMap<>());
+            } else {
+                out.writeMap(options);
+            }
+
+            out.writeMap(params);
+        // Prior to version 5.1 the Script members were possibly written as optional or null, though there is no case where a null
+        // value wasn't equivalent to it's default value when actually compiling/executing a script.  Meaning, there are no
+        // backwards compatibility issues, and now there's enforced consistency.  Also the only supported compiler
+        // option was XContentType.
         } else {
             out.writeString(idOrCode);
             out.writeBoolean(true);
             type.writeTo(out);
-            out.writeBoolean(true);
-            out.writeString(lang);
-            out.writeMap(params.isEmpty() ? null : params);
+            out.writeOptionalString(lang);
 
-            if (options.containsKey(CONTENT_TYPE_OPTION)) {
+            if (params.isEmpty()) {
+                out.writeMap(null);
+            } else {
+                out.writeMap(params);
+            }
+
+            if (options != null && options.containsKey(CONTENT_TYPE_OPTION)) {
                 XContentType contentType = XContentType.fromMediaTypeOrFormat(options.get(CONTENT_TYPE_OPTION));
                 out.writeBoolean(true);
                 contentType.writeTo(out);
@@ -490,7 +641,7 @@ public final class Script implements ToXContent, Writeable {
      * }
      * }
      *
-     * Note that options and params will only be included if there have been any specified.
+     * Note that lang, options, and params will only be included if there have been any specified.
      *
      * This also handles templates in a special way.  If the {@link Script#CONTENT_TYPE_OPTION} option
      * is provided and the {@link ScriptType#INLINE} is specified then the template will be preserved as a raw field.
@@ -516,7 +667,7 @@ public final class Script implements ToXContent, Writeable {
     public XContentBuilder toXContent(XContentBuilder builder, Params builderParams) throws IOException {
         builder.startObject();
 
-        String contentType = options.get(CONTENT_TYPE_OPTION);
+        String contentType = options == null ? null : options.get(CONTENT_TYPE_OPTION);
 
         if (type == ScriptType.INLINE && contentType != null && builder.contentType().mediaType().equals(contentType)) {
             builder.rawField(type.getParseField().getPreferredName(), new BytesArray(idOrCode));
@@ -524,9 +675,11 @@ public final class Script implements ToXContent, Writeable {
             builder.field(type.getParseField().getPreferredName(), idOrCode);
         }
 
-        builder.field(LANG_PARSE_FIELD.getPreferredName(), lang);
+        if (lang != null) {
+            builder.field(LANG_PARSE_FIELD.getPreferredName(), lang);
+        }
 
-        if (!options.isEmpty()) {
+        if (options != null && !options.isEmpty()) {
             builder.field(OPTIONS_PARSE_FIELD.getPreferredName(), options);
         }
 
@@ -539,6 +692,22 @@ public final class Script implements ToXContent, Writeable {
         return builder;
     }
 
+    /**
+     * @return The {@link ScriptType} for this {@link Script}.
+     */
+    public ScriptType getType() {
+        return type;
+    }
+
+    /**
+     * @return The language for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE} or
+     *         {@link ScriptType#FILE}.  For {@link ScriptType#STORED} scripts this should be null, but can
+     *         be specified to access scripts stored as part of the stored scripts deprecated API.
+     */
+    public String getLang() {
+        return lang;
+    }
+
     /**
      * @return The id for this {@link Script} if the {@link ScriptType} is {@link ScriptType#FILE} or {@link ScriptType#STORED}.
      *         The code for this {@link Script} if the {@link ScriptType} is {@link ScriptType#INLINE}.
@@ -548,21 +717,8 @@ public final class Script implements ToXContent, Writeable {
     }
 
     /**
-     * @return The {@link ScriptType} for this {@link Script}.
-     */
-    public ScriptType getType() {
-        return type;
-    }
-
-    /**
-     * @return The language for this {@link Script}.
-     */
-    public String getLang() {
-        return lang;
-    }
-
-    /**
-     * @return The map of compiler options for this {@link Script}.
+     * @return The map of compiler options for this {@link Script} if the {@link ScriptType}
+     *         is {@link ScriptType#INLINE}, {@code null} otherwise.
      */
     public Map<String, String> getOptions() {
         return options;
@@ -583,9 +739,9 @@ public final class Script implements ToXContent, Writeable {
         Script script = (Script)o;
 
         if (type != script.type) return false;
-        if (!lang.equals(script.lang)) return false;
+        if (lang != null ? !lang.equals(script.lang) : script.lang != null) return false;
         if (!idOrCode.equals(script.idOrCode)) return false;
-        if (!options.equals(script.options)) return false;
+        if (options != null ? !options.equals(script.options) : script.options != null) return false;
         return params.equals(script.params);
 
     }
@@ -593,9 +749,9 @@ public final class Script implements ToXContent, Writeable {
     @Override
     public int hashCode() {
         int result = type.hashCode();
-        result = 31 * result + lang.hashCode();
+        result = 31 * result + (lang != null ? lang.hashCode() : 0);
         result = 31 * result + idOrCode.hashCode();
-        result = 31 * result + options.hashCode();
+        result = 31 * result + (options != null ? options.hashCode() : 0);
         result = 31 * result + params.hashCode();
         return result;
     }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptException.java b/core/src/main/java/org/elasticsearch/script/ScriptException.java
index 475091f9f6d..91e6ad401fc 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptException.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptException.java
@@ -87,8 +87,7 @@ public class ScriptException extends ElasticsearchException {
     }
     
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
-        super.innerToXContent(builder, params);
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("script_stack", scriptStack);
         builder.field("script", script);
         builder.field("lang", lang);
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
index 84855da2f94..87afc21a8c0 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
@@ -18,22 +18,24 @@
  */
 package org.elasticsearch.script;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.DiffableUtils;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -41,210 +43,128 @@ import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 
-public final class ScriptMetaData implements MetaData.Custom {
-
-    public static final String TYPE = "stored_scripts";
-    public static final ScriptMetaData PROTO = new ScriptMetaData(Collections.emptyMap());
-
-    private final Map<String, ScriptAsBytes> scripts;
-
-    ScriptMetaData(Map<String, ScriptAsBytes> scripts) {
-        this.scripts = scripts;
-    }
-
-    public BytesReference getScriptAsBytes(String language, String id) {
-        ScriptAsBytes scriptAsBytes = scripts.get(toKey(language, id));
-        if (scriptAsBytes != null) {
-            return scriptAsBytes.script;
-        } else {
-            return null;
-        }
-    }
-
-    public String getScript(String language, String id) {
-        BytesReference scriptAsBytes = getScriptAsBytes(language, id);
-        if (scriptAsBytes == null) {
-            return null;
-        }
-        return scriptAsBytes.utf8ToString();
-    }
-
-    public static String parseStoredScript(BytesReference scriptAsBytes) {
-        // Scripts can be stored via API in several ways:
-        // 1) wrapped into a 'script' json object or field
-        // 2) wrapped into a 'template' json object or field
-        // 3) just as is
-        // In order to fetch the actual script in consistent manner this parsing logic is needed:
-        try (XContentParser parser = XContentHelper.createParser(scriptAsBytes);
-             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
-            parser.nextToken();
-            parser.nextToken();
-            if (parser.currentToken() == Token.END_OBJECT) {
-                throw new IllegalArgumentException("Empty script");
-            }
-            switch (parser.currentName()) {
-                case "script":
-                case "template":
-                    if (parser.nextToken() == Token.VALUE_STRING) {
-                        return parser.text();
-                    } else {
-                        builder.copyCurrentStructure(parser);
-                    }
-                    break;
-                default:
-                    // There is no enclosing 'script' or 'template' object so we just need to return the script as is...
-                    // because the parsers current location is already beyond the beginning we need to add a START_OBJECT:
-                    builder.startObject();
-                    builder.copyCurrentStructure(parser);
-                    builder.endObject();
-                    break;
-            }
-            return builder.string();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public String type() {
-        return TYPE;
-    }
-
-    @Override
-    public ScriptMetaData fromXContent(XContentParser parser) throws IOException {
-        Map<String, ScriptAsBytes> scripts = new HashMap<>();
-        String key = null;
-        for (Token token = parser.nextToken(); token != Token.END_OBJECT; token = parser.nextToken()) {
-            switch (token) {
-                case FIELD_NAME:
-                    key = parser.currentName();
-                    break;
-                case VALUE_STRING:
-                    scripts.put(key, new ScriptAsBytes(new BytesArray(parser.text())));
-                    break;
-                default:
-                    throw new ParsingException(parser.getTokenLocation(), "Unexpected token [" + token + "]");
-            }
-        }
-        return new ScriptMetaData(scripts);
-    }
-
-    @Override
-    public EnumSet<MetaData.XContentContext> context() {
-        return MetaData.ALL_CONTEXTS;
-    }
-
-    @Override
-    public ScriptMetaData readFrom(StreamInput in) throws IOException {
-        int size = in.readVInt();
-        Map<String, ScriptAsBytes> scripts = new HashMap<>();
-        for (int i = 0; i < size; i++) {
-            String languageAndId = in.readString();
-            BytesReference script = in.readBytesReference();
-            scripts.put(languageAndId, new ScriptAsBytes(script));
-        }
-        return new ScriptMetaData(scripts);
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        for (Map.Entry<String, ScriptAsBytes> entry : scripts.entrySet()) {
-            builder.field(entry.getKey(), entry.getValue().script.utf8ToString());
-        }
-        return builder;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(scripts.size());
-        for (Map.Entry<String, ScriptAsBytes> entry : scripts.entrySet()) {
-            out.writeString(entry.getKey());
-            entry.getValue().writeTo(out);
-        }
-    }
-
-    @Override
-    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
-        return new ScriptMetadataDiff((ScriptMetaData) before, this);
-    }
-
-    @Override
-    public Diff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
-        return new ScriptMetadataDiff(in);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        ScriptMetaData other = (ScriptMetaData) o;
-        return scripts.equals(other.scripts);
-    }
-
-    @Override
-    public int hashCode() {
-        return scripts.hashCode();
-    }
-
-    @Override
-    public String toString() {
-        return "ScriptMetaData{" +
-            "scripts=" + scripts +
-            '}';
-    }
-
-    static String toKey(String language, String id) {
-        if (id.contains("#")) {
-            throw new IllegalArgumentException("stored script id can't contain: '#'");
-        }
-        if (language.contains("#")) {
-            throw new IllegalArgumentException("stored script language can't contain: '#'");
-        }
-
-        return language + "#" + id;
-    }
+/**
+ * {@link ScriptMetaData} is used to store user-defined scripts
+ * as part of the {@link ClusterState}.  Currently scripts can
+ * be stored as part of the new namespace for a stored script where
+ * only an id is used or as part of the deprecated namespace where
+ * both a language and an id are used.
+ */
+public final class ScriptMetaData implements MetaData.Custom, Writeable, ToXContent {
 
+    /**
+     * A builder used to modify the currently stored scripts data held within
+     * the {@link ClusterState}.  Scripts can be added or deleted, then built
+     * to generate a new {@link Map} of scripts that will be used to update
+     * the current {@link ClusterState}.
+     */
     public static final class Builder {
 
-        private Map<String, ScriptAsBytes> scripts;
+        private final Map<String, StoredScriptSource> scripts;
 
+        /**
+         * @param previous The current {@link ScriptMetaData} or {@code null} if there
+         *                 is no existing {@link ScriptMetaData}.
+         */
         public Builder(ScriptMetaData previous) {
-            if (previous != null) {
-                this.scripts = new HashMap<>(previous.scripts);
-            } else {
-                this.scripts = new HashMap<>();
-            }
+            this.scripts = previous == null ? new HashMap<>() :new HashMap<>(previous.scripts);
         }
 
-        public Builder storeScript(String lang, String id, BytesReference script) {
-            BytesReference scriptBytest = new BytesArray(parseStoredScript(script));
-            scripts.put(toKey(lang, id), new ScriptAsBytes(scriptBytest));
+        /**
+         * Add a new script to the existing stored scripts.  The script will be added under
+         * both the new namespace and the deprecated namespace, so that look ups under
+         * the deprecated namespace will continue to work.  Should a script already exist under
+         * the new namespace using a different language, it will be replaced and a deprecation
+         * warning will be issued.  The replaced script will still exist under the deprecated
+         * namespace and can continue to be looked up this way until it is deleted.
+         * <p>
+         * Take for example script 'A' with lang 'L0' and data 'D0'.  If we add script 'A' to the
+         * empty set, the scripts {@link Map} will be ["A" -- D0, "A#L0" -- D0].  If a script
+         * 'A' with lang 'L1' and data 'D1' is then added, the scripts {@link Map} will be
+         * ["A" -- D1, "A#L1" -- D1, "A#L0" -- D0].
+         * @param id The user-specified id to use for the look up.
+         * @param source The user-specified stored script data held in {@link StoredScriptSource}.
+         */
+        public Builder storeScript(String id, StoredScriptSource source) {
+            StoredScriptSource previous = scripts.put(id, source);
+            scripts.put(source.getLang() + "#" + id, source);
+
+            if (previous != null && previous.getLang().equals(source.getLang()) == false) {
+                DEPRECATION_LOGGER.deprecated("stored script [" + id + "] already exists using a different lang " +
+                    "[" + previous.getLang() + "], the new namespace for stored scripts will only use (id) instead of (lang, id)");
+            }
+
             return this;
         }
 
-        public Builder deleteScript(String lang, String id) {
-            if (scripts.remove(toKey(lang, id)) == null) {
-                throw new ResourceNotFoundException("Stored script with id [{}] for language [{}] does not exist", id, lang);
+        /**
+         * Delete a script from the existing stored scripts.  The script will be removed from the
+         * new namespace if the script language matches the current script under the same id or
+         * if the script language is {@code null}.  The script will be removed from the deprecated
+         * namespace on any delete either using using the specified lang parameter or the language
+         * found from looking up the script in the new namespace.
+         * <p>
+         * Take for example a scripts {@link Map} with {"A" -- D1, "A#L1" -- D1, "A#L0" -- D0}.
+         * If a script is removed specified by an id 'A' and lang {@code null} then the scripts
+         * {@link Map} will be {"A#L0" -- D0}.  To remove the final script, the deprecated
+         * namespace must be used, so an id 'A' and lang 'L0' would need to be specified.
+         * @param id The user-specified id to use for the look up.
+         * @param lang The user-specified language to use for the look up if using the deprecated
+         *             namespace, otherwise {@code null}.
+         */
+        public Builder deleteScript(String id, String lang) {
+            StoredScriptSource source = scripts.get(id);
+
+            if (lang == null) {
+                if (source == null) {
+                    throw new ResourceNotFoundException("stored script [" + id + "] does not exist and cannot be deleted");
+                }
+
+                lang = source.getLang();
             }
+
+            if (source != null) {
+                if (lang.equals(source.getLang())) {
+                    scripts.remove(id);
+                }
+            }
+
+            source = scripts.get(lang + "#" + id);
+
+            if (source == null) {
+                throw new ResourceNotFoundException(
+                    "stored script [" + id + "] using lang [" + lang + "] does not exist and cannot be deleted");
+            }
+
+            scripts.remove(lang + "#" + id);
+
             return this;
         }
 
+        /**
+         * @return A {@link ScriptMetaData} with the updated {@link Map} of scripts.
+         */
         public ScriptMetaData build() {
-            return new ScriptMetaData(Collections.unmodifiableMap(scripts));
+            return new ScriptMetaData(scripts);
         }
     }
 
-    static final class ScriptMetadataDiff implements Diff<MetaData.Custom> {
+    static final class ScriptMetadataDiff implements NamedDiff<MetaData.Custom> {
 
-        final Diff<Map<String, ScriptAsBytes>> pipelines;
+        final Diff<Map<String, StoredScriptSource>> pipelines;
 
         ScriptMetadataDiff(ScriptMetaData before, ScriptMetaData after) {
             this.pipelines = DiffableUtils.diff(before.scripts, after.scripts, DiffableUtils.getStringKeySerializer());
         }
 
-        public ScriptMetadataDiff(StreamInput in) throws IOException {
-            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), new ScriptAsBytes(null));
+        ScriptMetadataDiff(StreamInput in) throws IOException {
+            pipelines = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(),
+                StoredScriptSource::new, StoredScriptSource::readDiffFrom);
+        }
+
+        @Override
+        public String getWriteableName() {
+            return TYPE;
         }
 
         @Override
@@ -258,38 +178,281 @@ public final class ScriptMetaData implements MetaData.Custom {
         }
     }
 
-    static final class ScriptAsBytes extends AbstractDiffable<ScriptAsBytes> {
+    /**
+     * Convenience method to build and return a new
+     * {@link ScriptMetaData} adding the specified stored script.
+     */
+    static ScriptMetaData putStoredScript(ScriptMetaData previous, String id, StoredScriptSource source) {
+        Builder builder = new Builder(previous);
+        builder.storeScript(id, source);
 
-        public ScriptAsBytes(BytesReference script) {
-            this.script = script;
+        return builder.build();
+    }
+
+    /**
+     * Convenience method to build and return a new
+     * {@link ScriptMetaData} deleting the specified stored script.
+     */
+    static ScriptMetaData deleteStoredScript(ScriptMetaData previous, String id, String lang) {
+        Builder builder = new ScriptMetaData.Builder(previous);
+        builder.deleteScript(id, lang);
+
+        return builder.build();
+    }
+
+    /**
+     * Standard logger necessary for allocation of the deprecation logger.
+     */
+    private static final Logger LOGGER = ESLoggerFactory.getLogger(ScriptMetaData.class);
+
+    /**
+     * Deprecation logger necessary for namespace changes related to stored scripts.
+     */
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LOGGER);
+
+    /**
+     * The type of {@link ClusterState} data.
+     */
+    public static final String TYPE = "stored_scripts";
+
+    /**
+     * This will parse XContent into {@link ScriptMetaData}.
+     *
+     * The following format will be parsed for the new namespace:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
+     *     "<id>" : "<{@link StoredScriptSource#fromXContent(XContentParser)}>",
+     *     ...
+     * }
+     * }
+     *
+     * The following format will be parsed for the deprecated namespace:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<code>",
+     *     "<id>" : "<code>",
+     *     ...
+     * }
+     * }
+     *
+     * Note when using the deprecated namespace, the language will be pulled from
+     * the id and options will be set to an empty {@link Map}.
+     */
+    public static ScriptMetaData fromXContent(XContentParser parser) throws IOException {
+        Map<String, StoredScriptSource> scripts = new HashMap<>();
+        String id = null;
+        StoredScriptSource source;
+
+        Token token = parser.currentToken();
+
+        if (token == null) {
+            token = parser.nextToken();
         }
 
-        private final BytesReference script;
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeBytesReference(script);
+        if (token != Token.START_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{]");
         }
 
-        @Override
-        public ScriptAsBytes readFrom(StreamInput in) throws IOException {
-            return new ScriptAsBytes(in.readBytesReference());
+        token = parser.nextToken();
+
+        while (token != Token.END_OBJECT) {
+            switch (token) {
+                case FIELD_NAME:
+                    id = parser.currentName();
+                    break;
+                case VALUE_STRING:
+                    if (id == null) {
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "unexpected token [" + token + "], expected [<id>, <code>, {]");
+                    }
+
+                    int split = id.indexOf('#');
+
+                    if (split == -1) {
+                        throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
+                    } else {
+                        source = new StoredScriptSource(id.substring(0, split), parser.text(), Collections.emptyMap());
+                    }
+                    scripts.put(id, source);
+
+                    id = null;
+
+                    break;
+                case START_OBJECT:
+                    if (id == null) {
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "unexpected token [" + token + "], expected [<id>, <code>, {]");
+                    }
+
+                    source = StoredScriptSource.fromXContent(parser);
+                    scripts.put(id, source);
+
+                    break;
+                default:
+                    throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [<id>, <code>, {]");
+            }
+
+            token = parser.nextToken();
         }
 
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+        return new ScriptMetaData(scripts);
+    }
 
-            ScriptAsBytes that = (ScriptAsBytes) o;
+    public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+        return new ScriptMetadataDiff(in);
+    }
 
-            return script.equals(that.script);
+    private final Map<String, StoredScriptSource> scripts;
 
+    /**
+     * Standard constructor to create metadata to store scripts.
+     * @param scripts The currently stored scripts.  Must not be {@code null},
+     *                use and empty {@link Map} to specify there were no
+     *                previously stored scripts.
+     */
+    ScriptMetaData(Map<String, StoredScriptSource> scripts) {
+        this.scripts = Collections.unmodifiableMap(scripts);
+    }
+
+    public ScriptMetaData(StreamInput in) throws IOException {
+        Map<String, StoredScriptSource> scripts = new HashMap<>();
+        StoredScriptSource source;
+        int size = in.readVInt();
+
+        for (int i = 0; i < size; i++) {
+            String id = in.readString();
+
+            // Prior to version 5.3 all scripts were stored using the deprecated namespace.
+            // Split the id to find the language then use StoredScriptSource to parse the
+            // expected BytesReference after which a new StoredScriptSource is created
+            // with the appropriate language and options.
+            if (in.getVersion().before(Version.V_5_3_0_UNRELEASED)) {
+                int split = id.indexOf('#');
+
+                if (split == -1) {
+                    throw new IllegalArgumentException("illegal stored script id [" + id + "], does not contain lang");
+                } else {
+                    source = new StoredScriptSource(in);
+                    source = new StoredScriptSource(id.substring(0, split), source.getCode(), Collections.emptyMap());
+                }
+            // Version 5.3+ can just be parsed normally using StoredScriptSource.
+            } else {
+                source = new StoredScriptSource(in);
+            }
+
+            scripts.put(id, source);
         }
 
-        @Override
-        public int hashCode() {
-            return script.hashCode();
+        this.scripts = Collections.unmodifiableMap(scripts);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // Version 5.3+ will output the contents of the scripts' Map using
+        // StoredScriptSource to stored the language, code, and options.
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            out.writeVInt(scripts.size());
+
+            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
+        // Prior to Version 5.3, stored scripts can only be read using the deprecated
+        // namespace.  Scripts using the deprecated namespace are first isolated in a
+        // temporary Map, then written out.  Since all scripts will be stored using the
+        // deprecated namespace, no scripts will be lost.
+        } else {
+            Map<String, StoredScriptSource> filtered = new HashMap<>();
+
+            for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+                if (entry.getKey().contains("#")) {
+                    filtered.put(entry.getKey(), entry.getValue());
+                }
+            }
+
+            out.writeVInt(filtered.size());
+
+            for (Map.Entry<String, StoredScriptSource> entry : filtered.entrySet()) {
+                out.writeString(entry.getKey());
+                entry.getValue().writeTo(out);
+            }
         }
     }
+
+
+
+    /**
+     * This will write XContent from {@link ScriptMetaData}.  The following format will be written:
+     *
+     * {@code
+     * {
+     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
+     *     "<id>" : "<{@link StoredScriptSource#toXContent(XContentBuilder, Params)}>",
+     *     ...
+     * }
+     * }
+     */
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        for (Map.Entry<String, StoredScriptSource> entry : scripts.entrySet()) {
+            builder.field(entry.getKey());
+            entry.getValue().toXContent(builder, params);
+        }
+
+        return builder;
+    }
+
+    @Override
+    public Diff<MetaData.Custom> diff(MetaData.Custom before) {
+        return new ScriptMetadataDiff((ScriptMetaData)before, this);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public EnumSet<MetaData.XContentContext> context() {
+        return MetaData.ALL_CONTEXTS;
+    }
+
+    /**
+     * Retrieves a stored script from the new namespace if lang is {@code null}.
+     * Otherwise, returns a stored script from the deprecated namespace.  Either
+     * way an id is required.
+     */
+    StoredScriptSource getStoredScript(String id, String lang) {
+        if (lang == null) {
+            return scripts.get(id);
+        } else {
+            return scripts.get(lang + "#" + id);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ScriptMetaData that = (ScriptMetaData)o;
+
+        return scripts.equals(that.scripts);
+
+    }
+
+    @Override
+    public int hashCode() {
+        return scripts.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "ScriptMetaData{" +
+            "scripts=" + scripts +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 478aac6a55c..692e081a7ba 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -38,7 +38,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.cache.RemovalListener;
@@ -46,7 +45,6 @@ import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.Streams;
-import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -209,18 +207,44 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
     /**
      * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script.
      */
-    public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
-        if (script == null) {
-            throw new IllegalArgumentException("The parameter script (Script) must not be null.");
-        }
-        if (scriptContext == null) {
-            throw new IllegalArgumentException("The parameter scriptContext (ScriptContext) must not be null.");
-        }
+    public CompiledScript compile(Script script, ScriptContext scriptContext) {
+        Objects.requireNonNull(script);
+        Objects.requireNonNull(scriptContext);
 
+        ScriptType type = script.getType();
         String lang = script.getLang();
-        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
-        if (canExecuteScript(lang, script.getType(), scriptContext) == false) {
-            throw new IllegalStateException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
+        String idOrCode = script.getIdOrCode();
+        Map<String, String> options = script.getOptions();
+
+        String id = idOrCode;
+
+        // lang may be null when looking up a stored script, so we must get the
+        // source to retrieve the lang before checking if the context is supported
+        if (type == ScriptType.STORED) {
+            // search template requests can possibly pass in the entire path instead
+            // of just an id for looking up a stored script, so we parse the path and
+            // check for appropriate errors
+            String[] path = id.split("/");
+
+            if (path.length == 3) {
+                if (lang != null && lang.equals(path[1]) == false) {
+                    throw new IllegalStateException("conflicting script languages, found [" + path[1] + "] but expected [" + lang + "]");
+                }
+
+                id = path[2];
+
+                deprecationLogger.deprecated("use of </lang/id> [" + idOrCode + "] for looking up" +
+                    " stored scripts/templates has been deprecated, use only <id> [" + id + "] instead");
+            } else if (path.length != 1) {
+                throw new IllegalArgumentException("illegal stored script format [" + id + "] use only <id>");
+            }
+
+            // a stored script must be pulled from the cluster state every time in case
+            // the script has been updated since the last compilation
+            StoredScriptSource source = getScriptFromClusterState(id, lang);
+            lang = source.getLang();
+            idOrCode = source.getCode();
+            options = source.getOptions();
         }
 
         // TODO: fix this through some API or something, that's wrong
@@ -229,10 +253,71 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey());
         if (expression && notSupported) {
             throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," +
-                    " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
+                " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
         }
 
-        return compileInternal(script, params);
+        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
+
+        if (canExecuteScript(lang, type, scriptContext) == false) {
+            throw new IllegalStateException("scripts of type [" + script.getType() + "]," +
+                " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
+        }
+
+        if (logger.isTraceEnabled()) {
+            logger.trace("compiling lang: [{}] type: [{}] script: {}", lang, type, idOrCode);
+        }
+
+        if (type == ScriptType.FILE) {
+            CacheKey cacheKey = new CacheKey(lang, idOrCode, options);
+            CompiledScript compiledScript = staticCache.get(cacheKey);
+
+            if (compiledScript == null) {
+                throw new IllegalArgumentException("unable to find file script [" + idOrCode + "] using lang [" + lang + "]");
+            }
+
+            return compiledScript;
+        }
+
+        CacheKey cacheKey = new CacheKey(lang, idOrCode, options);
+        CompiledScript compiledScript = cache.get(cacheKey);
+
+        if (compiledScript != null) {
+            return compiledScript;
+        }
+
+        // Synchronize so we don't compile scripts many times during multiple shards all compiling a script
+        synchronized (this) {
+            // Retrieve it again in case it has been put by a different thread
+            compiledScript = cache.get(cacheKey);
+
+            if (compiledScript == null) {
+                try {
+                    // Either an un-cached inline script or indexed script
+                    // If the script type is inline the name will be the same as the code for identification in exceptions
+
+                    // but give the script engine the chance to be better, give it separate name + source code
+                    // for the inline case, then its anonymous: null.
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("compiling script, type: [{}], lang: [{}], options: [{}]", type, lang, options);
+                    }
+                    // Check whether too many compilations have happened
+                    checkCompilationLimit();
+                    compiledScript = new CompiledScript(type, id, lang, scriptEngineService.compile(id, idOrCode, options));
+                } catch (ScriptException good) {
+                    // TODO: remove this try-catch completely, when all script engines have good exceptions!
+                    throw good; // its already good
+                } catch (Exception exception) {
+                    throw new GeneralScriptException("Failed to compile " + type + " script [" + id + "] using lang [" + lang + "]", exception);
+                }
+
+                // Since the cache key is the script content itself we don't need to
+                // invalidate/check the cache if an indexed script changes.
+                scriptMetrics.onCompilation();
+                cache.put(cacheKey, compiledScript);
+            }
+
+            return compiledScript;
+        }
     }
 
     /**
@@ -267,152 +352,71 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         }
     }
 
-    /**
-     * Compiles a script straight-away, or returns the previously compiled and cached script,
-     * without checking if it can be executed based on settings.
-     */
-    CompiledScript compileInternal(Script script, Map<String, String> params) {
-        if (script == null) {
-            throw new IllegalArgumentException("The parameter script (Script) must not be null.");
-        }
+    public boolean isLangSupported(String lang) {
+        Objects.requireNonNull(lang);
 
-        String lang = script.getLang();
-        ScriptType type = script.getType();
-        //script.getIdOrCode() could return either a name or code for a script,
-        //but we check for a file script name first and an indexed script name second
-        String name = script.getIdOrCode();
-
-        if (logger.isTraceEnabled()) {
-            logger.trace("Compiling lang: [{}] type: [{}] script: {}", lang, type, name);
-        }
-
-        ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
-
-        if (type == ScriptType.FILE) {
-            CacheKey cacheKey = new CacheKey(scriptEngineService, name, null, params);
-            //On disk scripts will be loaded into the staticCache by the listener
-            CompiledScript compiledScript = staticCache.get(cacheKey);
-
-            if (compiledScript == null) {
-                throw new IllegalArgumentException("Unable to find on disk file script [" + name + "] using lang [" + lang + "]");
-            }
-
-            return compiledScript;
-        }
-
-        //script.getIdOrCode() will be code if the script type is inline
-        String code = script.getIdOrCode();
-
-        if (type == ScriptType.STORED) {
-            //The look up for an indexed script must be done every time in case
-            //the script has been updated in the index since the last look up.
-            final IndexedScript indexedScript = new IndexedScript(lang, name);
-            name = indexedScript.id;
-            code = getScriptFromClusterState(indexedScript.lang, indexedScript.id);
-        }
-
-        CacheKey cacheKey = new CacheKey(scriptEngineService, type == ScriptType.INLINE ? null : name, code, params);
-        CompiledScript compiledScript = cache.get(cacheKey);
-
-        if (compiledScript != null) {
-            return compiledScript;
-        }
-
-        // Synchronize so we don't compile scripts many times during multiple shards all compiling a script
-        synchronized (this) {
-            // Retrieve it again in case it has been put by a different thread
-            compiledScript = cache.get(cacheKey);
-
-            if (compiledScript == null) {
-                try {
-                    // Either an un-cached inline script or indexed script
-                    // If the script type is inline the name will be the same as the code for identification in exceptions
-
-                    // but give the script engine the chance to be better, give it separate name + source code
-                    // for the inline case, then its anonymous: null.
-                    String actualName = (type == ScriptType.INLINE) ? null : name;
-                    if (logger.isTraceEnabled()) {
-                        logger.trace("compiling script, type: [{}], lang: [{}], params: [{}]", type, lang, params);
-                    }
-                    // Check whether too many compilations have happened
-                    checkCompilationLimit();
-                    compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(actualName, code, params));
-                } catch (ScriptException good) {
-                    // TODO: remove this try-catch completely, when all script engines have good exceptions!
-                    throw good; // its already good
-                } catch (Exception exception) {
-                    throw new GeneralScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
-                }
-
-                // Since the cache key is the script content itself we don't need to
-                // invalidate/check the cache if an indexed script changes.
-                scriptMetrics.onCompilation();
-                cache.put(cacheKey, compiledScript);
-            }
-
-            return compiledScript;
-        }
+        return scriptEnginesByLang.containsKey(lang);
     }
 
-    private String validateScriptLanguage(String scriptLang) {
-        Objects.requireNonNull(scriptLang);
-        if (scriptEnginesByLang.containsKey(scriptLang) == false) {
-            throw new IllegalArgumentException("script_lang not supported [" + scriptLang + "]");
+    StoredScriptSource getScriptFromClusterState(String id, String lang) {
+        if (lang != null && isLangSupported(lang) == false) {
+            throw new IllegalArgumentException("unable to get stored script with unsupported lang [" + lang + "]");
         }
-        return scriptLang;
-    }
 
-    String getScriptFromClusterState(String scriptLang, String id) {
-        scriptLang = validateScriptLanguage(scriptLang);
         ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE);
+
         if (scriptMetadata == null) {
-            throw new ResourceNotFoundException("Unable to find script [" + scriptLang + "/" + id + "] in cluster state");
+            throw new ResourceNotFoundException("unable to find script [" + id + "]" +
+                (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state");
         }
 
-        String script = scriptMetadata.getScript(scriptLang, id);
-        if (script == null) {
-            throw new ResourceNotFoundException("Unable to find script [" + scriptLang + "/" + id + "] in cluster state");
+        StoredScriptSource source = scriptMetadata.getStoredScript(id, lang);
+
+        if (source == null) {
+            throw new ResourceNotFoundException("unable to find script [" + id + "]" +
+                (lang == null ? "" : " using lang [" + lang + "]") + " in cluster state");
         }
-        return script;
+
+        return source;
     }
 
-    void validateStoredScript(String id, String scriptLang, BytesReference scriptBytes) {
-        validateScriptSize(id, scriptBytes.length());
-        String script = ScriptMetaData.parseStoredScript(scriptBytes);
-        if (Strings.hasLength(scriptBytes)) {
-            //Just try and compile it
-            try {
-                ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(scriptLang);
-                //we don't know yet what the script will be used for, but if all of the operations for this lang with
-                //indexed scripts are disabled, it makes no sense to even compile it.
-                if (isAnyScriptContextEnabled(scriptLang, ScriptType.STORED)) {
-                    Object compiled = scriptEngineService.compile(id, script, Collections.emptyMap());
-                    if (compiled == null) {
-                        throw new IllegalArgumentException("Unable to parse [" + script + "] lang [" + scriptLang +
-                                "] (ScriptService.compile returned null)");
-                    }
-                } else {
-                    logger.warn(
-                            "skipping compile of script [{}], lang [{}] as all scripted operations are disabled for indexed scripts",
-                            script, scriptLang);
+    public void putStoredScript(ClusterService clusterService, PutStoredScriptRequest request,
+                                ActionListener<PutStoredScriptResponse> listener) {
+        int max = SCRIPT_MAX_SIZE_IN_BYTES.get(settings);
+
+        if (request.content().length() > max) {
+            throw new IllegalArgumentException("exceeded max allowed stored script size in bytes [" + max + "] with size [" +
+                request.content().length() + "] for script [" + request.id() + "]");
+        }
+
+        StoredScriptSource source = StoredScriptSource.parse(request.lang(), request.content(), request.xContentType());
+
+        if (isLangSupported(source.getLang()) == false) {
+            throw new IllegalArgumentException("unable to put stored script with unsupported lang [" + source.getLang() + "]");
+        }
+
+        try {
+            ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(source.getLang());
+
+            if (isAnyScriptContextEnabled(source.getLang(), ScriptType.STORED)) {
+                Object compiled = scriptEngineService.compile(request.id(), source.getCode(), Collections.emptyMap());
+
+                if (compiled == null) {
+                    throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]" +
+                        (source.getCode() == null ? "" : " using code [" + source.getCode() + "]"));
                 }
-            } catch (ScriptException good) {
-                // TODO: remove this when all script engines have good exceptions!
-                throw good; // its already good!
-            } catch (Exception e) {
-                throw new IllegalArgumentException("Unable to parse [" + script +
-                        "] lang [" + scriptLang + "]", e);
+            } else {
+                throw new IllegalArgumentException(
+                    "cannot put stored script [" + request.id() + "], stored scripts cannot be run under any context");
             }
-        } else {
-            throw new IllegalArgumentException("Unable to find script in : " + scriptBytes.utf8ToString());
+        } catch (ScriptException good) {
+            throw good;
+        } catch (Exception exception) {
+            throw new IllegalArgumentException("failed to parse/compile stored script [" + request.id() + "]", exception);
         }
-    }
 
-    public void storeScript(ClusterService clusterService, PutStoredScriptRequest request, ActionListener<PutStoredScriptResponse> listener) {
-        String scriptLang = validateScriptLanguage(request.scriptLang());
-        //verify that the script compiles
-        validateStoredScript(request.id(), scriptLang, request.script());
-        clusterService.submitStateUpdateTask("put-script-" + request.id(), new AckedClusterStateUpdateTask<PutStoredScriptResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("put-script-" + request.id(),
+            new AckedClusterStateUpdateTask<PutStoredScriptResponse>(request, listener) {
 
             @Override
             protected PutStoredScriptResponse newResponse(boolean acknowledged) {
@@ -421,23 +425,23 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
-                return innerStoreScript(currentState, scriptLang, request);
+                ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE);
+                smd = ScriptMetaData.putStoredScript(smd, request.id(), source);
+                MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+
+                return ClusterState.builder(currentState).metaData(mdb).build();
             }
         });
     }
 
-    static ClusterState innerStoreScript(ClusterState currentState, String validatedScriptLang, PutStoredScriptRequest request) {
-        ScriptMetaData scriptMetadata = currentState.metaData().custom(ScriptMetaData.TYPE);
-        ScriptMetaData.Builder scriptMetadataBuilder = new ScriptMetaData.Builder(scriptMetadata);
-        scriptMetadataBuilder.storeScript(validatedScriptLang, request.id(), request.script());
-        MetaData.Builder metaDataBuilder = MetaData.builder(currentState.getMetaData())
-                .putCustom(ScriptMetaData.TYPE, scriptMetadataBuilder.build());
-        return ClusterState.builder(currentState).metaData(metaDataBuilder).build();
-    }
+    public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request,
+                                   ActionListener<DeleteStoredScriptResponse> listener) {
+        if (request.lang() != null && isLangSupported(request.lang()) == false) {
+            throw new IllegalArgumentException("unable to delete stored script with unsupported lang [" + request.lang() +"]");
+        }
 
-    public void deleteStoredScript(ClusterService clusterService, DeleteStoredScriptRequest request, ActionListener<DeleteStoredScriptResponse> listener) {
-        String scriptLang = validateScriptLanguage(request.scriptLang());
-        clusterService.submitStateUpdateTask("delete-script-" + request.id(), new AckedClusterStateUpdateTask<DeleteStoredScriptResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("delete-script-" + request.id(),
+            new AckedClusterStateUpdateTask<DeleteStoredScriptResponse>(request, listener) {
 
             @Override
             protected DeleteStoredScriptResponse newResponse(boolean acknowledged) {
@@ -446,24 +450,20 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
-                return innerDeleteScript(currentState, scriptLang, request);
+                ScriptMetaData smd = currentState.metaData().custom(ScriptMetaData.TYPE);
+                smd = ScriptMetaData.deleteStoredScript(smd, request.id(), request.lang());
+                MetaData.Builder mdb = MetaData.builder(currentState.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+
+                return ClusterState.builder(currentState).metaData(mdb).build();
             }
         });
     }
 
-    static ClusterState innerDeleteScript(ClusterState currentState, String validatedLang, DeleteStoredScriptRequest request) {
-        ScriptMetaData scriptMetadata = currentState.metaData().custom(ScriptMetaData.TYPE);
-        ScriptMetaData.Builder scriptMetadataBuilder = new ScriptMetaData.Builder(scriptMetadata);
-        scriptMetadataBuilder.deleteScript(validatedLang, request.id());
-        MetaData.Builder metaDataBuilder = MetaData.builder(currentState.getMetaData())
-                .putCustom(ScriptMetaData.TYPE, scriptMetadataBuilder.build());
-        return ClusterState.builder(currentState).metaData(metaDataBuilder).build();
-    }
-
-    public String getStoredScript(ClusterState state, GetStoredScriptRequest request) {
+    public StoredScriptSource getStoredScript(ClusterState state, GetStoredScriptRequest request) {
         ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE);
+
         if (scriptMetadata != null) {
-            return scriptMetadata.getScript(request.lang(), request.id());
+            return scriptMetadata.getStoredScript(request.id(), request.lang());
         } else {
             return null;
         }
@@ -473,7 +473,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
      * Compiles (or retrieves from cache) and executes the provided script
      */
     public ExecutableScript executable(Script script, ScriptContext scriptContext) {
-        return executable(compile(script, scriptContext, script.getOptions()), script.getParams());
+        return executable(compile(script, scriptContext), script.getParams());
     }
 
     /**
@@ -487,7 +487,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
      * Compiles (or retrieves from cache) and executes the provided search script
      */
     public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext) {
-        CompiledScript compiledScript = compile(script, scriptContext, script.getOptions());
+        CompiledScript compiledScript = compile(script, scriptContext);
         return search(lookup, compiledScript, script.getParams());
     }
 
@@ -520,18 +520,6 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         return scriptMetrics.stats();
     }
 
-    private void validateScriptSize(String identifier, int scriptSizeInBytes) {
-        int allowedScriptSizeInBytes = SCRIPT_MAX_SIZE_IN_BYTES.get(settings);
-        if (scriptSizeInBytes > allowedScriptSizeInBytes) {
-            String message = LoggerMessageFormat.format(
-                    "Limit of script size in bytes [{}] has been exceeded for script [{}] with size [{}]",
-                    allowedScriptSizeInBytes,
-                    identifier,
-                    scriptSizeInBytes);
-            throw new IllegalArgumentException(message);
-        }
-    }
-
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         clusterState = event.state();
@@ -592,11 +580,11 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                         logger.info("compiling script file [{}]", file.toAbsolutePath());
                         try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
                             String script = Streams.copyToString(reader);
-                            String name = scriptNameExt.v1();
-                            CacheKey cacheKey = new CacheKey(engineService, name, null, Collections.emptyMap());
+                            String id = scriptNameExt.v1();
+                            CacheKey cacheKey = new CacheKey(engineService.getType(), id, null);
                             // pass the actual file name to the compiler (for script engines that care about this)
                             Object executable = engineService.compile(file.getFileName().toString(), script, Collections.emptyMap());
-                            CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, name, engineService.getType(), executable);
+                            CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, id, engineService.getType(), executable);
                             staticCache.put(cacheKey, compiledScript);
                             scriptMetrics.onCompilation();
                         }
@@ -607,7 +595,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                     try (XContentBuilder builder = JsonXContent.contentBuilder()) {
                         builder.prettyPrint();
                         builder.startObject();
-                        ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, e);
+                        ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e);
                         builder.endObject();
                         logger.warn("failed to load/compile script [{}]: {}", scriptNameExt.v1(), builder.string());
                     } catch (IOException ioe) {
@@ -637,7 +625,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                 ScriptEngineService engineService = getScriptEngineServiceForFileExt(scriptNameExt.v2());
                 assert engineService != null;
                 logger.info("removing script file [{}]", file.toAbsolutePath());
-                staticCache.remove(new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap()));
+                staticCache.remove(new CacheKey(engineService.getType(), scriptNameExt.v1(), null));
             }
         }
 
@@ -650,15 +638,13 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
 
     private static final class CacheKey {
         final String lang;
-        final String name;
-        final String code;
-        final Map<String, String> params;
+        final String idOrCode;
+        final Map<String, String> options;
 
-        private CacheKey(final ScriptEngineService service, final String name, final String code, final Map<String, String> params) {
-            this.lang = service.getType();
-            this.name = name;
-            this.code = code;
-            this.params = params;
+        private CacheKey(String lang, String idOrCode, Map<String, String> options) {
+            this.lang = lang;
+            this.idOrCode = idOrCode;
+            this.options = options;
         }
 
         @Override
@@ -668,44 +654,18 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
 
             CacheKey cacheKey = (CacheKey)o;
 
-            if (!lang.equals(cacheKey.lang)) return false;
-            if (name != null ? !name.equals(cacheKey.name) : cacheKey.name != null) return false;
-            if (code != null ? !code.equals(cacheKey.code) : cacheKey.code != null) return false;
-            return params.equals(cacheKey.params);
+            if (lang != null ? !lang.equals(cacheKey.lang) : cacheKey.lang != null) return false;
+            if (!idOrCode.equals(cacheKey.idOrCode)) return false;
+            return options != null ? options.equals(cacheKey.options) : cacheKey.options == null;
 
         }
 
         @Override
         public int hashCode() {
-            int result = lang.hashCode();
-            result = 31 * result + (name != null ? name.hashCode() : 0);
-            result = 31 * result + (code != null ? code.hashCode() : 0);
-            result = 31 * result + params.hashCode();
+            int result = lang != null ? lang.hashCode() : 0;
+            result = 31 * result + idOrCode.hashCode();
+            result = 31 * result + (options != null ? options.hashCode() : 0);
             return result;
         }
     }
-
-
-    private static class IndexedScript {
-        private final String lang;
-        private final String id;
-
-        IndexedScript(String lang, String script) {
-            this.lang = lang;
-            final String[] parts = script.split("/");
-            if (parts.length == 1) {
-                this.id = script;
-            } else {
-                if (parts.length != 3) {
-                    throw new IllegalArgumentException("Illegal index script format [" + script + "]" +
-                            " should be /lang/id");
-                } else {
-                    if (!parts[1].equals(this.lang)) {
-                        throw new IllegalStateException("Conflicting script language, found [" + parts[1] + "] expected + ["+ this.lang + "]");
-                    }
-                    this.id = parts[2];
-                }
-            }
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java
new file mode 100644
index 00000000000..e6c5b09362c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/script/StoredScriptSource.java
@@ -0,0 +1,491 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * {@link StoredScriptSource} represents user-defined parameters for a script
+ * saved in the {@link ClusterState}.
+ */
+public class StoredScriptSource extends AbstractDiffable<StoredScriptSource> implements Writeable, ToXContent {
+
+    /**
+     * Standard {@link ParseField} for outer level of stored script source.
+     */
+    public static final ParseField SCRIPT_PARSE_FIELD = new ParseField("script");
+
+    /**
+     * Standard {@link ParseField} for outer level of stored script source.
+     */
+    public static final ParseField TEMPLATE_PARSE_FIELD = new ParseField("template");
+
+    /**
+     * Standard {@link ParseField} for lang on the inner level.
+     */
+    public static final ParseField LANG_PARSE_FIELD = new ParseField("lang");
+
+    /**
+     * Standard {@link ParseField} for code on the inner level.
+     */
+    public static final ParseField CODE_PARSE_FIELD = new ParseField("code");
+
+    /**
+     * Standard {@link ParseField} for options on the inner level.
+     */
+    public static final ParseField OPTIONS_PARSE_FIELD = new ParseField("options");
+
+    /**
+     * Helper class used by {@link ObjectParser} to store mutable {@link StoredScriptSource} variables and then
+     * construct an immutable {@link StoredScriptSource} object based on parsed XContent.
+     */
+    private static final class Builder {
+        private String lang;
+        private String code;
+        private Map<String, String> options;
+
+        private Builder() {
+            // This cannot default to an empty map because options are potentially added at multiple points.
+            this.options = new HashMap<>();
+        }
+
+        private void setLang(String lang) {
+            this.lang = lang;
+        }
+
+        /**
+         * Since stored scripts can accept templates rather than just scripts, they must also be able
+         * to handle template parsing, hence the need for custom parsing code.  Templates can
+         * consist of either an {@link String} or a JSON object.  If a JSON object is discovered
+         * then the content type option must also be saved as a compiler option.
+         */
+        private void setCode(XContentParser parser) {
+            try {
+                if (parser.currentToken() == Token.START_OBJECT) {
+                    XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
+                    code = builder.copyCurrentStructure(parser).bytes().utf8ToString();
+                    options.put(Script.CONTENT_TYPE_OPTION, parser.contentType().mediaType());
+                } else {
+                    code = parser.text();
+                }
+            } catch (IOException exception) {
+                throw new UncheckedIOException(exception);
+            }
+        }
+
+        /**
+         * Options may have already been added if a template was specified.
+         * Appends the user-defined compiler options with the internal compiler options.
+         */
+        private void setOptions(Map<String, String> options) {
+            if (options.containsKey(Script.CONTENT_TYPE_OPTION)) {
+                throw new IllegalArgumentException(Script.CONTENT_TYPE_OPTION + " cannot be user-specified");
+            }
+
+            this.options.putAll(options);
+        }
+
+        /**
+         * Validates the parameters and creates an {@link StoredScriptSource}.
+         */
+        private StoredScriptSource build() {
+            if (lang == null) {
+                throw new IllegalArgumentException("must specify lang for stored script");
+            } else if (lang.isEmpty()) {
+                throw new IllegalArgumentException("lang cannot be empty");
+            }
+
+            if (code == null) {
+                throw new IllegalArgumentException("must specify code for stored script");
+            } else if (code.isEmpty()) {
+                throw new IllegalArgumentException("code cannot be empty");
+            }
+
+            if (options.size() > 1 || options.size() == 1 && options.get(Script.CONTENT_TYPE_OPTION) == null) {
+                throw new IllegalArgumentException("illegal compiler options [" + options + "] specified");
+            }
+
+            return new StoredScriptSource(lang, code, options);
+        }
+    }
+
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>("stored script source", Builder::new);
+
+    static {
+        // Defines the fields necessary to parse a Script as XContent using an ObjectParser.
+        PARSER.declareString(Builder::setLang, LANG_PARSE_FIELD);
+        PARSER.declareField(Builder::setCode, parser -> parser, CODE_PARSE_FIELD, ValueType.OBJECT_OR_STRING);
+        PARSER.declareField(Builder::setOptions, XContentParser::mapStrings, OPTIONS_PARSE_FIELD, ValueType.OBJECT);
+    }
+
+    /**
+     * This will parse XContent into a {@link StoredScriptSource}.  The following formats can be parsed:
+     *
+     * The simple script format with no compiler options or user-defined params:
+     *
+     * Example:
+     * {@code
+     * {"script": "return Math.log(doc.popularity) * 100;"}
+     * }
+     *
+     * The above format requires the lang to be specified using the deprecated stored script namespace
+     * (as a url parameter during a put request).  See {@link ScriptMetaData} for more information about
+     * the stored script namespaces.
+     *
+     * The complex script format using the new stored script namespace
+     * where lang and code are required but options is optional:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "code" : "<code>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Example:
+     * {@code
+     * {
+     *     "script": {
+     *         "lang" : "painless",
+     *         "code" : "return Math.log(doc.popularity) * params.multiplier"
+     *     }
+     * }
+     * }
+     *
+     * The simple template format:
+     *
+     * {@code
+     * {
+     *     "query" : ...
+     * }
+     * }
+     *
+     * The complex template format:
+     *
+     * {@code
+     * {
+     *     "template": {
+     *         "query" : ...
+     *     }
+     * }
+     * }
+     *
+     * Note that templates can be handled as both strings and complex JSON objects.
+     * Also templates may be part of the 'code' parameter in a script.  The Parser
+     * can handle this case as well.
+     *
+     * @param lang    An optional parameter to allow for use of the deprecated stored
+     *                script namespace.  This will be used to specify the language
+     *                coming in as a url parameter from a request or for stored templates.
+     * @param content The content from the request to be parsed as described above.
+     * @return        The parsed {@link StoredScriptSource}.
+     */
+    public static StoredScriptSource parse(String lang, BytesReference content, XContentType xContentType) {
+        try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, content)) {
+            Token token = parser.nextToken();
+
+            if (token != Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{]");
+            }
+
+            token = parser.nextToken();
+
+            if (token != Token.FIELD_NAME) {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + ", expected [" +
+                    SCRIPT_PARSE_FIELD.getPreferredName() + ", " + TEMPLATE_PARSE_FIELD.getPreferredName());
+            }
+
+            String name = parser.currentName();
+
+            if (SCRIPT_PARSE_FIELD.getPreferredName().equals(name)) {
+                token = parser.nextToken();
+
+                if (token == Token.VALUE_STRING) {
+                    if (lang == null) {
+                        throw new IllegalArgumentException(
+                            "must specify lang as a url parameter when using the deprecated stored script namespace");
+                    }
+
+                    return new StoredScriptSource(lang, parser.text(), Collections.emptyMap());
+                } else if (token == Token.START_OBJECT) {
+                    if (lang == null) {
+                        return PARSER.apply(parser, null).build();
+                    } else {
+                        try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
+                            builder.copyCurrentStructure(parser);
+
+                            return new StoredScriptSource(lang, builder.string(),
+                                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, parser.contentType().mediaType()));
+                        }
+                    }
+
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "], expected [{, <code>]");
+                }
+            } else {
+                if (lang == null) {
+                    throw new IllegalArgumentException("unexpected stored script format");
+                }
+
+                if (TEMPLATE_PARSE_FIELD.getPreferredName().equals(name)) {
+                    token = parser.nextToken();
+
+                    if (token == Token.VALUE_STRING) {
+                        return new StoredScriptSource(lang, parser.text(),
+                            Collections.singletonMap(Script.CONTENT_TYPE_OPTION, parser.contentType().mediaType()));
+                    }
+                }
+
+                try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
+                    if (token != Token.START_OBJECT) {
+                        builder.startObject();
+                        builder.copyCurrentStructure(parser);
+                        builder.endObject();
+                    } else {
+                        builder.copyCurrentStructure(parser);
+                    }
+
+                    return new StoredScriptSource(lang, builder.string(),
+                        Collections.singletonMap(Script.CONTENT_TYPE_OPTION, parser.contentType().mediaType()));
+                }
+            }
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    /**
+     * This will parse XContent into a {@link StoredScriptSource}. The following format is what will be parsed:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "code" : "<code>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Note that the "code" parameter can also handle template parsing including from
+     * a complex JSON object.
+     */
+    public static StoredScriptSource fromXContent(XContentParser parser) throws IOException {
+        return PARSER.apply(parser, null).build();
+    }
+
+    /**
+     * Required for {@link ScriptMetaData.ScriptMetadataDiff}.  Uses
+     * the {@link StoredScriptSource#StoredScriptSource(StreamInput)}
+     * constructor.
+     */
+    public static Diff<StoredScriptSource> readDiffFrom(StreamInput in) throws IOException {
+        return readDiffFrom(StoredScriptSource::new, in);
+    }
+
+    private final String lang;
+    private final String code;
+    private final Map<String, String> options;
+
+    /**
+     * Constructor for use with {@link GetStoredScriptResponse}
+     * to support the deprecated stored script namespace.
+     */
+    public StoredScriptSource(String code) {
+        this.lang = null;
+        this.code = Objects.requireNonNull(code);
+        this.options = null;
+    }
+
+    /**
+     * Standard StoredScriptSource constructor.
+     * @param lang    The language to compile the script with.  Must not be {@code null}.
+     * @param code    The source code to compile with.  Must not be {@code null}.
+     * @param options Compiler options to be compiled with.  Must not be {@code null},
+     *                use an empty {@link Map} to represent no options.
+     */
+    public StoredScriptSource(String lang, String code, Map<String, String> options) {
+        this.lang = Objects.requireNonNull(lang);
+        this.code = Objects.requireNonNull(code);
+        this.options = Collections.unmodifiableMap(Objects.requireNonNull(options));
+    }
+
+    /**
+     * Reads a {@link StoredScriptSource} from a stream.  Version 5.3+ will read
+     * all of the lang, code, and options parameters.  For versions prior to 5.3,
+     * only the code parameter will be read in as a bytes reference.
+     */
+    public StoredScriptSource(StreamInput in) throws IOException {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            this.lang = in.readString();
+            this.code = in.readString();
+            @SuppressWarnings("unchecked")
+            Map<String, String> options = (Map<String, String>)(Map)in.readMap();
+            this.options = options;
+        } else {
+            this.lang = null;
+            this.code = in.readBytesReference().utf8ToString();
+            this.options = null;
+        }
+    }
+
+    /**
+     * Writes a {@link StoredScriptSource} to a stream.  Version 5.3+ will write
+     * all of the lang, code, and options parameters.  For versions prior to 5.3,
+     * only the code parameter will be read in as a bytes reference.
+     */
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            out.writeString(lang);
+            out.writeString(code);
+            @SuppressWarnings("unchecked")
+            Map<String, Object> options = (Map<String, Object>)(Map)this.options;
+            out.writeMap(options);
+        } else {
+            out.writeBytesReference(new BytesArray(code));
+        }
+    }
+
+    /**
+     * This will write XContent from a {@link StoredScriptSource}. The following format will be written:
+     *
+     * {@code
+     * {
+     *     "script" : {
+     *         "lang" : "<lang>",
+     *         "code" : "<code>",
+     *         "options" : {
+     *             "option0" : "<option0>",
+     *             "option1" : "<option1>",
+     *             ...
+     *         }
+     *     }
+     * }
+     * }
+     *
+     * Note that the 'code' parameter can also handle templates written as complex JSON.
+     */
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(LANG_PARSE_FIELD.getPreferredName(), lang);
+        builder.field(CODE_PARSE_FIELD.getPreferredName(), code);
+        builder.field(OPTIONS_PARSE_FIELD.getPreferredName(), options);
+        builder.endObject();
+
+        return builder;
+    }
+
+    @Override
+    public boolean isFragment() {
+        return false;
+    }
+
+    /**
+     * @return The language used for compiling this script.
+     */
+    public String getLang() {
+        return lang;
+    }
+
+    /**
+     * @return The code used for compiling this script.
+     */
+    public String getCode() {
+        return code;
+    }
+
+    /**
+     * @return The compiler options used for this script.
+     */
+    public Map<String, String> getOptions() {
+        return options;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        StoredScriptSource that = (StoredScriptSource)o;
+
+        if (lang != null ? !lang.equals(that.lang) : that.lang != null) return false;
+        if (code != null ? !code.equals(that.code) : that.code != null) return false;
+        return options != null ? options.equals(that.options) : that.options == null;
+
+    }
+
+    @Override
+    public int hashCode() {
+        int result = lang != null ? lang.hashCode() : 0;
+        result = 31 * result + (code != null ? code.hashCode() : 0);
+        result = 31 * result + (options != null ? options.hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "StoredScriptSource{" +
+            "lang='" + lang + '\'' +
+            ", code='" + code + '\'' +
+            ", options=" + options +
+            '}';
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
index 0c08d6c949e..d981f4e68df 100644
--- a/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
@@ -30,7 +30,6 @@ import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
@@ -53,6 +52,7 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
@@ -114,6 +114,7 @@ final class DefaultSearchContext extends SearchContext {
     private Float minimumScore;
     private boolean trackScores = false; // when sorting, track scores as well...
     private FieldDoc searchAfter;
+    private CollapseContext collapse;
     private boolean lowLevelCancellation;
     // filter for sliced scroll
     private SliceBuilder sliceBuilder;
@@ -152,9 +153,7 @@ final class DefaultSearchContext extends SearchContext {
 
     DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
                          IndexService indexService, IndexShard indexShard,
-                         BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher, TimeValue timeout,
-                         FetchPhase fetchPhase) {
-        super(parseFieldMatcher);
+                         BigArrays bigArrays, Counter timeEstimateCounter, TimeValue timeout, FetchPhase fetchPhase) {
         this.id = id;
         this.request = request;
         this.fetchPhase = fetchPhase;
@@ -584,6 +583,17 @@ final class DefaultSearchContext extends SearchContext {
         return searchAfter;
     }
 
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        this.collapse = collapse;
+        return this;
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return collapse;
+    }
+
     public SearchContext sliceBuilder(SliceBuilder sliceBuilder) {
         this.sliceBuilder = sliceBuilder;
         return this;
diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
index 8d75216fe12..bf696fcc917 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchExtBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,13 +35,12 @@ import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec;
  * read from the incoming stream, usually done adding a constructor that takes {@link StreamInput} as
  * an argument.
  *
- * Registration happens through {@link SearchPlugin#getSearchExts()}, which also needs a {@link SearchExtParser} that's able to parse
+ * Registration happens through {@link SearchPlugin#getSearchExts()}, which also needs a {@link CheckedFunction} that's able to parse
  * the incoming request from the REST layer into the proper {@link SearchExtBuilder} subclass.
  *
  * {@link #getWriteableName()} must return the same name as the one used for the registration
  * of the {@link SearchExtSpec}.
  *
- * @see SearchExtParser
  * @see SearchExtSpec
  */
 public abstract class SearchExtBuilder implements NamedWriteable, ToXContent {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHit.java b/core/src/main/java/org/elasticsearch/search/SearchHit.java
index c9ccddd05e6..ef9aef8fbb7 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -24,7 +24,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 
 import java.util.Map;
@@ -34,7 +34,7 @@ import java.util.Map;
  *
  * @see SearchHits
  */
-public interface SearchHit extends Streamable, ToXContent, Iterable<SearchHitField> {
+public interface SearchHit extends Streamable, ToXContentObject, Iterable<SearchHitField> {
 
     /**
      * The score.
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index cdfcaeab90b..01c4e3488e7 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -20,14 +20,20 @@
 package org.elasticsearch.search;
 
 import org.apache.lucene.search.BooleanQuery;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.CommonTermsQueryBuilder;
@@ -54,6 +60,7 @@ import org.elasticsearch.index.query.NestedQueryBuilder;
 import org.elasticsearch.index.query.ParentIdQueryBuilder;
 import org.elasticsearch.index.query.PrefixQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.RegexpQueryBuilder;
@@ -79,10 +86,8 @@ import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.LinearDecayFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
-import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
 import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.plugins.SearchPlugin.AggregationSpec;
 import org.elasticsearch.plugins.SearchPlugin.FetchPhaseConstructionContext;
@@ -91,11 +96,14 @@ import org.elasticsearch.plugins.SearchPlugin.QuerySpec;
 import org.elasticsearch.plugins.SearchPlugin.ScoreFunctionSpec;
 import org.elasticsearch.plugins.SearchPlugin.SearchExtSpec;
 import org.elasticsearch.plugins.SearchPlugin.SearchExtensionSpec;
+import org.elasticsearch.plugins.SearchPlugin.SuggesterSpec;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.InternalAdjacencyMatrix;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@@ -215,6 +223,7 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
+import org.elasticsearch.search.collapse.ExpandCollapseSearchResponseListener;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
@@ -229,6 +238,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightPhase;
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter;
+import org.elasticsearch.search.fetch.subphase.highlight.UnifiedHighlighter;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -236,20 +246,19 @@ import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.completion.CompletionSuggester;
+import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.Laplace;
 import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
 import org.elasticsearch.search.suggest.phrase.SmoothingModel;
 import org.elasticsearch.search.suggest.phrase.StupidBackoff;
-import org.elasticsearch.search.suggest.term.TermSuggester;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
@@ -265,29 +274,26 @@ public class SearchModule {
 
     private final boolean transportClient;
     private final Map<String, Highlighter> highlighters;
-    private final Map<String, Suggester<?>> suggesters;
-    private final ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionParserRegistry = new ParseFieldRegistry<>("score_function");
-    private final IndicesQueriesRegistry queryParserRegistry = new IndicesQueriesRegistry();
-    private final ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry = new ParseFieldRegistry<>("aggregation");
-    private final ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry = new ParseFieldRegistry<>(
-            "pipline_aggregation");
-    private final AggregatorParsers aggregatorParsers = new AggregatorParsers(aggregationParserRegistry, pipelineAggregationParserRegistry);
     private final ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry = new ParseFieldRegistry<>(
             "significance_heuristic");
     private final ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageModelParserRegistry = new ParseFieldRegistry<>(
             "moving_avg_model");
 
     private final List<FetchSubPhase> fetchSubPhases = new ArrayList<>();
-    private final SearchExtRegistry searchExtParserRegistry = new SearchExtRegistry();
+    private final List<BiConsumer<SearchRequest, SearchResponse> > searchResponseListeners = new ArrayList<> ();
 
     private final Settings settings;
-    private final List<Entry> namedWriteables = new ArrayList<>();
-    private final SearchRequestParsers searchRequestParsers;
+    private final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
+    private final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
 
     public SearchModule(Settings settings, boolean transportClient, List<SearchPlugin> plugins) {
+        this(settings, transportClient, plugins, null);
+    }
+
+    public SearchModule(Settings settings, boolean transportClient, List<SearchPlugin> plugins, Client client) {
         this.settings = settings;
         this.transportClient = transportClient;
-        suggesters = setupSuggesters(plugins);
+        registerSuggesters(plugins);
         highlighters = setupHighlighters(settings, plugins);
         registerScoreFunctions(plugins);
         registerQueryParsers(plugins);
@@ -300,24 +306,18 @@ public class SearchModule {
         registerPipelineAggregations(plugins);
         registerFetchSubPhases(plugins);
         registerSearchExts(plugins);
+        if (false == transportClient) {
+            registerSearchResponseListeners(client, plugins);
+        }
         registerShapes();
-        searchRequestParsers = new SearchRequestParsers(queryParserRegistry, aggregatorParsers, getSuggesters(), searchExtParserRegistry);
     }
 
-    public List<Entry> getNamedWriteables() {
+    public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
         return namedWriteables;
     }
 
-    public Suggesters getSuggesters() {
-        return new Suggesters(suggesters);
-    }
-
-    public IndicesQueriesRegistry getQueryParserRegistry() {
-        return queryParserRegistry;
-    }
-
-    public SearchRequestParsers getSearchRequestParsers() {
-        return searchRequestParsers;
+    public List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return namedXContents;
     }
 
     /**
@@ -342,10 +342,10 @@ public class SearchModule {
     }
 
     /**
-     * Parsers for {@link AggregationBuilder}s and {@link PipelineAggregationBuilder}s.
+     * Returns the search response listeners registry
      */
-    public AggregatorParsers getAggregatorParsers() {
-        return aggregatorParsers;
+    public List<BiConsumer<SearchRequest, SearchResponse> > getSearchResponseListeners() {
+        return searchResponseListeners;
     }
 
     private void registerAggregations(List<SearchPlugin> plugins) {
@@ -381,6 +381,8 @@ public class SearchModule {
                 FilterAggregationBuilder::parse).addResultReader(InternalFilter::new));
         registerAggregation(new AggregationSpec(FiltersAggregationBuilder.NAME, FiltersAggregationBuilder::new,
                 FiltersAggregationBuilder::parse).addResultReader(InternalFilters::new));
+        registerAggregation(new AggregationSpec(AdjacencyMatrixAggregationBuilder.NAME, AdjacencyMatrixAggregationBuilder::new,
+                AdjacencyMatrixAggregationBuilder.getParser()).addResultReader(InternalAdjacencyMatrix::new));
         registerAggregation(new AggregationSpec(SamplerAggregationBuilder.NAME, SamplerAggregationBuilder::new,
                 SamplerAggregationBuilder::parse)
                     .addResultReader(InternalSampler.NAME, InternalSampler::new)
@@ -433,13 +435,17 @@ public class SearchModule {
 
     private void registerAggregation(AggregationSpec spec) {
         if (false == transportClient) {
-            aggregationParserRegistry.register(spec.getParser(), spec.getName());
+            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+                return spec.getParser().parse(context.name, context.queryParseContext);
+            }));
         }
-        namedWriteables.add(new Entry(AggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(AggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
         for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.getResultReaders().entrySet()) {
             String writeableName = t.getKey();
             Writeable.Reader<? extends InternalAggregation> internalReader = t.getValue();
-            namedWriteables.add(new Entry(InternalAggregation.class, writeableName, internalReader));
+            namedWriteables.add(new NamedWriteableRegistry.Entry(InternalAggregation.class, writeableName, internalReader));
         }
     }
 
@@ -526,12 +532,18 @@ public class SearchModule {
 
     private void registerPipelineAggregation(PipelineAggregationSpec spec) {
         if (false == transportClient) {
-            pipelineAggregationParserRegistry.register(spec.getParser(), spec.getName());
+            namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, spec.getName(), (p, c) -> {
+                AggregatorFactories.AggParseContext context = (AggregatorFactories.AggParseContext) c;
+                return spec.getParser().parse(context.name, context.queryParseContext);
+            }));
         }
-        namedWriteables.add(new Entry(PipelineAggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
-        namedWriteables.add(new Entry(PipelineAggregator.class, spec.getName().getPreferredName(), spec.getAggregatorReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(PipelineAggregationBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(PipelineAggregator.class, spec.getName().getPreferredName(), spec.getAggregatorReader()));
         for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> resultReader : spec.getResultReaders().entrySet()) {
-            namedWriteables.add(new Entry(InternalAggregation.class, resultReader.getKey(), resultReader.getValue()));
+            namedWriteables
+                    .add(new NamedWriteableRegistry.Entry(InternalAggregation.class, resultReader.getKey(), resultReader.getValue()));
         }
     }
 
@@ -542,14 +554,14 @@ public class SearchModule {
     }
 
     private void registerRescorers() {
-        namedWriteables.add(new Entry(RescoreBuilder.class, QueryRescorerBuilder.NAME, QueryRescorerBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(RescoreBuilder.class, QueryRescorerBuilder.NAME, QueryRescorerBuilder::new));
     }
 
     private void registerSorts() {
-        namedWriteables.add(new Entry(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
-        namedWriteables.add(new Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, GeoDistanceSortBuilder.NAME, GeoDistanceSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
     }
 
     private <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchPlugin, List<T>> producer, Consumer<T> consumer) {
@@ -561,28 +573,26 @@ public class SearchModule {
     }
 
     public static void registerSmoothingModels(List<Entry> namedWriteables) {
-        namedWriteables.add(new Entry(SmoothingModel.class, Laplace.NAME, Laplace::new));
-        namedWriteables.add(new Entry(SmoothingModel.class, LinearInterpolation.NAME, LinearInterpolation::new));
-        namedWriteables.add(new Entry(SmoothingModel.class, StupidBackoff.NAME, StupidBackoff::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, Laplace.NAME, Laplace::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, LinearInterpolation.NAME, LinearInterpolation::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SmoothingModel.class, StupidBackoff.NAME, StupidBackoff::new));
     }
 
-    private Map<String, Suggester<?>> setupSuggesters(List<SearchPlugin> plugins) {
+    private void registerSuggesters(List<SearchPlugin> plugins) {
         registerSmoothingModels(namedWriteables);
 
-        // Suggester<?> is weird - it is both a Parser and a reader....
-        NamedRegistry<Suggester<?>> suggesters = new NamedRegistry<Suggester<?>>("suggester") {
-            @Override
-            public void register(String name, Suggester<?> t) {
-                super.register(name, t);
-                namedWriteables.add(new Entry(SuggestionBuilder.class, name, t));
-            }
-        };
-        suggesters.register("phrase", PhraseSuggester.INSTANCE);
-        suggesters.register("term", TermSuggester.INSTANCE);
-        suggesters.register("completion", CompletionSuggester.INSTANCE);
+        registerSuggester(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
+        registerSuggester(new SuggesterSpec<>("phrase", PhraseSuggestionBuilder::new, PhraseSuggestionBuilder::fromXContent));
+        registerSuggester(new SuggesterSpec<>("completion", CompletionSuggestionBuilder::new, CompletionSuggestionBuilder::fromXContent));
 
-        suggesters.extractAndRegister(plugins, SearchPlugin::getSuggesters);
-        return unmodifiableMap(suggesters.getRegistry());
+        registerFromPlugin(plugins, SearchPlugin::getSuggesters, this::registerSuggester);
+    }
+
+    private void registerSuggester(SuggesterSpec<?> suggester) {
+        namedWriteables.add(new NamedWriteableRegistry.Entry(
+                SuggestionBuilder.class, suggester.getName().getPreferredName(), suggester.getReader()));
+        namedXContents.add(new NamedXContentRegistry.Entry(SuggestionBuilder.class, suggester.getName(),
+                suggester.getParser()));
     }
 
     private Map<String, Highlighter> setupHighlighters(Settings settings, List<SearchPlugin> plugins) {
@@ -590,7 +600,7 @@ public class SearchModule {
         highlighters.register("fvh",  new FastVectorHighlighter(settings));
         highlighters.register("plain", new PlainHighlighter());
         highlighters.register("postings", new PostingsHighlighter());
-
+        highlighters.register("unified", new UnifiedHighlighter());
         highlighters.extractAndRegister(plugins, SearchPlugin::getHighlighters);
 
         return unmodifiableMap(highlighters.getRegistry());
@@ -612,14 +622,18 @@ public class SearchModule {
 
         //weight doesn't have its own parser, so every function supports it out of the box.
         //Can be a single function too when not associated to any other function, which is why it needs to be registered manually here.
-        namedWriteables.add(new Entry(ScoreFunctionBuilder.class, WeightBuilder.NAME, WeightBuilder::new));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(ScoreFunctionBuilder.class, WeightBuilder.NAME, WeightBuilder::new));
 
         registerFromPlugin(plugins, SearchPlugin::getScoreFunctions, this::registerScoreFunction);
     }
 
     private void registerScoreFunction(ScoreFunctionSpec<?> scoreFunction) {
-        scoreFunctionParserRegistry.register(scoreFunction.getParser(), scoreFunction.getName());
-        namedWriteables.add(new Entry(ScoreFunctionBuilder.class, scoreFunction.getName().getPreferredName(), scoreFunction.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(
+                ScoreFunctionBuilder.class, scoreFunction.getName().getPreferredName(), scoreFunction.getReader()));
+        // TODO remove funky contexts
+        namedXContents.add(new NamedXContentRegistry.Entry(
+                ScoreFunctionBuilder.class, scoreFunction.getName(),
+                (XContentParser p, Object c) -> scoreFunction.getParser().fromXContent((QueryParseContext) c)));
     }
 
     private void registerValueFormats() {
@@ -635,7 +649,7 @@ public class SearchModule {
      * Register a new ValueFormat.
      */
     private void registerValueFormat(String name, Writeable.Reader<? extends DocValueFormat> reader) {
-        namedWriteables.add(new Entry(DocValueFormat.class, name, reader));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, name, reader));
     }
 
     private void registerSignificanceHeuristics(List<SearchPlugin> plugins) {
@@ -651,7 +665,8 @@ public class SearchModule {
 
     private void registerSignificanceHeuristic(SearchExtensionSpec<SignificanceHeuristic, SignificanceHeuristicParser> heuristic) {
         significanceHeuristicParserRegistry.register(heuristic.getParser(), heuristic.getName());
-        namedWriteables.add(new Entry(SignificanceHeuristic.class, heuristic.getName().getPreferredName(), heuristic.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SignificanceHeuristic.class, heuristic.getName().getPreferredName(),
+                heuristic.getReader()));
     }
 
     private void registerMovingAverageModels(List<SearchPlugin> plugins) {
@@ -666,7 +681,8 @@ public class SearchModule {
 
     private void registerMovingAverageModel(SearchExtensionSpec<MovAvgModel, MovAvgModel.AbstractModelParser> movAvgModel) {
         movingAverageModelParserRegistry.register(movAvgModel.getParser(), movAvgModel.getName());
-        namedWriteables.add(new Entry(MovAvgModel.class, movAvgModel.getName().getPreferredName(), movAvgModel.getReader()));
+        namedWriteables.add(
+                new NamedWriteableRegistry.Entry(MovAvgModel.class, movAvgModel.getName().getPreferredName(), movAvgModel.getReader()));
     }
 
     private void registerFetchSubPhases(List<SearchPlugin> plugins) {
@@ -683,13 +699,20 @@ public class SearchModule {
         registerFromPlugin(plugins, p -> p.getFetchSubPhases(context), this::registerFetchSubPhase);
     }
 
+    private void registerSearchResponseListeners(Client client, List<SearchPlugin> plugins) {
+        if (client != null) {
+            registerSearchResponseListener(new ExpandCollapseSearchResponseListener(client));
+        }
+        registerFromPlugin(plugins, p -> p.getSearchResponseListeners(), this::registerSearchResponseListener);
+    }
+
     private void registerSearchExts(List<SearchPlugin> plugins) {
         registerFromPlugin(plugins, SearchPlugin::getSearchExts, this::registerSearchExt);
     }
 
     private void registerSearchExt(SearchExtSpec<?> spec) {
-        searchExtParserRegistry.register(spec.getParser(), spec.getName());
-        namedWriteables.add(new Entry(SearchExtBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedXContents.add(new NamedXContentRegistry.Entry(SearchExtBuilder.class, spec.getName(), spec.getParser()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SearchExtBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
     }
 
     private void registerFetchSubPhase(FetchSubPhase subPhase) {
@@ -742,7 +765,7 @@ public class SearchModule {
         registerQuery(
                 new QuerySpec<>(SpanMultiTermQueryBuilder.NAME, SpanMultiTermQueryBuilder::new, SpanMultiTermQueryBuilder::fromXContent));
         registerQuery(new QuerySpec<>(FunctionScoreQueryBuilder.NAME, FunctionScoreQueryBuilder::new,
-                c -> FunctionScoreQueryBuilder.fromXContent(scoreFunctionParserRegistry, c)));
+                FunctionScoreQueryBuilder::fromXContent));
         registerQuery(
                 new QuerySpec<>(SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new, SimpleQueryStringBuilder::fromXContent));
         registerQuery(new QuerySpec<>(TypeQueryBuilder.NAME, TypeQueryBuilder::new, TypeQueryBuilder::fromXContent));
@@ -763,15 +786,16 @@ public class SearchModule {
     }
 
     private void registerQuery(QuerySpec<?> spec) {
-        queryParserRegistry.register(spec.getParser(), spec.getName());
-        namedWriteables.add(new Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader()));
+        namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(),
+                (p, c) -> spec.getParser().fromXContent((QueryParseContext) c)));
+    }
+
+    private void registerSearchResponseListener(BiConsumer<SearchRequest, SearchResponse> listener) {
+        searchResponseListeners.add(requireNonNull(listener, "SearchResponseListener must not be null"));
     }
 
     public FetchPhase getFetchPhase() {
         return new FetchPhase(fetchSubPhases);
     }
-
-    public SearchExtRegistry getSearchExtRegistry() {
-        return searchExtParserRegistry;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchParseException.java b/core/src/main/java/org/elasticsearch/search/SearchParseException.java
index 223225af2a6..f8ae3c7f674 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchParseException.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchParseException.java
@@ -72,12 +72,11 @@ public class SearchParseException extends SearchContextException {
     }
 
     @Override
-    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+    protected void metadataToXContent(XContentBuilder builder, Params params) throws IOException {
         if (lineNumber != UNKNOWN_POSITION) {
             builder.field("line", lineNumber);
             builder.field("col", columnNumber);
         }
-        super.innerToXContent(builder, params);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java b/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java
deleted file mode 100644
index ba3afc15771..00000000000
--- a/core/src/main/java/org/elasticsearch/search/SearchRequestParsers.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search;
-
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.suggest.Suggesters;
-
-/**
- * A container for all parsers used to parse
- * {@link org.elasticsearch.action.search.SearchRequest} objects from a rest request.
- */
-public class SearchRequestParsers {
-    // TODO: this class should be renamed to SearchRequestParser, and all the parse
-    // methods split across RestSearchAction and SearchSourceBuilder should be moved here
-    // TODO: make all members private once parsing functions are moved here
-
-    // TODO: IndicesQueriesRegistry should be removed and just have the map of query parsers here
-    /**
-     * Query parsers that may be used in search requests.
-     * @see org.elasticsearch.index.query.QueryParseContext
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers,
-     *      Suggesters, SearchExtRegistry)
-     */
-    public final IndicesQueriesRegistry queryParsers;
-
-    // TODO: AggregatorParsers should be removed and the underlying maps of agg
-    // and pipeline agg parsers should be here
-    /**
-     * Agg and pipeline agg parsers that may be used in search requests.
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers,
-     *      Suggesters, SearchExtRegistry)
-     */
-    public final AggregatorParsers aggParsers;
-
-    // TODO: Suggesters should be removed and the underlying map moved here
-    /**
-     * Suggesters that may be used in search requests.
-     * @see org.elasticsearch.search.builder.SearchSourceBuilder#fromXContent(QueryParseContext, AggregatorParsers,
-     *      Suggesters, SearchExtRegistry)
-     */
-    public final Suggesters suggesters;
-
-    /**
-     * Pluggable section that can be parsed out of a search section, within the ext element
-     */
-    public final SearchExtRegistry searchExtParsers;
-
-    public SearchRequestParsers(IndicesQueriesRegistry queryParsers, AggregatorParsers aggParsers, Suggesters suggesters,
-                                SearchExtRegistry searchExtParsers) {
-        this.queryParsers = queryParsers;
-        this.aggParsers = aggParsers;
-        this.suggesters = suggesters;
-        this.searchExtParsers = searchExtParsers;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 39c029996f7..3f2fd3b31a4 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Setting;
@@ -41,6 +40,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -147,12 +147,9 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
 
     private final ConcurrentMapLong<SearchContext> activeContexts = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
-    private final ParseFieldMatcher parseFieldMatcher;
-
     public SearchService(ClusterService clusterService, IndicesService indicesService,
                          ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, FetchPhase fetchPhase) {
         super(clusterService.getSettings());
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.indicesService = indicesService;
@@ -579,8 +576,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Engine.Searcher engineSearcher = searcher == null ? indexShard.acquireSearcher("search") : searcher;
 
         final DefaultSearchContext searchContext = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget,
-            engineSearcher, indexService, indexShard, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
-            timeout, fetchPhase);
+            engineSearcher, indexService, indexShard, bigArrays, threadPool.estimatedTimeInMillisCounter(), timeout, fetchPhase);
         boolean success = false;
         try {
             // we clone the query shard context here just for rewriting otherwise we
@@ -797,6 +793,11 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             }
             context.storedFieldsContext(source.storedFields());
         }
+
+        if (source.collapse() != null) {
+            final CollapseContext collapseContext = source.collapse().build(context);
+            context.collapse(collapseContext);
+        }
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
index 9fb83227188..5fd20555f81 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchShardTarget.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.Index;
@@ -35,21 +34,20 @@ import java.io.IOException;
  */
 public class SearchShardTarget implements Writeable, Comparable<SearchShardTarget> {
 
-    private Text nodeId;
-    private Text index;
-    private ShardId shardId;
+    private final Text nodeId;
+    private final ShardId shardId;
 
     public SearchShardTarget(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             nodeId = in.readText();
+        } else {
+            nodeId = null;
         }
         shardId = ShardId.readShardId(in);
-        index = new Text(shardId.getIndexName());
     }
 
     public SearchShardTarget(String nodeId, ShardId shardId) {
         this.nodeId = nodeId == null ? null : new Text(nodeId);
-        this.index = new Text(shardId.getIndexName());
         this.shardId = shardId;
     }
 
@@ -58,33 +56,16 @@ public class SearchShardTarget implements Writeable, Comparable<SearchShardTarge
     }
 
     @Nullable
-    public String nodeId() {
+    public String getNodeId() {
         return nodeId.string();
     }
 
-    @Nullable
-    public String getNodeId() {
-        return nodeId();
-    }
-
-    public Text nodeIdText() {
+    public Text getNodeIdText() {
         return this.nodeId;
     }
 
-    public String index() {
-        return index.string();
-    }
-
     public String getIndex() {
-        return index();
-    }
-
-    public Text indexText() {
-        return this.index;
-    }
-
-    public ShardId shardId() {
-        return shardId;
+        return shardId.getIndexName();
     }
 
     public ShardId getShardId() {
@@ -93,7 +74,7 @@ public class SearchShardTarget implements Writeable, Comparable<SearchShardTarge
 
     @Override
     public int compareTo(SearchShardTarget o) {
-        int i = index.string().compareTo(o.index());
+        int i = shardId.getIndexName().compareTo(o.getIndex());
         if (i == 0) {
             i = shardId.getId() - o.shardId.id();
         }
@@ -125,7 +106,7 @@ public class SearchShardTarget implements Writeable, Comparable<SearchShardTarge
     @Override
     public int hashCode() {
         int result = nodeId != null ? nodeId.hashCode() : 0;
-        result = 31 * result + (index != null ? index.hashCode() : 0);
+        result = 31 * result + (shardId.getIndexName() != null ? shardId.getIndexName().hashCode() : 0);
         result = 31 * result + shardId.hashCode();
         return result;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
index 47e512325fe..ef82c48b4e7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -40,17 +39,16 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
      * Constructs a new aggregation builder.
      *
      * @param name  The aggregation name
-     * @param type  The aggregation type
      */
-    public AbstractAggregationBuilder(String name, Type type) {
-        super(name, type);
+    public AbstractAggregationBuilder(String name) {
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
-    protected AbstractAggregationBuilder(StreamInput in, Type type) throws IOException {
-        super(in.readString(), type);
+    protected AbstractAggregationBuilder(StreamInput in) throws IOException {
+        super(in.readString());
         factoriesBuilder = new AggregatorFactories.Builder(in);
         metaData = in.readMap();
     }
@@ -117,8 +115,10 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
         return (AB) this;
     }
 
-    public String getType() {
-        return type.name();
+    @Override
+    public final String getWriteableName() {
+        // We always use the type of the aggregation as the writeable name
+        return getType();
     }
 
     @Override
@@ -137,7 +137,7 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
         if (this.metaData != null) {
             builder.field("meta", this.metaData);
         }
-        builder.field(type.name());
+        builder.field(getType());
         internalXContent(builder, params);
 
         if (factoriesBuilder != null && (factoriesBuilder.count()) > 0) {
@@ -153,7 +153,7 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
 
     @Override
     public int hashCode() {
-        return Objects.hash(factoriesBuilder, metaData, name, type, doHashCode());
+        return Objects.hash(factoriesBuilder, metaData, name, doHashCode());
     }
 
     protected abstract int doHashCode();
@@ -168,8 +168,6 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
         AbstractAggregationBuilder<AB> other = (AbstractAggregationBuilder<AB>) obj;
         if (!Objects.equals(name, other.name))
             return false;
-        if (!Objects.equals(type, other.type))
-            return false;
         if (!Objects.equals(metaData, other.metaData))
             return false;
         if (!Objects.equals(factoriesBuilder, other.factoriesBuilder))
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 95317c293c4..14875895b77 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -34,27 +34,21 @@ import java.util.Map;
  */
 public abstract class AggregationBuilder
     extends ToXContentToBytes
-    implements NamedWriteable, ToXContent {
+    implements NamedWriteable, ToXContent, BaseAggregationBuilder {
 
     protected final String name;
-    protected final Type type;
     protected AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder();
 
     /**
      * Constructs a new aggregation builder.
      *
      * @param name  The aggregation name
-     * @param type  The aggregation type
      */
-    protected AggregationBuilder(String name, Type type) {
+    protected AggregationBuilder(String name) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
         }
-        if (type == null) {
-            throw new IllegalArgumentException("[type] must not be null: [" + name + "]");
-        }
         this.name = name;
-        this.type = type;
     }
 
     /** Return this aggregation's name. */
@@ -66,6 +60,7 @@ public abstract class AggregationBuilder
     protected abstract AggregatorFactory<?> build(SearchContext context, AggregatorFactory<?> parent) throws IOException;
 
     /** Associate metadata with this {@link AggregationBuilder}. */
+    @Override
     public abstract AggregationBuilder setMetaData(Map<String, Object> metaData);
 
     /** Add a sub aggregation to this builder. */
@@ -77,13 +72,14 @@ public abstract class AggregationBuilder
     /**
      * Internal: Registers sub-factories with this factory. The sub-factory will be
      * responsible for the creation of sub-aggregators under the aggregator
-     * created by this factory. This is only for use by {@link AggregatorParsers}.
+     * created by this factory. This is only for use by {@link AggregatorFactories#parseAggregators(QueryParseContext)}.
      *
      * @param subFactories
      *            The sub-factories
      * @return this factory (fluent interface)
      */
-    protected abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
+    @Override
+    public abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
 
     /** Common xcontent fields shared among aggregator builders */
     public static final class CommonFields extends ParseField.CommonFields {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index b1818971d6b..f43c2670abd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -21,6 +21,8 @@ package org.elasticsearch.search.aggregations;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.Children;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
@@ -82,6 +84,8 @@ import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationB
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
+import java.util.Map;
+
 /**
  * Utility class to create aggregations.
  */
@@ -159,6 +163,20 @@ public class AggregationBuilders {
     public static FiltersAggregationBuilder filters(String name, QueryBuilder... filters) {
         return new FiltersAggregationBuilder(name, filters);
     }
+    
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name.
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, filters);
+    }    
+    
+    /**
+     * Create a new {@link AdjacencyMatrix} aggregation with the given name and separator
+     */
+    public static AdjacencyMatrixAggregationBuilder adjacencyMatrix(String name, String separator,  Map<String, QueryBuilder> filters) {
+        return new AdjacencyMatrixAggregationBuilder(name, separator, filters);
+    }     
 
     /**
      * Create a new {@link Sampler} aggregation with the given name.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index 37cfe18e83b..44adb33c01b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -129,10 +128,10 @@ public abstract class Aggregator extends BucketCollector implements Releasable {
             return parseField;
         }
 
-        public static SubAggCollectionMode parse(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static SubAggCollectionMode parse(String value) {
             SubAggCollectionMode[] modes = SubAggCollectionMode.values();
             for (SubAggCollectionMode mode : modes) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 200232930a9..106335380cc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -19,10 +19,13 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement;
@@ -32,6 +35,7 @@ import org.elasticsearch.search.profile.aggregation.ProfilingAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -39,8 +43,126 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class AggregatorFactories {
+    public static final Pattern VALID_AGG_NAME = Pattern.compile("[^\\[\\]>]+");
+
+    /**
+     * Parses the aggregation request recursively generating aggregator factories in turn.
+     *
+     * @param parseContext   The parse context.
+     *
+     * @return          The parsed aggregator factories.
+     *
+     * @throws IOException When parsing fails for unknown reasons.
+     */
+    public static AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext) throws IOException {
+        return parseAggregators(parseContext, 0);
+    }
+
+    private static AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext, int level) throws IOException {
+        Matcher validAggMatcher = VALID_AGG_NAME.matcher("");
+        AggregatorFactories.Builder factories = new AggregatorFactories.Builder();
+
+        XContentParser.Token token = null;
+        XContentParser parser = parseContext.parser();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token != XContentParser.Token.FIELD_NAME) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " in [aggs]: aggregations definitions must start with the name of the aggregation.");
+            }
+            final String aggregationName = parser.currentName();
+            if (!validAggMatcher.reset(aggregationName).matches()) {
+                throw new ParsingException(parser.getTokenLocation(), "Invalid aggregation name [" + aggregationName
+                        + "]. Aggregation names must be alpha-numeric and can only contain '_' and '-'");
+            }
+
+            token = parser.nextToken();
+            if (token != XContentParser.Token.START_OBJECT) {
+                throw new ParsingException(parser.getTokenLocation(), "Aggregation definition for [" + aggregationName + " starts with a ["
+                        + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
+            }
+
+            BaseAggregationBuilder aggBuilder = null;
+            AggregatorFactories.Builder subFactories = null;
+
+            Map<String, Object> metaData = null;
+
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token != XContentParser.Token.FIELD_NAME) {
+                    throw new ParsingException(
+                            parser.getTokenLocation(), "Expected [" + XContentParser.Token.FIELD_NAME + "] under a ["
+                            + XContentParser.Token.START_OBJECT + "], but got a [" + token + "] in [" + aggregationName + "]",
+                            parser.getTokenLocation());
+                }
+                final String fieldName = parser.currentName();
+
+                token = parser.nextToken();
+                if (token == XContentParser.Token.START_OBJECT) {
+                    switch (fieldName) {
+                    case "meta":
+                        metaData = parser.map();
+                        break;
+                    case "aggregations":
+                    case "aggs":
+                        if (subFactories != null) {
+                            throw new ParsingException(parser.getTokenLocation(),
+                                    "Found two sub aggregation definitions under [" + aggregationName + "]");
+                        }
+                        subFactories = parseAggregators(parseContext, level + 1);
+                        break;
+                    default:
+                        if (aggBuilder != null) {
+                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
+                                    + aggregationName + "]: [" + aggBuilder.getType() + "] and [" + fieldName + "]");
+                        }
+
+                        aggBuilder = parser.namedObject(BaseAggregationBuilder.class, fieldName,
+                                new AggParseContext(aggregationName, parseContext));
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] under ["
+                            + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]");
+                }
+            }
+
+            if (aggBuilder == null) {
+                throw new ParsingException(parser.getTokenLocation(), "Missing definition for aggregation [" + aggregationName + "]",
+                        parser.getTokenLocation());
+            } else {
+                if (metaData != null) {
+                    aggBuilder.setMetaData(metaData);
+                }
+
+                if (subFactories != null) {
+                    aggBuilder.subAggregations(subFactories);
+                }
+
+                if (aggBuilder instanceof AggregationBuilder) {
+                    factories.addAggregator((AggregationBuilder) aggBuilder);
+                } else {
+                    factories.addPipelineAggregator((PipelineAggregationBuilder) aggBuilder);
+                }
+            }
+        }
+
+        return factories;
+    }
+
+    /**
+     * Context to parse and aggregation. This should eventually be removed and replaced with a String.
+     */
+    public static final class AggParseContext {
+        public final String name;
+        public final QueryParseContext queryParseContext;
+
+        public AggParseContext(String name, QueryParseContext queryParseContext) {
+            this.name = name;
+            this.queryParseContext = queryParseContext;
+        }
+    }
 
     public static final AggregatorFactories EMPTY = new AggregatorFactories(null, new AggregatorFactory<?>[0],
             new ArrayList<PipelineAggregationBuilder>());
@@ -255,7 +377,7 @@ public class AggregatorFactories {
                             } else {
                                 // Check the non-pipeline sub-aggregator
                                 // factories
-                                AggregationBuilder[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
+                                List<AggregationBuilder> subBuilders = aggBuilder.factoriesBuilder.aggregationBuilders;
                                 boolean foundSubBuilder = false;
                                 for (AggregationBuilder subBuilder : subBuilders) {
                                     if (aggName.equals(subBuilder.name)) {
@@ -297,12 +419,12 @@ public class AggregatorFactories {
             }
         }
 
-        AggregationBuilder[] getAggregatorFactories() {
-            return this.aggregationBuilders.toArray(new AggregationBuilder[this.aggregationBuilders.size()]);
+        public List<AggregationBuilder> getAggregatorFactories() {
+            return Collections.unmodifiableList(aggregationBuilders);
         }
 
-        List<PipelineAggregationBuilder> getPipelineAggregatorFactories() {
-            return this.pipelineAggregatorBuilders;
+        public List<PipelineAggregationBuilder> getPipelineAggregatorFactories() {
+            return Collections.unmodifiableList(pipelineAggregatorBuilders);
         }
 
         public int count() {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 151fc57dded..14b66c475ec 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
@@ -163,7 +162,6 @@ public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
     }
 
     protected final String name;
-    protected final Type type;
     protected final AggregatorFactory<?> parent;
     protected final AggregatorFactories factories;
     protected final Map<String, Object> metaData;
@@ -174,15 +172,12 @@ public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
      *
      * @param name
      *            The aggregation name
-     * @param type
-     *            The aggregation type
      * @throws IOException
      *             if an error occurs creating the factory
      */
-    public AggregatorFactory(String name, Type type, SearchContext context, AggregatorFactory<?> parent,
+    public AggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         this.name = name;
-        this.type = type;
         this.context = context;
         this.parent = parent;
         this.factories = subFactoriesBuilder.build(context, this);
@@ -226,10 +221,6 @@ public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
         return createInternal(parent, collectsFromSingleBucket, this.factories.createPipelineAggregators(), this.metaData);
     }
 
-    public String getType() {
-        return type.name();
-    }
-
     public AggregatorFactory<?> getParent() {
         return parent;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
deleted file mode 100644
index df64d440dc0..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A registry for all the aggregator parser, also servers as the main parser for the aggregations module
- */
-public class AggregatorParsers {
-    public static final Pattern VALID_AGG_NAME = Pattern.compile("[^\\[\\]>]+");
-
-    private final ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry;
-    private final ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry;
-
-    public AggregatorParsers(ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry,
-            ParseFieldRegistry<PipelineAggregator.Parser> pipelineAggregationParserRegistry) {
-        this.aggregationParserRegistry = aggregationParserRegistry;
-        this.pipelineAggregationParserRegistry = pipelineAggregationParserRegistry;
-    }
-
-    /**
-     * Returns the parser that is registered under the given aggregation type.
-     *
-     * @param type The aggregation type
-     * @param parseFieldMatcher used for making error messages.
-     * @return The parser associated with the given aggregation type or null if it wasn't found.
-     */
-    public Aggregator.Parser parser(String type, ParseFieldMatcher parseFieldMatcher) {
-        return aggregationParserRegistry.lookupReturningNullIfNotFound(type, parseFieldMatcher);
-    }
-
-    /**
-     * Returns the parser that is registered under the given pipeline aggregator type.
-     *
-     * @param type The pipeline aggregator type
-     * @param parseFieldMatcher used for making error messages.
-     * @return The parser associated with the given pipeline aggregator type or null if it wasn't found.
-     */
-    public PipelineAggregator.Parser pipelineParser(String type, ParseFieldMatcher parseFieldMatcher) {
-        return pipelineAggregationParserRegistry.lookupReturningNullIfNotFound(type, parseFieldMatcher);
-    }
-
-    /**
-     * Parses the aggregation request recursively generating aggregator factories in turn.
-     *
-     * @param parseContext   The parse context.
-     *
-     * @return          The parsed aggregator factories.
-     *
-     * @throws IOException When parsing fails for unknown reasons.
-     */
-    public AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext) throws IOException {
-        return parseAggregators(parseContext, 0);
-    }
-
-    private AggregatorFactories.Builder parseAggregators(QueryParseContext parseContext, int level) throws IOException {
-        Matcher validAggMatcher = VALID_AGG_NAME.matcher("");
-        AggregatorFactories.Builder factories = new AggregatorFactories.Builder();
-
-        XContentParser.Token token = null;
-        XContentParser parser = parseContext.parser();
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token != XContentParser.Token.FIELD_NAME) {
-                throw new ParsingException(parser.getTokenLocation(),
-                        "Unexpected token " + token + " in [aggs]: aggregations definitions must start with the name of the aggregation.");
-            }
-            final String aggregationName = parser.currentName();
-            if (!validAggMatcher.reset(aggregationName).matches()) {
-                throw new ParsingException(parser.getTokenLocation(), "Invalid aggregation name [" + aggregationName
-                        + "]. Aggregation names must be alpha-numeric and can only contain '_' and '-'");
-            }
-
-            token = parser.nextToken();
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ParsingException(parser.getTokenLocation(), "Aggregation definition for [" + aggregationName + " starts with a ["
-                        + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
-            }
-
-            AggregationBuilder aggFactory = null;
-            PipelineAggregationBuilder pipelineAggregatorFactory = null;
-            AggregatorFactories.Builder subFactories = null;
-
-            Map<String, Object> metaData = null;
-
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token != XContentParser.Token.FIELD_NAME) {
-                    throw new ParsingException(
-                            parser.getTokenLocation(), "Expected [" + XContentParser.Token.FIELD_NAME + "] under a ["
-                            + XContentParser.Token.START_OBJECT + "], but got a [" + token + "] in [" + aggregationName + "]",
-                            parser.getTokenLocation());
-                }
-                final String fieldName = parser.currentName();
-
-                token = parser.nextToken();
-                if (token == XContentParser.Token.START_OBJECT) {
-                    switch (fieldName) {
-                    case "meta":
-                        metaData = parser.map();
-                        break;
-                    case "aggregations":
-                    case "aggs":
-                        if (subFactories != null) {
-                            throw new ParsingException(parser.getTokenLocation(),
-                                    "Found two sub aggregation definitions under [" + aggregationName + "]");
-                        }
-                        subFactories = parseAggregators(parseContext, level + 1);
-                        break;
-                    default:
-                        if (aggFactory != null) {
-                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
-                                    + aggregationName + "]: [" + aggFactory.type + "] and [" + fieldName + "]");
-                        }
-                        if (pipelineAggregatorFactory != null) {
-                            throw new ParsingException(parser.getTokenLocation(), "Found two aggregation type definitions in ["
-                                    + aggregationName + "]: [" + pipelineAggregatorFactory + "] and [" + fieldName + "]");
-                        }
-
-                        Aggregator.Parser aggregatorParser = parser(fieldName, parseContext.getParseFieldMatcher());
-                        if (aggregatorParser == null) {
-                            PipelineAggregator.Parser pipelineAggregatorParser = pipelineParser(fieldName,
-                                    parseContext.getParseFieldMatcher());
-                            if (pipelineAggregatorParser == null) {
-                                throw new ParsingException(parser.getTokenLocation(),
-                                        "Could not find aggregator type [" + fieldName + "] in [" + aggregationName + "]");
-                            } else {
-                                pipelineAggregatorFactory = pipelineAggregatorParser.parse(aggregationName, parseContext);
-                            }
-                        } else {
-                            aggFactory = aggregatorParser.parse(aggregationName, parseContext);
-                        }
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.START_OBJECT + "] under ["
-                            + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]");
-                }
-            }
-
-            if (aggFactory == null && pipelineAggregatorFactory == null) {
-                throw new ParsingException(parser.getTokenLocation(), "Missing definition for aggregation [" + aggregationName + "]",
-                        parser.getTokenLocation());
-            } else if (aggFactory != null) {
-                assert pipelineAggregatorFactory == null;
-            if (metaData != null) {
-                    aggFactory.setMetaData(metaData);
-            }
-
-            if (subFactories != null) {
-                    aggFactory.subAggregations(subFactories);
-            }
-
-                factories.addAggregator(aggFactory);
-            } else {
-                assert pipelineAggregatorFactory != null;
-                if (subFactories != null) {
-                    throw new ParsingException(parser.getTokenLocation(),
-                            "Aggregation [" + aggregationName + "] cannot define sub-aggregations",
-                            parser.getTokenLocation());
-                }
-                if (metaData != null) {
-                    pipelineAggregatorFactory.setMetaData(metaData);
-                }
-                factories.addPipelineAggregator(pipelineAggregatorFactory);
-            }
-        }
-
-        return factories;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchExtParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java
similarity index 51%
rename from core/src/main/java/org/elasticsearch/search/SearchExtParser.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java
index a2fe4cfe0cb..8f076cfe456 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchExtParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/BaseAggregationBuilder.java
@@ -17,27 +17,30 @@
  * under the License.
  */
 
-package org.elasticsearch.search;
+package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 
-import java.io.IOException;
+import java.util.Map;
 
 /**
- * Defines a parser that is able to parse {@link org.elasticsearch.search.SearchExtBuilder}s
- * from {@link org.elasticsearch.common.xcontent.XContent}.
- *
- * Registration happens through {@link org.elasticsearch.plugins.SearchPlugin#getSearchExts()}, which also needs a {@link SearchExtBuilder}
- * implementation which is the object that this parser returns when reading an incoming request form the REST layer.
- *
- * @see SearchExtBuilder
- * @see org.elasticsearch.plugins.SearchPlugin.SearchExtSpec
+ * Interface shared by {@link AggregationBuilder} and {@link PipelineAggregationBuilder} so they can conveniently share the same namespace
+ * for {@link XContentParser#namedObject(Class, String, Object)}.
  */
-@FunctionalInterface
-public interface SearchExtParser<T extends SearchExtBuilder> {
+public interface BaseAggregationBuilder {
+    /**
+     * The name of the type of aggregation built by this builder. 
+     */
+    String getType();
 
     /**
-     * Parses the supported element placed within the ext section of a search request
+     * Set the aggregation's metadata. Returns {@code this} for chaining.
      */
-    T fromXContent(XContentParser parser) throws IOException;
+    BaseAggregationBuilder setMetaData(Map<String, Object> metaData);
+
+    /**
+     * Set the sub aggregations if this aggregation supports sub aggregations. Returns {@code this} for chaining.
+     */
+    BaseAggregationBuilder subAggregations(Builder subFactories);
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index df25f0e2635..752b3497a92 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -32,38 +32,12 @@ import org.elasticsearch.search.aggregations.support.AggregationPath;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
  */
 public abstract class InternalAggregation implements Aggregation, ToXContent, NamedWriteable {
-    /**
-     * The aggregation type that holds all the string types that are associated with an aggregation:
-     * <ul>
-     *     <li>name - used as the parser type</li>
-     * </ul>
-     */
-    public static class Type {
-        private final String name;
-
-        public Type(String name) {
-            this.name = name;
-        }
-
-        /**
-         * @return The name of the type of aggregation.  This is the key for parsing the aggregation from XContent and is the name of the
-         * aggregation's builder when serialized.
-         */
-        public String name() {
-            return name;
-        }
-
-        @Override
-        public String toString() {
-            return name;
-        }
-    }
-
     public static class ReduceContext {
 
         private final BigArrays bigArrays;
@@ -189,6 +163,47 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, Na
 
     public abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, metaData, pipelineAggregators, doHashCode());
+    }
+
+    /**
+     * Opportunity for subclasses to the {@link #hashCode()} for this
+     * class.
+     **/
+    protected int doHashCode() {
+        return System.identityHashCode(this);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        InternalAggregation other = (InternalAggregation) obj;
+        return Objects.equals(name, other.name) &&
+                Objects.equals(pipelineAggregators, other.pipelineAggregators) &&
+                Objects.equals(metaData, other.metaData) &&
+                doEquals(obj);
+    }
+
+    // norelease: make this abstract when all InternalAggregations implement this method
+    /**
+     * Opportunity for subclasses to add criteria to the {@link #equals(Object)}
+     * method for this class.
+     *
+     * This method can safely cast <code>obj</code> to the subclass since the
+     * {@link #equals(Object)} method checks that <code>obj</code> is the same
+     * class as <code>this</code>
+     */
+    protected boolean doEquals(Object obj) {
+        return this == obj;
+    }
+
     /**
      * Common xcontent fields that are shared among addAggregation
      */
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index f9462325471..8f965d1d87e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -32,7 +32,7 @@ import java.util.Map;
  * specific type.
  */
 public abstract class PipelineAggregationBuilder extends ToXContentToBytes
-        implements NamedWriteable {
+        implements NamedWriteable, BaseAggregationBuilder {
 
     protected final String name;
     protected final String[] bucketsPaths;
@@ -79,6 +79,11 @@ public abstract class PipelineAggregationBuilder extends ToXContentToBytes
     protected abstract PipelineAggregator create() throws IOException;
 
     /** Associate metadata with this {@link PipelineAggregationBuilder}. */
+    @Override
     public abstract PipelineAggregationBuilder setMetaData(Map<String, Object> metaData);
 
+    @Override
+    public PipelineAggregationBuilder subAggregations(Builder subFactories) {
+        throw new IllegalArgumentException("Aggregation [" + name + "] cannot define sub-aggregations");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 7348046494f..61d85b80ad9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -49,7 +49,7 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
         final PackedLongValues docDeltas;
         final PackedLongValues buckets;
 
-        public Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
+        Entry(LeafReaderContext context, PackedLongValues docDeltas, PackedLongValues buckets) {
             this.context = context;
             this.docDeltas = docDeltas;
             this.buckets = buckets;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
index 90316c1a001..8d699b7c438 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
@@ -163,7 +163,7 @@ public class BestDocsDeferringCollector extends DeferringBucketCollector impleme
         private long parentBucket;
         private int matchedDocs;
 
-        public PerParentBucketSamples(long parentBucket, Scorer scorer, LeafReaderContext readerContext) {
+        PerParentBucketSamples(long parentBucket, Scorer scorer, LeafReaderContext readerContext) {
             try {
                 this.parentBucket = parentBucket;
                 tdc = createTopDocsCollector(shardSize);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
index 2d8d26dd35c..54b33bb76a0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
@@ -64,7 +64,7 @@ public interface MultiBucketsAggregation extends Aggregation {
 
         Object getProperty(String containingAggName, List<String> path);
 
-        static class SubAggregationComparator<B extends Bucket> implements java.util.Comparator<B> {
+        class SubAggregationComparator<B extends Bucket> implements java.util.Comparator<B> {
 
             private final AggregationPath path;
             private final boolean asc;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java
new file mode 100644
index 00000000000..365fbb5547b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrix.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+
+import java.util.List;
+
+/**
+ * A multi bucket aggregation where the buckets are defined by a set of filters
+ * (a bucket is produced per filter plus a bucket for each non-empty filter 
+ * intersection so A, B and A&amp;B).
+ */
+public interface AdjacencyMatrix extends MultiBucketsAggregation {
+
+    /**
+     * A bucket associated with a specific filter or pair (identified by its
+     * key)
+     */
+    interface Bucket extends MultiBucketsAggregation.Bucket {
+    }
+
+    /**
+     * The buckets created by this aggregation.
+     */
+    @Override
+    List<? extends Bucket> getBuckets();
+
+    Bucket getBucketByKey(String key);
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
new file mode 100644
index 00000000000..526db6a79b7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregationBuilder.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.query.QueryPhaseExecutionException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+public class AdjacencyMatrixAggregationBuilder extends AbstractAggregationBuilder<AdjacencyMatrixAggregationBuilder> {
+    public static final String NAME = "adjacency_matrix";
+
+    private static final String DEFAULT_SEPARATOR = "&";
+
+    private static final ParseField SEPARATOR_FIELD = new ParseField("separator");
+    private static final ParseField FILTERS_FIELD = new ParseField("filters");
+    private List<KeyedFilter> filters;
+    private String separator = DEFAULT_SEPARATOR;
+
+    public static Aggregator.Parser getParser() {
+        ObjectParser<AdjacencyMatrixAggregationBuilder, QueryParseContext> parser = new ObjectParser<>(
+                AdjacencyMatrixAggregationBuilder.NAME);
+        parser.declareString(AdjacencyMatrixAggregationBuilder::separator, SEPARATOR_FIELD);
+        parser.declareNamedObjects(AdjacencyMatrixAggregationBuilder::setFiltersAsList, KeyedFilter.PARSER, FILTERS_FIELD);
+        return new Aggregator.Parser() {
+            @Override
+            public AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+                AdjacencyMatrixAggregationBuilder result = parser.parse(context.parser(),
+                        new AdjacencyMatrixAggregationBuilder(aggregationName), context);
+                result.checkConsistency();
+                return result;
+            }
+        };
+    }
+
+    protected void checkConsistency() {
+        if ((filters == null) || (filters.size() == 0)) {
+            throw new IllegalStateException("[" + name  + "] is missing : " + FILTERS_FIELD.getPreferredName() + " parameter");
+        }        
+    }
+
+
+    protected void setFiltersAsMap(Map<String, QueryBuilder> filters) {
+        // Convert uniquely named objects into internal KeyedFilters
+        this.filters = new ArrayList<>(filters.size());
+        for (Entry<String, QueryBuilder> kv : filters.entrySet()) {
+            this.filters.add(new KeyedFilter(kv.getKey(), kv.getValue()));
+        }
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+
+    protected void setFiltersAsList(List<KeyedFilter> filters) {
+        this.filters = new ArrayList<>(filters);
+        // internally we want to have a fixed order of filters, regardless of
+        // the order of the filters in the request
+        Collections.sort(this.filters, Comparator.comparing(KeyedFilter::key));
+    }
+    
+   
+    /**
+     * @param name
+     *            the name of this aggregation
+     */
+    protected AdjacencyMatrixAggregationBuilder(String name) {
+        super(name);
+    }    
+    
+    
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param filters
+     *            the filters and their keys to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, Map<String, QueryBuilder> filters) {
+        this(name, DEFAULT_SEPARATOR, filters);
+    }
+
+    /**
+     * @param name
+     *            the name of this aggregation
+     * @param separator
+     *            the string used to separate keys in intersections buckets e.g.
+     *            &amp; character for keyed filters A and B would return an
+     *            intersection bucket named A&amp;B
+     * @param filters
+     *            the filters and their key to use with this aggregation.
+     */
+    public AdjacencyMatrixAggregationBuilder(String name, String separator, Map<String, QueryBuilder> filters) {
+        super(name);
+        this.separator = separator;
+        setFiltersAsMap(filters);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public AdjacencyMatrixAggregationBuilder(StreamInput in) throws IOException {
+        super(in);
+        int filtersSize = in.readVInt();
+        separator = in.readString();
+        filters = new ArrayList<>(filtersSize);
+        for (int i = 0; i < filtersSize; i++) {
+            filters.add(new KeyedFilter(in));
+        }
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(filters.size());
+        out.writeString(separator);
+        for (KeyedFilter keyedFilter : filters) {
+            keyedFilter.writeTo(out);
+        }
+    }
+
+    /**
+     * Set the separator used to join pairs of bucket keys
+     */
+    public AdjacencyMatrixAggregationBuilder separator(String separator) {
+        if (separator == null) {
+            throw new IllegalArgumentException("[separator] must not be null: [" + name + "]");
+        }
+        this.separator = separator;
+        return this;
+    }
+
+    /**
+     * Get the separator used to join pairs of bucket keys
+     */
+    public String separator() {
+        return separator;
+    }        
+    
+    /**
+     * Get the filters. This will be an unmodifiable map
+     */
+    public Map<String, QueryBuilder> filters() {
+        Map<String, QueryBuilder>result = new HashMap<>(this.filters.size());
+        for (KeyedFilter keyedFilter : this.filters) {
+            result.put(keyedFilter.key(), keyedFilter.filter());
+        }
+        return result;
+    }    
+    
+
+    @Override
+    protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+            throws IOException {
+        int maxFilters = context.indexShard().indexSettings().getMaxAdjacencyMatrixFilters();
+        if (filters.size() > maxFilters){
+            throw new QueryPhaseExecutionException(context,
+                    "Number of filters is too large, must be less than or equal to: [" + maxFilters + "] but was ["
+                            + filters.size() + "]."
+                            + "This limit can be set by changing the [" + IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey()
+                            + "] index level setting.");
+        }
+
+        List<KeyedFilter> rewrittenFilters = new ArrayList<>();
+        for (KeyedFilter kf : filters) {
+            rewrittenFilters.add(new KeyedFilter(kf.key(), QueryBuilder.rewriteQuery(kf.filter(), context.getQueryShardContext())));
+        }
+
+        return new AdjacencyMatrixAggregatorFactory(name, rewrittenFilters, separator, context, parent,
+                subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field(SEPARATOR_FIELD.getPreferredName(), separator);
+        builder.startObject(AdjacencyMatrixAggregator.FILTERS_FIELD.getPreferredName());
+        for (KeyedFilter keyedFilter : filters) {
+            builder.field(keyedFilter.key(), keyedFilter.filter());
+        }
+        builder.endObject();
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(filters, separator);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        AdjacencyMatrixAggregationBuilder other = (AdjacencyMatrixAggregationBuilder) obj;
+        return Objects.equals(filters, other.filters) && Objects.equals(separator, other.separator);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
new file mode 100644
index 00000000000..8f8ed143ca3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Aggregation for adjacency matrices.
+ * 
+ * NOTE! This is an experimental class.
+ * 
+ * TODO the aggregation produces a sparse response but in the
+ * computation it uses a non-sparse structure (an array of Bits
+ * objects). This could be changed to a sparse structure in future.
+ * 
+ */
+public class AdjacencyMatrixAggregator extends BucketsAggregator {
+
+    public static final ParseField FILTERS_FIELD = new ParseField("filters");
+
+    protected static class KeyedFilter implements Writeable, ToXContent {
+        private final String key;
+        private final QueryBuilder filter;
+        
+        public static final NamedObjectParser<KeyedFilter, QueryParseContext> PARSER = 
+                (XContentParser p, QueryParseContext c, String name) -> 
+                     new KeyedFilter(name, c.parseInnerQueryBuilder());
+        
+
+        public KeyedFilter(String key, QueryBuilder filter) {
+            if (key == null) {
+                throw new IllegalArgumentException("[key] must not be null");
+            }
+            if (filter == null) {
+                throw new IllegalArgumentException("[filter] must not be null");
+            }
+            this.key = key;
+            this.filter = filter;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public KeyedFilter(StreamInput in) throws IOException {
+            key = in.readString();
+            filter = in.readNamedWriteable(QueryBuilder.class);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(key);
+            out.writeNamedWriteable(filter);
+        }
+
+        public String key() {
+            return key;
+        }
+
+        public QueryBuilder filter() {
+            return filter;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(key, filter);
+            return builder;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(key, filter);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null) {
+                return false;
+            }
+            if (getClass() != obj.getClass()) {
+                return false;
+            }
+            KeyedFilter other = (KeyedFilter) obj;
+            return Objects.equals(key, other.key) && Objects.equals(filter, other.filter);
+        }
+    }
+
+    private final String[] keys;
+    private Weight[] filters;
+    private final int totalNumKeys;
+    private final int totalNumIntersections;
+    private final String separator;
+
+    public AdjacencyMatrixAggregator(String name, AggregatorFactories factories, String separator, String[] keys, 
+            Weight[] filters, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, factories, context, parent, pipelineAggregators, metaData);
+        this.separator = separator;
+        this.keys = keys;
+        this.filters = filters;
+        this.totalNumIntersections = ((keys.length * keys.length) - keys.length) / 2;
+        this.totalNumKeys = keys.length + totalNumIntersections;
+    }
+
+    private static class BitsIntersector implements Bits {
+        Bits a;
+        Bits b;
+
+        BitsIntersector(Bits a, Bits b) {
+            super();
+            this.a = a;
+            this.b = b;
+        }
+
+        @Override
+        public boolean get(int index) {
+            return a.get(index) && b.get(index);
+        }
+
+        @Override
+        public int length() {
+            return Math.min(a.length(), b.length());
+        }
+
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) throws IOException {
+        // no need to provide deleted docs to the filter
+        final Bits[] bits = new Bits[filters.length + totalNumIntersections];
+        for (int i = 0; i < filters.length; ++i) {
+            bits[i] = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), filters[i].scorer(ctx));
+        }
+        // Add extra Bits for intersections
+        int pos = filters.length;
+        for (int i = 0; i < filters.length; i++) {
+            for (int j = i + 1; j < filters.length; j++) {
+                bits[pos++] = new BitsIntersector(bits[i], bits[j]);
+            }
+        }
+        assert pos == bits.length;
+        return new LeafBucketCollectorBase(sub, null) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                for (int i = 0; i < bits.length; i++) {
+                    if (bits[i].get(doc)) {
+                        collectBucket(sub, doc, bucketOrd(bucket, i));
+                    }
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException {
+
+        // Buckets are ordered into groups - [keyed filters] [key1&key2 intersects]
+
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(filters.length);
+        for (int i = 0; i < keys.length; i++) {
+            long bucketOrd = bucketOrd(owningBucketOrdinal, i);
+            int docCount = bucketDocCount(bucketOrd);
+            // Empty buckets are not returned because this aggregation will commonly be used under a
+            // a date-histogram where we will look for transactions over time and can expect many
+            // empty buckets.
+            if (docCount > 0) {
+                InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(keys[i], 
+                        docCount, bucketAggregations(bucketOrd));
+                buckets.add(bucket);                
+            }
+        }
+        int pos = keys.length;
+        for (int i = 0; i < keys.length; i++) {
+            for (int j = i + 1; j < keys.length; j++) {
+                long bucketOrd = bucketOrd(owningBucketOrdinal, pos);
+                int docCount = bucketDocCount(bucketOrd);
+                // Empty buckets are not returned due to potential for very sparse matrices
+                if (docCount > 0) {
+                    String intersectKey = keys[i] + separator + keys[j];
+                    InternalAdjacencyMatrix.InternalBucket bucket = new InternalAdjacencyMatrix.InternalBucket(intersectKey, 
+                            docCount, bucketAggregations(bucketOrd));
+                    buckets.add(bucket);
+                }
+                pos++;
+            }
+        }
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        List<InternalAdjacencyMatrix.InternalBucket> buckets = new ArrayList<>(0);
+        return new InternalAdjacencyMatrix(name, buckets, pipelineAggregators(), metaData());
+    }
+
+    final long bucketOrd(long owningBucketOrdinal, int filterOrd) {
+        return owningBucketOrdinal * totalNumKeys + filterOrd;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
new file mode 100644
index 00000000000..6df88379d4e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregatorFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregator.KeyedFilter;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class AdjacencyMatrixAggregatorFactory extends AggregatorFactory<AdjacencyMatrixAggregatorFactory> {
+
+    private final String[] keys;
+    private final Weight[] weights;
+    private final String separator;
+
+    public AdjacencyMatrixAggregatorFactory(String name, List<KeyedFilter> filters, String separator, 
+            SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, 
+            Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, subFactories, metaData);
+        IndexSearcher contextSearcher = context.searcher();
+        this.separator = separator;
+        weights = new Weight[filters.size()];
+        keys = new String[filters.size()];
+        for (int i = 0; i < filters.size(); ++i) {
+            KeyedFilter keyedFilter = filters.get(i);
+            this.keys[i] = keyedFilter.key();
+            Query filter = keyedFilter.filter().toFilter(context.getQueryShardContext());
+            this.weights[i] = contextSearcher.createNormalizedWeight(filter, false);
+        }
+    }
+
+    @Override
+    public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new AdjacencyMatrixAggregator(name, factories, separator, keys, weights, context, parent,
+                pipelineAggregators, metaData);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
new file mode 100644
index 00000000000..3a7dc284ab9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.adjacency;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InternalAdjacencyMatrix 
+    extends InternalMultiBucketAggregation<InternalAdjacencyMatrix,InternalAdjacencyMatrix.InternalBucket> 
+    implements AdjacencyMatrix {
+    public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements AdjacencyMatrix.Bucket {
+
+        private final String key;
+        private long docCount;
+        InternalAggregations aggregations;
+
+        public InternalBucket(String key, long docCount, InternalAggregations aggregations) {
+            this.key = key;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public InternalBucket(StreamInput in) throws IOException {
+            key = in.readOptionalString();
+            docCount = in.readVLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeVLong(docCount);
+            aggregations.writeTo(out);
+        }
+
+        @Override
+        public String getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        InternalBucket reduce(List<InternalBucket> buckets, ReduceContext context) {
+            InternalBucket reduced = null;
+            List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                if (reduced == null) {
+                    reduced = new InternalBucket(bucket.key, bucket.docCount, bucket.aggregations);
+                } else {
+                    reduced.docCount += bucket.docCount;
+                }
+                aggregationsList.add(bucket.aggregations);
+            }
+            reduced.aggregations = InternalAggregations.reduce(aggregationsList, context);
+            return reduced;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field(CommonFields.KEY, key);
+            builder.field(CommonFields.DOC_COUNT, docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+    }
+
+    private final List<InternalBucket> buckets;
+    private Map<String, InternalBucket> bucketMap;
+
+    public InternalAdjacencyMatrix(String name, List<InternalBucket> buckets, 
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.buckets = buckets;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public InternalAdjacencyMatrix(StreamInput in) throws IOException {
+        super(in);
+        int size = in.readVInt();
+        List<InternalBucket> buckets = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            buckets.add(new InternalBucket(in));
+        }
+        this.buckets = buckets;
+        this.bucketMap = null;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(buckets.size());
+        for (InternalBucket bucket : buckets) {
+            bucket.writeTo(out);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return AdjacencyMatrixAggregationBuilder.NAME;
+    }
+
+    @Override
+    public InternalAdjacencyMatrix create(List<InternalBucket> buckets) {
+        return new InternalAdjacencyMatrix(this.name, buckets, this.pipelineAggregators(), this.metaData);
+    }
+
+    @Override
+    public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) {
+        return new InternalBucket(prototype.key, prototype.docCount, aggregations);
+    }
+
+    @Override
+    public List<InternalBucket> getBuckets() {
+        return buckets;
+    }
+
+    @Override
+    public InternalBucket getBucketByKey(String key) {
+        if (bucketMap == null) {
+            bucketMap = new HashMap<>(buckets.size());
+            for (InternalBucket bucket : buckets) {
+                bucketMap.put(bucket.getKey(), bucket);
+            }
+        }
+        return bucketMap.get(key);
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        Map<String, List<InternalBucket>> bucketsMap = new HashMap<>();
+        for (InternalAggregation aggregation : aggregations) {
+            InternalAdjacencyMatrix filters = (InternalAdjacencyMatrix) aggregation;
+            for (InternalBucket bucket : filters.buckets) {
+                List<InternalBucket> sameRangeList = bucketsMap.get(bucket.key);
+                if(sameRangeList == null){
+                    sameRangeList = new ArrayList<>(aggregations.size());
+                    bucketsMap.put(bucket.key, sameRangeList);
+                }
+                sameRangeList.add(bucket);
+            }
+        }
+
+        ArrayList<InternalBucket> reducedBuckets = new ArrayList<>(bucketsMap.size());
+        for (List<InternalBucket> sameRangeList : bucketsMap.values()) {
+            InternalBucket reducedBucket = sameRangeList.get(0).reduce(sameRangeList, reduceContext);
+            if(reducedBucket.docCount >= 1){                
+                reducedBuckets.add(reducedBucket);
+            }
+        }
+        Collections.sort(reducedBuckets, Comparator.comparing(InternalBucket::getKey));
+        
+        InternalAdjacencyMatrix reduced = new InternalAdjacencyMatrix(name, reducedBuckets, pipelineAggregators(), 
+                getMetaData());
+
+        return reduced;
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray(CommonFields.BUCKETS);
+        for (InternalBucket bucket : buckets) {
+            bucket.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
index 48025e13aa1..ddd252a6f53 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
@@ -31,7 +31,6 @@ import org.elasticsearch.index.mapper.ParentFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.FieldContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
@@ -46,7 +45,6 @@ import java.util.Objects;
 
 public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<ParentChild, ChildrenAggregationBuilder> {
     public static final String NAME = "children";
-    private static final Type TYPE = new Type(NAME);
 
     private String parentType;
     private final String childType;
@@ -60,7 +58,7 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
      *            the type of children documents
      */
     public ChildrenAggregationBuilder(String name, String childType) {
-        super(name, TYPE, ValuesSourceType.BYTES, ValueType.STRING);
+        super(name, ValuesSourceType.BYTES, ValueType.STRING);
         if (childType == null) {
             throw new IllegalArgumentException("[childType] must not be null: [" + name + "]");
         }
@@ -71,7 +69,7 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
      * Read from a stream.
      */
     public ChildrenAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.BYTES, ValueType.STRING);
+        super(in, ValuesSourceType.BYTES, ValueType.STRING);
         childType = in.readString();
     }
 
@@ -83,7 +81,7 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
     @Override
     protected ValuesSourceAggregatorFactory<ParentChild, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ParentChild> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new ChildrenAggregatorFactory(name, type, config, parentType, childFilter, parentFilter, context, parent,
+        return new ChildrenAggregatorFactory(name, config, parentType, childFilter, parentFilter, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -163,7 +161,7 @@ public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<P
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java
index 0682fcaf415..b0a4c64305a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorFactory.java
@@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -44,10 +43,10 @@ public class ChildrenAggregatorFactory
     private final Query parentFilter;
     private final Query childFilter;
 
-    public ChildrenAggregatorFactory(String name, Type type, ValuesSourceConfig<ParentChild> config, String parentType, Query childFilter,
+    public ChildrenAggregatorFactory(String name, ValuesSourceConfig<ParentChild> config, String parentType, Query childFilter,
             Query parentFilter, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.parentType = parentType;
         this.childFilter = childFilter;
         this.parentFilter = parentFilter;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index ccf1883ecf4..9fb8b368e9d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -27,7 +27,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -35,7 +34,6 @@ import java.util.Objects;
 
 public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterAggregationBuilder> {
     public static final String NAME = "filter";
-    private static final Type TYPE = new Type(NAME);
 
     private final QueryBuilder filter;
 
@@ -48,7 +46,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
      *            {@link Filter} aggregation.
      */
     public FilterAggregationBuilder(String name, QueryBuilder filter) {
-        super(name, TYPE);
+        super(name);
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
         }
@@ -59,7 +57,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
      * Read from a stream.
      */
     public FilterAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         filter = in.readNamedWriteable(QueryBuilder.class);
     }
 
@@ -73,7 +71,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
             AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
         // TODO this sucks we need a rewrite phase for aggregations too
         final QueryBuilder rewrittenFilter = QueryBuilder.rewriteQuery(filter, context.getQueryShardContext());
-        return new FilterAggregatorFactory(name, type, rewrittenFilter, context, parent, subFactoriesBuilder, metaData);
+        return new FilterAggregatorFactory(name, rewrittenFilter, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -101,7 +99,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
index 01ae8f6d22e..baeaea434bd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
@@ -26,7 +26,6 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -38,9 +37,9 @@ public class FilterAggregatorFactory extends AggregatorFactory<FilterAggregatorF
 
     private final Weight weight;
 
-    public FilterAggregatorFactory(String name, Type type, QueryBuilder filterBuilder, SearchContext context,
+    public FilterAggregatorFactory(String name, QueryBuilder filterBuilder, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
+        super(name, context, parent, subFactoriesBuilder, metaData);
         IndexSearcher contextSearcher = context.searcher();
         Query filter = filterBuilder.toQuery(context.getQueryShardContext());
         weight = contextSearcher.createNormalizedWeight(filter, false);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 10bbc03dd46..f34df7368ac 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -30,7 +30,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -43,7 +42,6 @@ import java.util.Objects;
 
 public class FiltersAggregationBuilder extends AbstractAggregationBuilder<FiltersAggregationBuilder> {
     public static final String NAME = "filters";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ParseField FILTERS_FIELD = new ParseField("filters");
     private static final ParseField OTHER_BUCKET_FIELD = new ParseField("other_bucket");
@@ -65,7 +63,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
     }
 
     private FiltersAggregationBuilder(String name, List<KeyedFilter> filters) {
-        super(name, TYPE);
+        super(name);
         // internally we want to have a fixed order of filters, regardless of the order of the filters in the request
         this.filters = new ArrayList<>(filters);
         Collections.sort(this.filters, (KeyedFilter kf1, KeyedFilter kf2) -> kf1.key().compareTo(kf2.key()));
@@ -79,7 +77,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
      *            the filters to use with this aggregation
      */
     public FiltersAggregationBuilder(String name, QueryBuilder... filters) {
-        super(name, TYPE);
+        super(name);
         List<KeyedFilter> keyedFilters = new ArrayList<>(filters.length);
         for (int i = 0; i < filters.length; i++) {
             keyedFilters.add(new KeyedFilter(String.valueOf(i), filters[i]));
@@ -92,7 +90,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
      * Read from a stream.
      */
     public FiltersAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         keyed = in.readBoolean();
         int filtersSize = in.readVInt();
         filters = new ArrayList<>(filtersSize);
@@ -176,7 +174,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
             rewrittenFilters.add(new KeyedFilter(kf.key(), QueryBuilder.rewriteQuery(kf.filter(), 
                     context.getQueryShardContext())));
         }
-        return new FiltersAggregatorFactory(name, type, rewrittenFilters, keyed, otherBucket, otherBucketKey, context, parent,
+        return new FiltersAggregatorFactory(name, rewrittenFilters, keyed, otherBucket, otherBucketKey, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -217,21 +215,21 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-                if (context.getParseFieldMatcher().match(currentFieldName, OTHER_BUCKET_FIELD)) {
+                if (OTHER_BUCKET_FIELD.match(currentFieldName)) {
                     otherBucket = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, OTHER_BUCKET_KEY_FIELD)) {
+                if (OTHER_BUCKET_KEY_FIELD.match(currentFieldName)) {
                     otherBucketKey = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
+                if (FILTERS_FIELD.match(currentFieldName)) {
                     keyedFilters = new ArrayList<>();
                     String key = null;
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -247,7 +245,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
                             "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
+                if (FILTERS_FIELD.match(currentFieldName)) {
                     nonKeyedFilters = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         QueryBuilder filter = context.parseInnerQueryBuilder();
@@ -300,7 +298,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
index d8f109231be..2825c49b8b1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorFactory.java
@@ -25,7 +25,6 @@ import org.apache.lucene.search.Weight;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
@@ -42,10 +41,10 @@ public class FiltersAggregatorFactory extends AggregatorFactory<FiltersAggregato
     private final boolean otherBucket;
     private final String otherBucketKey;
 
-    public FiltersAggregatorFactory(String name, Type type, List<KeyedFilter> filters, boolean keyed, boolean otherBucket,
+    public FiltersAggregatorFactory(String name, List<KeyedFilter> filters, boolean keyed, boolean otherBucket,
             String otherBucketKey, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.keyed = keyed;
         this.otherBucket = otherBucket;
         this.otherBucketKey = otherBucketKey;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 26e58f51320..602c3a81c66 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -35,7 +35,6 @@ import org.elasticsearch.index.fielddata.SortingNumericDocValues;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -51,7 +50,6 @@ import java.util.Objects;
 
 public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoGridAggregationBuilder> {
     public static final String NAME = "geohash_grid";
-    private static final Type TYPE = new Type(NAME);
     public static final int DEFAULT_PRECISION = 5;
     public static final int DEFAULT_MAX_NUM_CELLS = 10000;
 
@@ -73,14 +71,14 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     private int shardSize = -1;
 
     public GeoGridAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoGridAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         precision = in.readVInt();
         requiredSize = in.readVInt();
         shardSize = in.readVInt();
@@ -150,7 +148,7 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
         if (shardSize < requiredSize) {
             shardSize = requiredSize;
         }
-        return new GeoHashGridAggregatorFactory(name, type, config, precision, requiredSize, shardSize, context, parent,
+        return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -185,7 +183,7 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -214,7 +212,7 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
         private final ValuesSource.GeoPoint valuesSource;
         private final int precision;
 
-        public CellIdSource(ValuesSource.GeoPoint valuesSource, int precision) {
+        CellIdSource(ValuesSource.GeoPoint valuesSource, int precision) {
             this.valuesSource = valuesSource;
             //different GeoPoints could map to the same or different geohash cells.
             this.precision = precision;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
index 7df0a4fee26..a24efe2df9b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
@@ -98,7 +98,7 @@ public class GeoHashGridAggregator extends BucketsAggregator {
 
         long bucketOrd;
 
-        public OrdinalBucket() {
+        OrdinalBucket() {
             super(0, 0, (InternalAggregations) null);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
index 548b0e2509e..4f87b82f788 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
@@ -23,7 +23,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder.CellIdSource;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -44,10 +43,10 @@ public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory<
     private final int requiredSize;
     private final int shardSize;
 
-    public GeoHashGridAggregatorFactory(String name, Type type, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
+    public GeoHashGridAggregatorFactory(String name, ValuesSourceConfig<GeoPoint> config, int precision, int requiredSize,
             int shardSize, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precision = precision;
         this.requiredSize = requiredSize;
         this.shardSize = shardSize;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
index eaf00755f76..085f18c0e1e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
@@ -52,7 +52,7 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation<Internal
         protected long docCount;
         protected InternalAggregations aggregations;
 
-        public Bucket(long geohashAsLong, long docCount, InternalAggregations aggregations) {
+        Bucket(long geohashAsLong, long docCount, InternalAggregations aggregations) {
             this.docCount = docCount;
             this.aggregations = aggregations;
             this.geohashAsLong = geohashAsLong;
@@ -218,7 +218,7 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation<Internal
 
     static class BucketPriorityQueue extends PriorityQueue<Bucket> {
 
-        public BucketPriorityQueue(int size) {
+        BucketPriorityQueue(int size) {
             super(size);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index c9941984d6e..2363ed498a9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -26,24 +26,22 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
 public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalAggregationBuilder> {
     public static final String NAME = "global";
-    private static final Type TYPE = new Type(NAME);
 
     public GlobalAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public GlobalAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
     }
 
     @Override
@@ -54,7 +52,7 @@ public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalA
     @Override
     protected AggregatorFactory<?> doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        return new GlobalAggregatorFactory(name, type, context, parent, subFactoriesBuilder, metaData);
+        return new GlobalAggregatorFactory(name, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -80,7 +78,7 @@ public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalA
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
index c511cbf3165..e46b7623e34 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregator.java
@@ -33,8 +33,8 @@ import java.util.Map;
 
 public class GlobalAggregator extends SingleBucketAggregator {
 
-    public GlobalAggregator(String name, AggregatorFactories subFactories, SearchContext aggregationContext, List<PipelineAggregator> pipelineAggregators,
-            Map<String, Object> metaData) throws IOException {
+    public GlobalAggregator(String name, AggregatorFactories subFactories, SearchContext aggregationContext,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, subFactories, aggregationContext, null, pipelineAggregators, metaData);
     }
 
@@ -59,6 +59,7 @@ public class GlobalAggregator extends SingleBucketAggregator {
 
     @Override
     public InternalAggregation buildEmptyAggregation() {
-        throw new UnsupportedOperationException("global aggregations cannot serve as sub-aggregations, hence should never be called on #buildEmptyAggregations");
+        throw new UnsupportedOperationException(
+                "global aggregations cannot serve as sub-aggregations, hence should never be called on #buildEmptyAggregations");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
index bbdd280839a..b11ffde6d7b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorFactory.java
@@ -23,7 +23,6 @@ import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -33,9 +32,9 @@ import java.util.Map;
 
 public class GlobalAggregatorFactory extends AggregatorFactory<GlobalAggregatorFactory> {
 
-    public GlobalAggregatorFactory(String name, Type type, SearchContext context, AggregatorFactory<?> parent,
+    public GlobalAggregatorFactory(String name, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
index f278e5e72b1..6ba3b79e968 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
@@ -32,7 +32,8 @@ import java.util.Map;
  * regardless the query.
  */
 public class InternalGlobal extends InternalSingleBucketAggregation implements Global {
-    InternalGlobal(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+    InternalGlobal(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
         super(name, docCount, aggregations, pipelineAggregators, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index 67f662bc334..e805abd86c9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -54,7 +54,7 @@ import static java.util.Collections.unmodifiableMap;
  */
 public class DateHistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, DateHistogramAggregationBuilder> {
-    public static final String NAME = InternalDateHistogram.TYPE.name();
+    public static final String NAME = "date_histogram";
 
     public static final Map<String, DateTimeUnit> DATE_FIELD_UNITS;
 
@@ -110,7 +110,7 @@ public class DateHistogramAggregationBuilder
 
         PARSER.declareLong(DateHistogramAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD);
 
-        PARSER.declareField(DateHistogramAggregationBuilder::extendedBounds, ExtendedBounds.PARSER::apply,
+        PARSER.declareField(DateHistogramAggregationBuilder::extendedBounds, parser -> ExtendedBounds.PARSER.apply(parser, null),
                 ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT);
 
         PARSER.declareField(DateHistogramAggregationBuilder::order, DateHistogramAggregationBuilder::parseOrder,
@@ -131,12 +131,12 @@ public class DateHistogramAggregationBuilder
 
     /** Create a new builder with the given name. */
     public DateHistogramAggregationBuilder(String name) {
-        super(name, InternalDateHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DATE);
+        super(name, ValuesSourceType.NUMERIC, ValueType.DATE);
     }
 
     /** Read from a stream, for internal use only. */
     public DateHistogramAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalDateHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DATE);
+        super(in, ValuesSourceType.NUMERIC, ValueType.DATE);
         if (in.readBoolean()) {
             order = InternalOrder.Streams.readOrder(in);
         }
@@ -315,7 +315,7 @@ public class DateHistogramAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -328,7 +328,7 @@ public class DateHistogramAggregationBuilder
             // parse any string bounds to longs and round
             roundedBounds = this.extendedBounds.parseAndValidate(name, context, config.format()).round(rounding);
         }
-        return new DateHistogramAggregatorFactory(name, type, config, interval, dateHistogramInterval, offset, order, keyed, minDocCount,
+        return new DateHistogramAggregatorFactory(name, config, interval, dateHistogramInterval, offset, order, keyed, minDocCount,
                 rounding, roundedBounds, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 48f14211361..a52f2b2cfb5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -62,7 +62,7 @@ class DateHistogramAggregator extends BucketsAggregator {
     private final LongHash bucketOrds;
     private long offset;
 
-    public DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, long offset, InternalOrder order,
+    DateHistogramAggregator(String name, AggregatorFactories factories, Rounding rounding, long offset, InternalOrder order,
             boolean keyed,
             long minDocCount, @Nullable ExtendedBounds extendedBounds, @Nullable ValuesSource.Numeric valuesSource,
             DocValueFormat formatter, SearchContext aggregationContext,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
index 4cde6f2530d..44bb3e02afe 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java
@@ -23,7 +23,6 @@ import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -47,11 +46,11 @@ public final class DateHistogramAggregatorFactory
     private final ExtendedBounds extendedBounds;
     private Rounding rounding;
 
-    public DateHistogramAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, long interval,
+    public DateHistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, long interval,
             DateHistogramInterval dateHistogramInterval, long offset, InternalOrder order, boolean keyed, long minDocCount,
             Rounding rounding, ExtendedBounds extendedBounds, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.interval = interval;
         this.dateHistogramInterval = dateHistogramInterval;
         this.offset = offset;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
index 85160347612..eeba2994c82 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.rounding.Rounding;
-import org.elasticsearch.common.xcontent.AbstractObjectParser.NoContextParser;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchParseException;
@@ -45,7 +45,7 @@ public class ExtendedBounds implements ToXContent, Writeable {
     static final ParseField MIN_FIELD = new ParseField("min");
     static final ParseField MAX_FIELD = new ParseField("max");
 
-    public static final ConstructingObjectParser<ExtendedBounds, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<ExtendedBounds, Void> PARSER = new ConstructingObjectParser<>(
             "extended_bounds", a -> {
         assert a.length == 2;
         Long min = null;
@@ -73,7 +73,7 @@ public class ExtendedBounds implements ToXContent, Writeable {
         return new ExtendedBounds(min, max, minAsStr, maxAsStr);
     });
     static {
-        NoContextParser<Object> longOrString = p -> {
+        CheckedFunction<XContentParser, Object, IOException> longOrString = p -> {
             if (p.currentToken() == Token.VALUE_NUMBER) {
                 return p.longValue(false);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 873f32c1fde..87c7404c088 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.bucket.histogram;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -49,9 +48,9 @@ import java.util.Objects;
  */
 public class HistogramAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, HistogramAggregationBuilder> {
-    public static final String NAME = InternalHistogram.TYPE.name();
+    public static final String NAME = "histogram";
 
-    private static final ObjectParser<double[], ParseFieldMatcherSupplier> EXTENDED_BOUNDS_PARSER = new ObjectParser<>(
+    private static final ObjectParser<double[], Void> EXTENDED_BOUNDS_PARSER = new ObjectParser<>(
             Histogram.EXTENDED_BOUNDS_FIELD.getPreferredName(),
             () -> new double[]{ Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY });
     static {
@@ -74,7 +73,7 @@ public class HistogramAggregationBuilder
 
         PARSER.declareField((histogram, extendedBounds) -> {
             histogram.extendedBounds(extendedBounds[0], extendedBounds[1]);
-        }, EXTENDED_BOUNDS_PARSER::apply, ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT);
+        }, parser -> EXTENDED_BOUNDS_PARSER.apply(parser, null), ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT);
 
         PARSER.declareField(HistogramAggregationBuilder::order, HistogramAggregationBuilder::parseOrder,
                 Histogram.ORDER_FIELD, ObjectParser.ValueType.OBJECT);
@@ -94,12 +93,12 @@ public class HistogramAggregationBuilder
 
     /** Create a new builder with the given name. */
     public HistogramAggregationBuilder(String name) {
-        super(name, InternalHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
+        super(name, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
     }
 
     /** Read from a stream, for internal use only. */
     public HistogramAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalHistogram.TYPE, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
+        super(in, ValuesSourceType.NUMERIC, ValueType.DOUBLE);
         if (in.readBoolean()) {
             order = InternalOrder.Streams.readOrder(in);
         }
@@ -260,14 +259,14 @@ public class HistogramAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
-        return InternalHistogram.TYPE.name();
+    public String getType() {
+        return NAME;
     }
 
     @Override
     protected ValuesSourceAggregatorFactory<Numeric, ?> innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new HistogramAggregatorFactory(name, type, config, interval, offset, order, keyed, minDocCount, minBound, maxBound,
+        return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound,
                 context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
index d459618d8d0..4b547989d8b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregator.java
@@ -61,7 +61,7 @@ class HistogramAggregator extends BucketsAggregator {
 
     private final LongHash bucketOrds;
 
-    public HistogramAggregator(String name, AggregatorFactories factories, double interval, double offset,
+    HistogramAggregator(String name, AggregatorFactories factories, double interval, double offset,
             InternalOrder order, boolean keyed, long minDocCount, double minBound, double maxBound,
             @Nullable ValuesSource.Numeric valuesSource, DocValueFormat formatter,
             SearchContext context, Aggregator parent,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
index af27b32b206..939210b63a6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.bucket.histogram;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -42,11 +41,11 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact
     private final long minDocCount;
     private final double minBound, maxBound;
 
-    HistogramAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double interval, double offset,
+    HistogramAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double interval, double offset,
             InternalOrder order, boolean keyed, long minDocCount, double minBound, double maxBound,
             SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.interval = interval;
         this.offset = offset;
         this.order = order;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 56d3792e0c6..f24fc5c127e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -43,13 +43,11 @@ import java.util.ListIterator;
 import java.util.Map;
 
 /**
- * Imelementation of {@link Histogram}.
+ * Implementation of {@link Histogram}.
  */
 public final class InternalDateHistogram extends InternalMultiBucketAggregation<InternalDateHistogram, InternalDateHistogram.Bucket>
         implements Histogram, HistogramFactory {
 
-    static final Type TYPE = new Type("date_histogram");
-
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket {
 
         final long key;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index 4dae51533db..e7f5b739216 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -40,13 +40,10 @@ import java.util.ListIterator;
 import java.util.Map;
 
 /**
- * Imelementation of {@link Histogram}.
+ * Implementation of {@link Histogram}.
  */
 public final class InternalHistogram extends InternalMultiBucketAggregation<InternalHistogram, InternalHistogram.Bucket>
         implements Histogram, HistogramFactory {
-
-    static final Type TYPE = new Type("histogram");
-
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket {
 
         final double key;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index 986416922b6..9361acc8fcb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -40,7 +39,6 @@ import java.io.IOException;
 
 public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, MissingAggregationBuilder> {
     public static final String NAME = "missing";
-    public static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<MissingAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -53,14 +51,14 @@ public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     public MissingAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public MissingAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
     }
 
     @Override
@@ -76,7 +74,7 @@ public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     @Override
     protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MissingAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MissingAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -95,7 +93,7 @@ public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
index 5de7fada428..5d05b79cef2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.bucket.missing;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -35,9 +34,9 @@ import java.util.Map;
 
 public class MissingAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, MissingAggregatorFactory> {
 
-    public MissingAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, SearchContext context,
+    public MissingAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index dbb08e792ad..f8f2602a474 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -30,7 +30,6 @@ import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -38,7 +37,6 @@ import java.util.Objects;
 
 public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedAggregationBuilder> {
     public static final String NAME = "nested";
-    private static final Type TYPE = new Type(NAME);
 
     private final String path;
 
@@ -50,7 +48,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
      *            match the path to a nested object in the mappings.
      */
     public NestedAggregationBuilder(String name, String path) {
-        super(name, TYPE);
+        super(name);
         if (path == null) {
             throw new IllegalArgumentException("[path] must not be null: [" + name + "]");
         }
@@ -61,7 +59,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
      * Read from a stream.
      */
     public NestedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         path = in.readString();
     }
 
@@ -83,7 +81,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
         ObjectMapper childObjectMapper = context.getObjectMapper(path);
         if (childObjectMapper == null) {
             // in case the path has been unmapped:
-            return new NestedAggregatorFactory(name, type, null, null, context, parent, subFactoriesBuilder, metaData);
+            return new NestedAggregatorFactory(name, null, null, context, parent, subFactoriesBuilder, metaData);
         }
 
         if (childObjectMapper.nested().isNested() == false) {
@@ -91,7 +89,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
         }
         try {
             ObjectMapper parentObjectMapper = context.getQueryShardContext().nestedScope().nextLevel(childObjectMapper);
-            return new NestedAggregatorFactory(name, type, parentObjectMapper, childObjectMapper, context, parent, subFactoriesBuilder,
+            return new NestedAggregatorFactory(name, parentObjectMapper, childObjectMapper, context, parent, subFactoriesBuilder,
                     metaData);
         } finally {
             context.getQueryShardContext().nestedScope().previousLevel();
@@ -116,7 +114,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, NestedAggregator.PATH_FIELD)) {
+                if (NestedAggregator.PATH_FIELD.match(currentFieldName)) {
                     path = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -148,7 +146,7 @@ public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedA
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
index 554e6c3455b..0ca0ef0a71e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
@@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
@@ -38,10 +37,10 @@ public class NestedAggregatorFactory extends AggregatorFactory<NestedAggregatorF
     private final ObjectMapper parentObjectMapper;
     private final ObjectMapper childObjectMapper;
 
-    public NestedAggregatorFactory(String name, Type type, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
+    public NestedAggregatorFactory(String name, ObjectMapper parentObjectMapper, ObjectMapper childObjectMapper,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories,
                                    Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.parentObjectMapper = parentObjectMapper;
         this.childObjectMapper = childObjectMapper;
     }
@@ -60,7 +59,7 @@ public class NestedAggregatorFactory extends AggregatorFactory<NestedAggregatorF
 
     private static final class Unmapped extends NonCollectingAggregator {
 
-        public Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index 6adadde12ce..30bd72c6c68 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -32,7 +32,6 @@ import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -40,19 +39,18 @@ import java.util.Objects;
 
 public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<ReverseNestedAggregationBuilder> {
     public static final String NAME = "reverse_nested";
-    private static final Type TYPE = new Type(NAME);
 
     private String path;
 
     public ReverseNestedAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public ReverseNestedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         path = in.readOptionalString();
     }
 
@@ -93,7 +91,7 @@ public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<
         if (path != null) {
             parentObjectMapper = context.getObjectMapper(path);
             if (parentObjectMapper == null) {
-                return new ReverseNestedAggregatorFactory(name, type, true, null, context, parent, subFactoriesBuilder, metaData);
+                return new ReverseNestedAggregatorFactory(name, true, null, context, parent, subFactoriesBuilder, metaData);
             }
             if (parentObjectMapper.nested().isNested() == false) {
                 throw new AggregationExecutionException("[reverse_nested] nested path [" + path + "] is not nested");
@@ -103,7 +101,7 @@ public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<
         NestedScope nestedScope = context.getQueryShardContext().nestedScope();
         try {
             nestedScope.nextLevel(parentObjectMapper);
-            return new ReverseNestedAggregatorFactory(name, type, false, parentObjectMapper, context, parent, subFactoriesBuilder,
+            return new ReverseNestedAggregatorFactory(name, false, parentObjectMapper, context, parent, subFactoriesBuilder,
                     metaData);
         } finally {
             nestedScope.previousLevel();
@@ -172,7 +170,7 @@ public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
index 2a691121efa..3ed6f21d99c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
@@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
@@ -38,11 +37,11 @@ public class ReverseNestedAggregatorFactory extends AggregatorFactory<ReverseNes
     private final boolean unmapped;
     private final ObjectMapper parentObjectMapper;
 
-    public ReverseNestedAggregatorFactory(String name, Type type, boolean unmapped, ObjectMapper parentObjectMapper,
+    public ReverseNestedAggregatorFactory(String name, boolean unmapped, ObjectMapper parentObjectMapper,
                                           SearchContext context, AggregatorFactory<?> parent,
                                           AggregatorFactories.Builder subFactories,
                                           Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.unmapped = unmapped;
         this.parentObjectMapper = parentObjectMapper;
     }
@@ -59,7 +58,7 @@ public class ReverseNestedAggregatorFactory extends AggregatorFactory<ReverseNes
 
     private static final class Unmapped extends NonCollectingAggregator {
 
-        public Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+        Unmapped(String name, SearchContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
             super(name, context, parent, pipelineAggregators, metaData);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
index 3de8e73ba93..156adcdc4f3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.bucket.range;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -43,10 +42,10 @@ public class AbstractRangeAggregatorFactory<AF extends AbstractRangeAggregatorFa
     private final R[] ranges;
     private final boolean keyed;
 
-    public AbstractRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, R[] ranges, boolean keyed,
+    public AbstractRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, R[] ranges, boolean keyed,
             InternalRange.Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
         this.keyed = keyed;
         this.rangeFactory = rangeFactory;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
index e7669df2d65..635a0a6015c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
@@ -44,7 +44,7 @@ public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R
     protected boolean keyed = false;
 
     protected AbstractRangeBuilder(String name, InternalRange.Factory<?, ?> rangeFactory) {
-        super(name, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
+        super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.rangeFactory = rangeFactory;
     }
 
@@ -53,7 +53,7 @@ public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R
      */
     protected AbstractRangeBuilder(StreamInput in, InternalRange.Factory<?, ?> rangeFactory, Writeable.Reader<R> rangeReader)
             throws IOException {
-        super(in, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
+        super(in, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.rangeFactory = rangeFactory;
         ranges = in.readList(rangeReader);
         keyed = in.readBoolean();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
index 86573d175c0..a8d74a7d836 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
@@ -18,33 +18,32 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
 public class BinaryRangeAggregatorFactory
         extends ValuesSourceAggregatorFactory<ValuesSource.Bytes, BinaryRangeAggregatorFactory> {
 
     private final List<BinaryRangeAggregator.Range> ranges;
     private final boolean keyed;
 
-    public BinaryRangeAggregatorFactory(String name, Type type,
+    public BinaryRangeAggregatorFactory(String name,
             ValuesSourceConfig<ValuesSource.Bytes> config,
             List<BinaryRangeAggregator.Range> ranges, boolean keyed,
             SearchContext context,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.ranges = ranges;
         this.keyed = keyed;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index c35956e2dcf..a2228b7a27a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -175,10 +175,6 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
     }
 
     public static class Factory<B extends Bucket, R extends InternalRange<B, R>> {
-        public Type type() {
-            return RangeAggregationBuilder.TYPE;
-        }
-
         public ValuesSourceType getValueSourceType() {
             return ValuesSourceType.NUMERIC;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 8f4f9d6ccd8..5692b34c57f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -23,10 +23,9 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
@@ -37,7 +36,6 @@ import java.io.IOException;
 
 public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregationBuilder, Range> {
     public static final String NAME = "range";
-    static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<RangeAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -57,7 +55,7 @@ public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregati
     }
 
     private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
-        return Range.fromXContent(parser, context.getParseFieldMatcher());
+        return Range.fromXContent(parser);
     }
 
     public RangeAggregationBuilder(String name) {
@@ -142,12 +140,12 @@ public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregati
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         // We need to call processRanges here so they are parsed before we make the decision of whether to cache the request
         Range[] ranges = processRanges(context, config);
-        return new RangeAggregatorFactory(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
+        return new RangeAggregatorFactory(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
                 metaData);
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index cdb1dda221d..a2e55c18fb7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -122,7 +121,7 @@ public class RangeAggregator extends BucketsAggregator {
             return new Range(key, from, fromAsStr, to, toAsStr);
         }
 
-        public static Range fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        public static Range fromXContent(XContentParser parser) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             double from = Double.NEGATIVE_INFINITY;
@@ -134,17 +133,17 @@ public class RangeAggregator extends BucketsAggregator {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
+                    if (FROM_FIELD.match(currentFieldName)) {
                         from = parser.doubleValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
+                    } else if (TO_FIELD.match(currentFieldName)) {
                         to = parser.doubleValue();
                     }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
+                    if (FROM_FIELD.match(currentFieldName)) {
                         fromAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
+                    } else if (TO_FIELD.match(currentFieldName)) {
                         toAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, KEY_FIELD)) {
+                    } else if (KEY_FIELD.match(currentFieldName)) {
                         key = parser.text();
                     }
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
index de512fbaad4..d1dc3e71b5c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorFactory.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -33,10 +32,10 @@ import java.util.Map;
 
 public class RangeAggregatorFactory extends AbstractRangeAggregatorFactory<RangeAggregatorFactory, RangeAggregator.Range> {
 
-    public RangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
+    public RangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
             Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
index c2b34aee40b..de5622299cf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
@@ -23,10 +23,9 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
@@ -40,7 +39,6 @@ import java.io.IOException;
 
 public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeAggregationBuilder, RangeAggregator.Range> {
     public static final String NAME = "date_range";
-    static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<DateRangeAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -60,7 +58,7 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeA
     }
 
     private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
-        return Range.fromXContent(parser, context.getParseFieldMatcher());
+        return Range.fromXContent(parser);
     }
 
     public DateRangeAggregationBuilder(String name) {
@@ -75,7 +73,7 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeA
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -288,7 +286,7 @@ public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeA
         // We need to call processRanges here so they are parsed and we know whether `now` has been used before we make 
         // the decision of whether to cache the request
         Range[] ranges = processRanges(context, config);
-        return new DateRangeAggregatorFactory(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
+        return new DateRangeAggregatorFactory(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
                 metaData);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
index 03afbff663e..3d674c5a040 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorFactory.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.range.date;
 
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
@@ -34,10 +33,10 @@ import java.util.Map;
 
 public class DateRangeAggregatorFactory extends AbstractRangeAggregatorFactory<DateRangeAggregatorFactory, Range> {
 
-    public DateRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
+    public DateRangeAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, Range[] ranges, boolean keyed,
             Factory<?, ?> rangeFactory, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
index f7b55ab9916..6a2bea74920 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
@@ -84,11 +84,6 @@ public class InternalDateRange extends InternalRange<InternalDateRange.Bucket, I
     }
 
     public static class Factory extends InternalRange.Factory<InternalDateRange.Bucket, InternalDateRange> {
-        @Override
-        public Type type() {
-            return DateRangeAggregationBuilder.TYPE;
-        }
-
         @Override
         public ValueType getValueType() {
             return ValueType.DATE;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
index d70c150363b..9278a0b73b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.bucket.range.geodistance;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -32,10 +31,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -52,7 +50,6 @@ import java.util.Objects;
 
 public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoDistanceAggregationBuilder> {
     public static final String NAME = "geo_distance";
-    public static final Type TYPE = new Type(NAME);
     static final ParseField ORIGIN_FIELD = new ParseField("origin", "center", "point", "por");
     static final ParseField UNIT_FIELD = new ParseField("unit");
     static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
@@ -168,7 +165,6 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
     }
 
     private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
-        ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
         String fromAsStr = null;
         String toAsStr = null;
         double from = 0.0;
@@ -180,17 +176,17 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
             if (token == XContentParser.Token.FIELD_NAME) {
                 toOrFromOrKey = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (parseFieldMatcher.match(toOrFromOrKey, Range.FROM_FIELD)) {
+                if (Range.FROM_FIELD.match(toOrFromOrKey)) {
                     from = parser.doubleValue();
-                } else if (parseFieldMatcher.match(toOrFromOrKey, Range.TO_FIELD)) {
+                } else if (Range.TO_FIELD.match(toOrFromOrKey)) {
                     to = parser.doubleValue();
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (parseFieldMatcher.match(toOrFromOrKey, Range.KEY_FIELD)) {
+                if (Range.KEY_FIELD.match(toOrFromOrKey)) {
                     key = parser.text();
-                } else if (parseFieldMatcher.match(toOrFromOrKey, Range.FROM_FIELD)) {
+                } else if (Range.FROM_FIELD.match(toOrFromOrKey)) {
                     fromAsStr = parser.text();
-                } else if (parseFieldMatcher.match(toOrFromOrKey, Range.TO_FIELD)) {
+                } else if (Range.TO_FIELD.match(toOrFromOrKey)) {
                     toAsStr = parser.text();
                 }
             }
@@ -205,7 +201,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
     private GeoPoint origin;
     private List<Range> ranges = new ArrayList<>();
     private DistanceUnit unit = DistanceUnit.DEFAULT;
-    private GeoDistance distanceType = GeoDistance.DEFAULT;
+    private GeoDistance distanceType = GeoDistance.ARC;
     private boolean keyed = false;
 
     public GeoDistanceAggregationBuilder(String name, GeoPoint origin) {
@@ -217,7 +213,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
 
     private GeoDistanceAggregationBuilder(String name, GeoPoint origin,
                                           InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory) {
-        super(name, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
+        super(name, rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.origin = origin;
     }
 
@@ -225,8 +221,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
      * Read from a stream.
      */
     public GeoDistanceAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalGeoDistance.FACTORY.type(), InternalGeoDistance.FACTORY.getValueSourceType(),
-                InternalGeoDistance.FACTORY.getValueType());
+        super(in, InternalGeoDistance.FACTORY.getValueSourceType(), InternalGeoDistance.FACTORY.getValueType());
         origin = new GeoPoint(in.readDouble(), in.readDouble());
         int size = in.readVInt();
         ranges = new ArrayList<>(size);
@@ -347,7 +342,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -389,7 +384,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde
             ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
                     throws IOException {
         Range[] ranges = this.ranges.toArray(new Range[this.range().size()]);
-        return new GeoDistanceRangeAggregatorFactory(name, type, config, origin, ranges, unit, distanceType, keyed, context, parent,
+        return new GeoDistanceRangeAggregatorFactory(name, config, origin, ranges, unit, distanceType, keyed, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
index 513c1fe8ee1..e12853e5b17 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceRangeAggregatorFactory.java
@@ -22,8 +22,8 @@ package org.elasticsearch.search.aggregations.bucket.range.geodistance;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
@@ -31,7 +31,6 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Unmapped;
@@ -56,10 +55,10 @@ public class GeoDistanceRangeAggregatorFactory
     private final GeoDistance distanceType;
     private final boolean keyed;
 
-    public GeoDistanceRangeAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config, GeoPoint origin,
+    public GeoDistanceRangeAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, GeoPoint origin,
             Range[] ranges, DistanceUnit unit, GeoDistance distanceType, boolean keyed, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.origin = origin;
         this.ranges = ranges;
         this.unit = unit;
@@ -86,16 +85,16 @@ public class GeoDistanceRangeAggregatorFactory
 
         private final ValuesSource.GeoPoint source;
         private final GeoDistance distanceType;
-        private final DistanceUnit unit;
+        private final DistanceUnit units;
         private final org.elasticsearch.common.geo.GeoPoint origin;
 
-        public DistanceSource(ValuesSource.GeoPoint source, GeoDistance distanceType, org.elasticsearch.common.geo.GeoPoint origin,
-                DistanceUnit unit) {
+        DistanceSource(ValuesSource.GeoPoint source, GeoDistance distanceType,
+                org.elasticsearch.common.geo.GeoPoint origin, DistanceUnit units) {
             this.source = source;
             // even if the geo points are unique, there's no guarantee the
             // distances are
             this.distanceType = distanceType;
-            this.unit = unit;
+            this.units = units;
             this.origin = origin;
         }
 
@@ -112,8 +111,7 @@ public class GeoDistanceRangeAggregatorFactory
         @Override
         public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) {
             final MultiGeoPointValues geoValues = source.geoPointValues(ctx);
-            final FixedSourceDistance distance = distanceType.fixedSourceDistance(origin.getLat(), origin.getLon(), unit);
-            return GeoDistance.distanceValues(geoValues, distance);
+            return GeoUtils.distanceValues(distanceType, units, geoValues, origin);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
index 86fc0372982..9e6518ca097 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
@@ -59,11 +59,6 @@ public class InternalGeoDistance extends InternalRange<InternalGeoDistance.Bucke
     }
 
     public static class Factory extends InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> {
-        @Override
-        public Type type() {
-            return GeoDistanceAggregationBuilder.TYPE;
-        }
-
         @Override
         public ValuesSourceType getValueSourceType() {
             return ValuesSourceType.GEOPOINT;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
index 507f1961d62..cb03ef7251a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.range.ip;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -33,10 +32,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
@@ -61,7 +59,6 @@ import java.util.Objects;
 public final class IpRangeAggregationBuilder
         extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, IpRangeAggregationBuilder> {
     public static final String NAME = "ip_range";
-    private static final InternalAggregation.Type TYPE = new InternalAggregation.Type(NAME);
     private static final ParseField MASK_FIELD = new ParseField("mask");
 
     private static final ObjectParser<IpRangeAggregationBuilder, QueryParseContext> PARSER;
@@ -81,7 +78,6 @@ public final class IpRangeAggregationBuilder
     }
 
     private static Range parseRange(XContentParser parser, QueryParseContext context) throws IOException {
-        final ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
         String key = null;
         String from = null;
         String to = null;
@@ -94,13 +90,13 @@ public final class IpRangeAggregationBuilder
             if (parser.currentToken() == Token.FIELD_NAME) {
                 continue;
             }
-            if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.KEY_FIELD)) {
+            if (RangeAggregator.Range.KEY_FIELD.match(parser.currentName())) {
                 key = parser.text();
-            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.FROM_FIELD)) {
+            } else if (RangeAggregator.Range.FROM_FIELD.match(parser.currentName())) {
                 from = parser.textOrNull();
-            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.TO_FIELD)) {
+            } else if (RangeAggregator.Range.TO_FIELD.match(parser.currentName())) {
                 to = parser.textOrNull();
-            } else if (parseFieldMatcher.match(parser.currentName(), MASK_FIELD)) {
+            } else if (MASK_FIELD.match(parser.currentName())) {
                 mask = parser.text();
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "Unexpected ip range parameter: [" + parser.currentName() + "]");
@@ -235,11 +231,11 @@ public final class IpRangeAggregationBuilder
     private List<Range> ranges = new ArrayList<>();
 
     public IpRangeAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+        super(name, ValuesSourceType.BYTES, ValueType.IP);
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
@@ -341,7 +337,7 @@ public final class IpRangeAggregationBuilder
     }
 
     public IpRangeAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+        super(in, ValuesSourceType.BYTES, ValueType.IP);
         final int numRanges = in.readVInt();
         for (int i = 0; i < numRanges; ++i) {
             addRange(new Range(in));
@@ -376,7 +372,7 @@ public final class IpRangeAggregationBuilder
         for (Range range : this.ranges) {
             ranges.add(new BinaryRangeAggregator.Range(range.key, toBytesRef(range.from), toBytesRef(range.to)));
         }
-        return new BinaryRangeAggregatorFactory(name, TYPE, config, ranges,
+        return new BinaryRangeAggregatorFactory(name, config, ranges,
                 keyed, context, parent, subFactoriesBuilder, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
index 2465ef97457..78f5bd0a7af 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -41,7 +40,6 @@ import java.util.Objects;
 
 public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, DiversifiedAggregationBuilder> {
     public static final String NAME = "diversified_sampler";
-    public static final Type TYPE = new Type(NAME);
 
     public static final int MAX_DOCS_PER_VALUE_DEFAULT = 1;
 
@@ -63,14 +61,14 @@ public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilde
     private String executionHint = null;
 
     public DiversifiedAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.ANY, null);
+        super(name, ValuesSourceType.ANY, null);
     }
 
     /**
      * Read from a stream.
      */
     public DiversifiedAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY, null);
+        super(in, ValuesSourceType.ANY, null);
         shardSize = in.readVInt();
         maxDocsPerValue = in.readVInt();
         executionHint = in.readOptionalString();
@@ -139,7 +137,7 @@ public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilde
     @Override
     protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context,
             ValuesSourceConfig<ValuesSource> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new DiversifiedAggregatorFactory(name, TYPE, config, shardSize, maxDocsPerValue, executionHint, context, parent,
+        return new DiversifiedAggregatorFactory(name, config, shardSize, maxDocsPerValue, executionHint, context, parent,
                 subFactoriesBuilder, metaData);
     }
 
@@ -167,7 +165,7 @@ public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilde
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
index 39e5af4b78c..97a68649ca2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorFactory.java
@@ -24,7 +24,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator.ExecutionMode;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -44,10 +43,10 @@ public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory<
     private final int maxDocsPerValue;
     private final String executionHint;
 
-    public DiversifiedAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, int shardSize, int maxDocsPerValue,
+    public DiversifiedAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, int shardSize, int maxDocsPerValue,
             String executionHint, SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.shardSize = shardSize;
         this.maxDocsPerValue = maxDocsPerValue;
         this.executionHint = executionHint;
@@ -65,7 +64,7 @@ public class DiversifiedAggregatorFactory extends ValuesSourceAggregatorFactory<
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
 
             // In some cases using ordinals is just not supported: override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
index e958639ab4e..f4d45a7d471 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedBytesHashSamplerAggregator.java
@@ -70,7 +70,7 @@ public class DiversifiedBytesHashSamplerAggregator extends SamplerAggregator {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -86,7 +86,7 @@ public class DiversifiedBytesHashSamplerAggregator extends SamplerAggregator {
 
             private SortedBinaryDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerValue) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerValue) {
                 super(numHits, maxHitsPerValue);
 
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
index e3e2b547285..08fa1bcb7fa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedMapSamplerAggregator.java
@@ -76,7 +76,7 @@ public class DiversifiedMapSamplerAggregator extends SamplerAggregator {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -92,7 +92,7 @@ public class DiversifiedMapSamplerAggregator extends SamplerAggregator {
 
             private SortedBinaryDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
 
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
index fb3f5cd4e30..c595fdb5c25 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedNumericSamplerAggregator.java
@@ -63,7 +63,7 @@ public class DiversifiedNumericSamplerAggregator extends SamplerAggregator {
      * This implementation is only for use with a single bucket aggregation.
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -78,7 +78,7 @@ public class DiversifiedNumericSamplerAggregator extends SamplerAggregator {
 
             private SortedNumericDocValues values;
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
index e33cf6b867e..5eb37f310ad 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedOrdinalsSamplerAggregator.java
@@ -65,7 +65,7 @@ public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator {
      */
     class DiverseDocsDeferringCollector extends BestDocsDeferringCollector {
 
-        public DiverseDocsDeferringCollector() {
+        DiverseDocsDeferringCollector() {
             super(shardSize, context.bigArrays());
         }
 
@@ -79,7 +79,7 @@ public class DiversifiedOrdinalsSamplerAggregator extends SamplerAggregator {
         class ValuesDiversifiedTopDocsCollector extends DiversifiedTopDocsCollector {
 
 
-            public ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
+            ValuesDiversifiedTopDocsCollector(int numHits, int maxHitsPerKey) {
                 super(numHits, maxHitsPerKey);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index 46353ad371e..f69b66ffd1e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -28,7 +28,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -36,21 +35,20 @@ import java.util.Objects;
 
 public class SamplerAggregationBuilder extends AbstractAggregationBuilder<SamplerAggregationBuilder> {
     public static final String NAME = "sampler";
-    private static final Type TYPE = new Type(NAME);
 
     public static final int DEFAULT_SHARD_SAMPLE_SIZE = 100;
 
     private int shardSize = DEFAULT_SHARD_SAMPLE_SIZE;
 
     public SamplerAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public SamplerAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         shardSize = in.readVInt();
     }
 
@@ -77,7 +75,7 @@ public class SamplerAggregationBuilder extends AbstractAggregationBuilder<Sample
     @Override
     protected SamplerAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
             throws IOException {
-        return new SamplerAggregatorFactory(name, type, shardSize, context, parent, subFactoriesBuilder, metaData);
+        return new SamplerAggregatorFactory(name, shardSize, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -98,7 +96,7 @@ public class SamplerAggregationBuilder extends AbstractAggregationBuilder<Sample
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SamplerAggregator.SHARD_SIZE_FIELD)) {
+                if (SamplerAggregator.SHARD_SIZE_FIELD.match(currentFieldName)) {
                     shardSize = parser.intValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -129,7 +127,7 @@ public class SamplerAggregationBuilder extends AbstractAggregationBuilder<Sample
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
index 7d15839062c..ca93b3603a2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregator.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.sampler;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -111,9 +110,9 @@ public class SamplerAggregator extends SingleBucketAggregator {
 
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
index a912f9bc999..4fb7e28f3d6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.bucket.sampler;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -34,9 +33,9 @@ public class SamplerAggregatorFactory extends AggregatorFactory<SamplerAggregato
 
     private final int shardSize;
 
-    public SamplerAggregatorFactory(String name, Type type, int shardSize, SearchContext context, AggregatorFactory<?> parent,
+    public SamplerAggregatorFactory(String name, int shardSize, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.shardSize = shardSize;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
index ffca3740184..ae33cc87308 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
@@ -40,19 +40,19 @@ public class SignificantLongTerms extends InternalMappedSignificantTerms<Signifi
 
         long term;
 
-        public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
+        Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
                 DocValueFormat format) {
             super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
             this.term = term;
         }
 
-        public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
+        Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
                 double score) {
             this(subsetDf, subsetSize, supersetDf, supersetSize, term, aggregations, null);
             this.score = score;
         }
 
-        public Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
+        Bucket(StreamInput in, long subsetSize, long supersetSize, DocValueFormat format) throws IOException {
             super(in, subsetSize, supersetSize, format);
             subsetDf = in.readVLong();
             supersetDf = in.readVLong();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
index 34d8361cbf0..aa0a25ac45f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
@@ -28,7 +28,7 @@ import java.util.List;
  * An aggregation that collects significant terms in comparison to a background set.
  */
 public interface SignificantTerms extends MultiBucketsAggregation, Iterable<SignificantTerms.Bucket> {
-    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
+    abstract class Bucket extends InternalMultiBucketAggregation.InternalBucket {
 
         long subsetDf;
         long subsetSize;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
index 6205ab2e787..72825dbd989 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
@@ -26,12 +26,10 @@ import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
@@ -53,7 +51,6 @@ import java.util.Objects;
 
 public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, SignificantTermsAggregationBuilder> {
     public static final String NAME = "significant_terms";
-    public static final InternalAggregation.Type TYPE = new Type(NAME);
 
     static final ParseField BACKGROUND_FILTER = new ParseField("background_filter");
     static final ParseField HEURISTIC = new ParseField("significance_heuristic");
@@ -91,7 +88,7 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
             parser.declareObject(SignificantTermsAggregationBuilder::significanceHeuristic,
                     (p, context) -> {
                         SignificanceHeuristicParser significanceHeuristicParser = significanceHeuristicParserRegistry
-                                .lookupReturningNullIfNotFound(name, context.getParseFieldMatcher());
+                                .lookupReturningNullIfNotFound(name);
                         return significanceHeuristicParser.parse(context);
                     },
                     new ParseField(name));
@@ -111,14 +108,14 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
     private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
 
     public SignificantTermsAggregationBuilder(String name, ValueType valueType) {
-        super(name, TYPE, ValuesSourceType.ANY, valueType);
+        super(name, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a Stream.
      */
     public SignificantTermsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         executionHint = in.readOptionalString();
         filterBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
@@ -267,7 +264,7 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
     protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         SignificanceHeuristic executionHeuristic = this.significanceHeuristic.rewrite(context);
-        return new SignificantTermsAggregatorFactory(name, type, config, includeExclude, executionHint, filterBuilder,
+        return new SignificantTermsAggregatorFactory(name, config, includeExclude, executionHint, filterBuilder,
                 bucketCountThresholds, executionHeuristic, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -303,7 +300,7 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 80e15394ede..71e13d49a30 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -29,7 +29,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lucene.index.FilterableTermsEnum;
 import org.elasticsearch.common.lucene.index.FreqTermsEnum;
@@ -41,7 +40,6 @@ import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
@@ -72,11 +70,11 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private final SignificanceHeuristic significanceHeuristic;
 
-    public SignificantTermsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, IncludeExclude includeExclude,
+    public SignificantTermsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, IncludeExclude includeExclude,
             String executionHint, QueryBuilder filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds,
             SignificanceHeuristic significanceHeuristic, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
         this.filter = filterBuilder == null
@@ -196,7 +194,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
             if (!(valuesSource instanceof ValuesSource.Bytes.WithOrdinals)) {
                 execution = ExecutionMode.MAP;
@@ -289,9 +287,9 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
             }
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
index 6c392f0f283..5968f42211e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/GND.java
@@ -119,7 +119,7 @@ public class GND extends NXYSignificanceHeuristic {
             boolean backgroundIsSuperset = true;
             XContentParser.Token token = parser.nextToken();
             while (!token.equals(XContentParser.Token.END_OBJECT)) {
-                if (context.getParseFieldMatcher().match(parser.currentName(), BACKGROUND_IS_SUPERSET)) {
+                if (BACKGROUND_IS_SUPERSET.match(parser.currentName())) {
                     parser.nextToken();
                     backgroundIsSuperset = parser.booleanValue();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
index 69a759a77fd..5f92b5b40e6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
@@ -160,10 +160,10 @@ public abstract class NXYSignificanceHeuristic extends SignificanceHeuristic {
             boolean backgroundIsSuperset = true;
             XContentParser.Token token = parser.nextToken();
             while (!token.equals(XContentParser.Token.END_OBJECT)) {
-                if (context.getParseFieldMatcher().match(parser.currentName(), INCLUDE_NEGATIVES_FIELD)) {
+                if (INCLUDE_NEGATIVES_FIELD.match(parser.currentName())) {
                     parser.nextToken();
                     includeNegatives = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(parser.currentName(), BACKGROUND_IS_SUPERSET)) {
+                } else if (BACKGROUND_IS_SUPERSET.match(parser.currentName())) {
                     parser.nextToken();
                     backgroundIsSuperset = parser.booleanValue();
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
index b5ee7e09ee5..083733f31ae 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
@@ -157,8 +157,8 @@ public class ScriptHeuristic extends SignificanceHeuristic {
             if (token.equals(XContentParser.Token.FIELD_NAME)) {
                 currentFieldName = parser.currentName();
             } else {
-                if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ElasticsearchParseException("failed to parse [{}] significance heuristic. unknown object [{}]", heuristicName, currentFieldName);
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
index f160c86943a..9b40fcc42dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
@@ -36,7 +36,7 @@ abstract class AbstractStringTermsAggregator extends TermsAggregator {
 
     protected final boolean showTermDocCountError;
 
-    public AbstractStringTermsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
+    AbstractStringTermsAggregator(String name, AggregatorFactories factories, SearchContext context, Aggregator parent,
             Terms.Order order, DocValueFormat format, BucketCountThresholds bucketCountThresholds, SubAggCollectionMode subAggCollectMode,
             boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException {
         super(name, factories, context, parent, bucketCountThresholds, order, format, subAggCollectMode, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
index 7e3daf5034b..1e7be237f88 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
@@ -22,10 +22,12 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -38,7 +40,7 @@ public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bu
     static class Bucket extends InternalTerms.Bucket<Bucket> {
         private final double term;
 
-        public Bucket(double term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
+        Bucket(double term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat format) {
             super(docCount, aggregations, showDocCountError, docCountError, format);
             this.term = term;
@@ -47,7 +49,7 @@ public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bu
         /**
          * Read from a stream.
          */
-        public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+        Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
             super(in, format, showDocCountError);
             term = in.readDouble();
         }
@@ -152,4 +154,32 @@ public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bu
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
     }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        boolean promoteToDouble = false;
+        for (InternalAggregation agg : aggregations) {
+            if (agg instanceof LongTerms && ((LongTerms) agg).format == DocValueFormat.RAW) {
+                /**
+                 * this terms agg mixes longs and doubles, we must promote longs to doubles to make the internal aggs
+                 * compatible
+                 */
+                promoteToDouble = true;
+                break;
+            }
+        }
+        if (promoteToDouble == false) {
+            return super.doReduce(aggregations, reduceContext);
+        }
+        List<InternalAggregation> newAggs = new ArrayList<>();
+        for (InternalAggregation agg : aggregations) {
+            if (agg instanceof LongTerms) {
+                DoubleTerms dTerms = LongTerms.convertLongTermsToDouble((LongTerms) agg, format);
+                newAggs.add(dTerms);
+            } else {
+                newAggs.add(agg);
+            }
+        }
+        return newAggs.get(0).doReduce(newAggs, reduceContext);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
index e904c667906..513e7a1ac0e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
@@ -241,11 +241,11 @@ class InternalOrder extends Terms.Order {
 
         private final List<Terms.Order> orderElements;
 
-        public CompoundOrder(List<Terms.Order> compoundOrder) {
+        CompoundOrder(List<Terms.Order> compoundOrder) {
             this(compoundOrder, true);
         }
 
-        public CompoundOrder(List<Terms.Order> compoundOrder, boolean absoluteOrdering) {
+        CompoundOrder(List<Terms.Order> compoundOrder, boolean absoluteOrdering) {
             this.orderElements = new LinkedList<>(compoundOrder);
             Terms.Order lastElement = compoundOrder.get(compoundOrder.size() - 1);
             if (absoluteOrdering && !(InternalOrder.TERM_ASC == lastElement || InternalOrder.TERM_DESC == lastElement)) {
@@ -300,7 +300,7 @@ class InternalOrder extends Terms.Order {
             private List<Terms.Order> compoundOrder;
             private Aggregator aggregator;
 
-            public CompoundOrderComparator(List<Terms.Order> compoundOrder, Aggregator aggregator) {
+            CompoundOrderComparator(List<Terms.Order> compoundOrder, Aggregator aggregator) {
                 this.compoundOrder = compoundOrder;
                 this.aggregator = aggregator;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
index 41092dba176..b1799b52ace 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
@@ -22,10 +22,12 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -152,4 +154,32 @@ public class LongTerms extends InternalMappedTerms<LongTerms, LongTerms.Bucket>
     protected Bucket[] createBucketsArray(int size) {
         return new Bucket[size];
     }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        for (InternalAggregation agg : aggregations) {
+            if (agg instanceof DoubleTerms) {
+                return agg.doReduce(aggregations, reduceContext);
+            }
+        }
+        return super.doReduce(aggregations, reduceContext);
+    }
+
+    /**
+     * Converts a {@link LongTerms} into a {@link DoubleTerms}, returning the value of the specified long terms as doubles.
+     */
+    static DoubleTerms convertLongTermsToDouble(LongTerms longTerms, DocValueFormat decimalFormat) {
+        List<Terms.Bucket> buckets = longTerms.getBuckets();
+        List<DoubleTerms.Bucket> newBuckets = new ArrayList<>();
+        for (Terms.Bucket bucket : buckets) {
+            newBuckets.add(new DoubleTerms.Bucket(bucket.getKeyAsNumber().doubleValue(),
+                bucket.getDocCount(), (InternalAggregations) bucket.getAggregations(), longTerms.showTermDocCountError,
+                longTerms.showTermDocCountError ? bucket.getDocCountError() : 0, decimalFormat));
+        }
+        return new DoubleTerms(longTerms.getName(), longTerms.order, longTerms.requiredSize,
+            longTerms.minDocCount, longTerms.pipelineAggregators(),
+            longTerms.metaData, longTerms.format, longTerms.shardSize,
+            longTerms.showTermDocCountError, longTerms.otherDocCount,
+            newBuckets, longTerms.docCountError);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
index aa70309da95..e4ff4cd3944 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
@@ -33,7 +33,7 @@ import java.util.List;
  */
 public interface Terms extends MultiBucketsAggregation {
 
-    static enum ValueType {
+    enum ValueType {
 
         STRING(org.elasticsearch.search.aggregations.support.ValueType.STRING),
         LONG(org.elasticsearch.search.aggregations.support.ValueType.LONG),
@@ -41,7 +41,7 @@ public interface Terms extends MultiBucketsAggregation {
 
         final org.elasticsearch.search.aggregations.support.ValueType scriptValueType;
 
-        private ValueType(org.elasticsearch.search.aggregations.support.ValueType scriptValueType) {
+        ValueType(org.elasticsearch.search.aggregations.support.ValueType scriptValueType) {
             this.scriptValueType = scriptValueType;
         }
 
@@ -62,7 +62,7 @@ public interface Terms extends MultiBucketsAggregation {
     /**
      * A bucket that is associated with a single term
      */
-    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
+    abstract class Bucket extends InternalMultiBucketAggregation.InternalBucket {
 
         public abstract Number getKeyAsNumber();
 
@@ -97,7 +97,7 @@ public interface Terms extends MultiBucketsAggregation {
     /**
      * Determines the order by which the term buckets will be sorted
      */
-    abstract static class Order implements ToXContent {
+    abstract class Order implements ToXContent {
 
         /**
          * @return a bucket ordering strategy that sorts buckets by their document counts (ascending or descending)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 857b3350c85..944f9fd96a4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -26,12 +26,10 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
@@ -50,7 +48,6 @@ import java.util.Objects;
 
 public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, TermsAggregationBuilder> {
     public static final String NAME = "terms";
-    private static final InternalAggregation.Type TYPE = new Type("terms");
 
     public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
     public static final ParseField SHARD_SIZE_FIELD_NAME = new ParseField("shard_size");
@@ -82,7 +79,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
         PARSER.declareString(TermsAggregationBuilder::executionHint, EXECUTION_HINT_FIELD_NAME);
 
         PARSER.declareField(TermsAggregationBuilder::collectMode,
-                (p, c) -> SubAggCollectionMode.parse(p.text(), c.getParseFieldMatcher()),
+                (p, c) -> SubAggCollectionMode.parse(p.text()),
                 SubAggCollectionMode.KEY, ObjectParser.ValueType.STRING);
 
         PARSER.declareObjectArray(TermsAggregationBuilder::order, TermsAggregationBuilder::parseOrderParam,
@@ -108,14 +105,14 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     private boolean showTermDocCountError = false;
 
     public TermsAggregationBuilder(String name, ValueType valueType) {
-        super(name, TYPE, ValuesSourceType.ANY, valueType);
+        super(name, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a stream.
      */
     public TermsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         collectMode = in.readOptionalWriteable(SubAggCollectionMode::readFromStream);
         executionHint = in.readOptionalString();
@@ -287,7 +284,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     @Override
     protected ValuesSourceAggregatorFactory<ValuesSource, ?> innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new TermsAggregatorFactory(name, type, config, order, includeExclude, executionHint, collectMode,
+        return new TermsAggregatorFactory(name, config, order, includeExclude, executionHint, collectMode,
                 bucketCountThresholds, showTermDocCountError, context, parent, subFactoriesBuilder, metaData);
     }
 
@@ -326,7 +323,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 37e6813ebcc..10fef554555 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.terms;
 
 import org.apache.lucene.search.IndexSearcher;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -29,7 +28,6 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
 import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
@@ -53,11 +51,11 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private boolean showTermDocCountError;
 
-    public TermsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, Terms.Order order,
+    public TermsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, Terms.Order order,
             IncludeExclude includeExclude, String executionHint, SubAggCollectionMode collectMode,
             TermsAggregator.BucketCountThresholds bucketCountThresholds, boolean showTermDocCountError, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.order = order;
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
@@ -104,7 +102,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
         if (valuesSource instanceof ValuesSource.Bytes) {
             ExecutionMode execution = null;
             if (executionHint != null) {
-                execution = ExecutionMode.fromString(executionHint, context.parseFieldMatcher());
+                execution = ExecutionMode.fromString(executionHint);
             }
 
             // In some cases, using ordinals is just not supported: override it
@@ -306,9 +304,9 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
             }
         };
 
-        public static ExecutionMode fromString(String value, ParseFieldMatcher parseFieldMatcher) {
+        public static ExecutionMode fromString(String value) {
             for (ExecutionMode mode : values()) {
-                if (parseFieldMatcher.match(value, mode.parseField)) {
+                if (mode.parseField.match(value)) {
                     return mode;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
index 725a8c437c0..5925cc2e4ad 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.terms.support;
 import com.carrotsearch.hppc.BitMixer;
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
-
 import org.apache.lucene.index.RandomAccessOrds;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Terms;
@@ -39,7 +38,6 @@ import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -103,7 +101,6 @@ public class IncludeExclude implements Writeable, ToXContent {
         } else if (token == XContentParser.Token.START_ARRAY) {
             return new IncludeExclude(new TreeSet<>(parseArrayToSet(parser)), null);
         } else if (token == XContentParser.Token.START_OBJECT) {
-            ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
             String currentFieldName = null;
             Integer partition = null, numPartitions = null;
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -112,11 +109,11 @@ public class IncludeExclude implements Writeable, ToXContent {
                 } else
                 // This "include":{"pattern":"foo.*"} syntax is undocumented since 2.0
                 // Regexes should be "include":"foo.*"
-                if (parseFieldMatcher.match(currentFieldName, PATTERN_FIELD)) {
+                if (PATTERN_FIELD.match(currentFieldName)) {
                     return new IncludeExclude(parser.text(), null);
-                } else if (parseFieldMatcher.match(currentFieldName, NUM_PARTITIONS_FIELD)) {
+                } else if (NUM_PARTITIONS_FIELD.match(currentFieldName)) {
                     numPartitions = parser.intValue();
-                } else if (parseFieldMatcher.match(currentFieldName, PARTITION_FIELD)) {
+                } else if (PARTITION_FIELD.match(currentFieldName)) {
                     partition = parser.intValue();
                 } else {
                     throw new ElasticsearchParseException(
@@ -226,7 +223,7 @@ public class IncludeExclude implements Writeable, ToXContent {
         private final Set<BytesRef> valids;
         private final Set<BytesRef> invalids;
 
-        public TermListBackedStringFilter(Set<BytesRef> includeValues, Set<BytesRef> excludeValues) {
+        TermListBackedStringFilter(Set<BytesRef> includeValues, Set<BytesRef> excludeValues) {
             this.valids = includeValues;
             this.invalids = excludeValues;
         }
@@ -298,7 +295,7 @@ public class IncludeExclude implements Writeable, ToXContent {
         private final SortedSet<BytesRef> includeValues;
         private final SortedSet<BytesRef> excludeValues;
 
-        public TermListBackedOrdinalsFilter(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
+        TermListBackedOrdinalsFilter(SortedSet<BytesRef> includeValues, SortedSet<BytesRef> excludeValues) {
             this.includeValues = includeValues;
             this.excludeValues = excludeValues;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
index 9f591c7d425..010e24346bf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
@@ -25,6 +25,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public abstract class InternalNumericMetricsAggregation extends InternalMetricsAggregation {
 
@@ -102,4 +103,22 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
     protected InternalNumericMetricsAggregation(StreamInput in) throws IOException {
         super(in);
     }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(format, super.hashCode());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (obj.getClass() != getClass()) {
+            return false;
+        }
+        InternalNumericMetricsAggregation other = (InternalNumericMetricsAggregation) obj;
+        return super.equals(obj) && 
+                Objects.equals(format, other.format);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
index 9269c2c7a31..0d9bd6fc1a9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,7 +40,6 @@ import java.io.IOException;
 
 public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregationBuilder> {
     public static final String NAME = "avg";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<AvgAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -54,14 +52,14 @@ public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     public AvgAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public AvgAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     @Override
     protected AvgAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new AvgAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new AvgAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -91,7 +89,7 @@ public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
index 29b52b929fd..f1fc12ef4e5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.avg;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -36,9 +35,9 @@ import java.util.Map;
 
 public class AvgAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, AvgAggregatorFactory> {
 
-    public AvgAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, SearchContext context,
+    public AvgAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
index bf2b65436e7..1f76d8530f7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
@@ -25,10 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -44,7 +43,6 @@ public final class CardinalityAggregationBuilder
     extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, CardinalityAggregationBuilder> {
 
     public static final String NAME = "cardinality";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ParseField REHASH = new ParseField("rehash").withAllDeprecated("no replacement - values will always be rehashed");
     public static final ParseField PRECISION_THRESHOLD_FIELD = new ParseField("precision_threshold");
@@ -64,14 +62,14 @@ public final class CardinalityAggregationBuilder
     private Long precisionThreshold = null;
 
     public CardinalityAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public CardinalityAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
         if (in.readBoolean()) {
             precisionThreshold = in.readLong();
         }
@@ -124,7 +122,7 @@ public final class CardinalityAggregationBuilder
     @Override
     protected CardinalityAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new CardinalityAggregatorFactory(name, type, config, precisionThreshold, context, parent, subFactoriesBuilder, metaData);
+        return new CardinalityAggregatorFactory(name, config, precisionThreshold, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -147,7 +145,7 @@ public final class CardinalityAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
index e9ca652f2c7..7d5db460ae6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
@@ -327,7 +327,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
 
             private final SortedNumericDocValues values;
 
-            public Long(SortedNumericDocValues values) {
+            Long(SortedNumericDocValues values) {
                 this.values = values;
             }
 
@@ -351,7 +351,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
 
             private final SortedNumericDoubleValues values;
 
-            public Double(SortedNumericDoubleValues values) {
+            Double(SortedNumericDoubleValues values) {
                 this.values = values;
             }
 
@@ -377,7 +377,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
 
             private final SortedBinaryDocValues values;
 
-            public Bytes(SortedBinaryDocValues values) {
+            Bytes(SortedBinaryDocValues values) {
                 this.values = values;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
index 049f4cec9d3..0d2d32f0469 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.cardinality;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -37,10 +36,10 @@ public class CardinalityAggregatorFactory extends ValuesSourceAggregatorFactory<
 
     private final Long precisionThreshold;
 
-    public CardinalityAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, Long precisionThreshold,
+    public CardinalityAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, Long precisionThreshold,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.precisionThreshold = precisionThreshold;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
index 568ecdbec59..42b4561e07b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/HyperLogLogPlusPlus.java
@@ -433,7 +433,7 @@ public final class HyperLogLogPlusPlus implements Releasable {
         private final BytesRef readSpare;
         private final ByteBuffer writeSpare;
 
-        public Hashset(long initialBucketCount) {
+        Hashset(long initialBucketCount) {
             capacity = m / 4; // because ints take 4 bytes
             threshold = (int) (capacity * MAX_LOAD_FACTOR);
             mask = capacity - 1;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
index c034c822f2a..be3ad4db802 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -41,7 +40,6 @@ import java.util.Objects;
 
 public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoBoundsAggregationBuilder> {
     public static final String NAME = "geo_bounds";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<GeoBoundsAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -57,14 +55,14 @@ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<
     private boolean wrapLongitude = true;
 
     public GeoBoundsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoBoundsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         wrapLongitude = in.readBoolean();
     }
 
@@ -91,7 +89,7 @@ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<
     @Override
     protected GeoBoundsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new GeoBoundsAggregatorFactory(name, type, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData);
+        return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -112,7 +110,7 @@ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
index 745dfa0ebc3..e67ad49115a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.geobounds;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -37,10 +36,10 @@ public class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory<Va
 
     private final boolean wrapLongitude;
 
-    public GeoBoundsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config, boolean wrapLongitude,
+    public GeoBoundsAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config, boolean wrapLongitude,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.wrapLongitude = wrapLongitude;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
index b999936693d..5a9df082965 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
@@ -188,7 +188,7 @@ public class InternalGeoBounds extends InternalMetricsAggregation implements Geo
         private final GeoPoint topLeft;
         private final GeoPoint bottomRight;
         
-        public BoundingBox(GeoPoint topLeft, GeoPoint bottomRight) {
+        BoundingBox(GeoPoint topLeft, GeoPoint bottomRight) {
             this.topLeft = topLeft;
             this.bottomRight = bottomRight;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
index 34689cc696d..8e173e65923 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -41,7 +40,6 @@ import java.io.IOException;
 public class GeoCentroidAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
     public static final String NAME = "geo_centroid";
-    public static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<GeoCentroidAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -54,14 +52,14 @@ public class GeoCentroidAggregationBuilder
     }
 
     public GeoCentroidAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoCentroidAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class GeoCentroidAggregationBuilder
     @Override
     protected GeoCentroidAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource.GeoPoint> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new GeoCentroidAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new GeoCentroidAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -91,7 +89,7 @@ public class GeoCentroidAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
index aa548dc0542..c21999d3fb4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.geocentroid;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -35,10 +34,10 @@ import java.util.Map;
 
 public class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, GeoCentroidAggregatorFactory> {
 
-    public GeoCentroidAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource.GeoPoint> config,
+    public GeoCentroidAggregatorFactory(String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
index c9d045a7235..c1634fdce4b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
@@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public class InternalMax extends InternalNumericMetricsAggregation.SingleValue implements Max {
     private final double max;
@@ -88,4 +89,15 @@ public class InternalMax extends InternalNumericMetricsAggregation.SingleValue i
         }
         return builder;
     }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(max);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalMax other = (InternalMax) obj;
+        return Objects.equals(max, other.max);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
index f198a00e402..dafad9dacb5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,7 +40,6 @@ import java.io.IOException;
 
 public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregationBuilder> {
     public static final String NAME = "max";
-    public static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<MaxAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -54,14 +52,14 @@ public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     public MaxAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MaxAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     @Override
     protected MaxAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MaxAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MaxAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -91,7 +89,7 @@ public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
index ebc5612554e..aedba76e0c7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.max;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -36,9 +35,9 @@ import java.util.Map;
 
 public class MaxAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MaxAggregatorFactory> {
 
-    public MaxAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, SearchContext context,
+    public MaxAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
index 09f9c3915fd..46481d1837f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
@@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public class InternalMin extends InternalNumericMetricsAggregation.SingleValue implements Min {
     private final double min;
@@ -89,4 +90,15 @@ public class InternalMin extends InternalNumericMetricsAggregation.SingleValue i
         return builder;
     }
 
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(min);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalMin other = (InternalMin) obj;
+        return Objects.equals(min, other.min);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
index 4c34fe8dd7e..0f85748416e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -42,7 +41,6 @@ import java.io.IOException;
 
 public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MinAggregationBuilder> {
     public static final String NAME = "min";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<MinAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -55,14 +53,14 @@ public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     public MinAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MinAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -73,7 +71,7 @@ public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     @Override
     protected MinAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new MinAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new MinAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -92,7 +90,7 @@ public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
index df914978808..8f5538fb7a2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.min;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -36,9 +35,9 @@ import java.util.Map;
 
 public class MinAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, MinAggregatorFactory> {
 
-    public MinAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, SearchContext context,
+    public MinAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
index 08c4d7ec817..db322a8e70e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
@@ -28,7 +28,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentileRanksAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentileRanksAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -47,7 +46,6 @@ import java.util.Objects;
 
 public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentileRanksAggregationBuilder> {
     public static final String NAME = PercentileRanks.TYPE_NAME;
-    public static final Type TYPE = new Type(NAME);
 
     public static final ParseField VALUES_FIELD = new ParseField("values");
 
@@ -109,14 +107,14 @@ public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Num
     private boolean keyed = true;
 
     public PercentileRanksAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentileRanksAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         values = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
@@ -226,10 +224,10 @@ public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Num
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
-            return new TDigestPercentileRanksAggregatorFactory(name, type, config, values, compression, keyed, context, parent,
+            return new TDigestPercentileRanksAggregatorFactory(name, config, values, compression, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         case HDR:
-            return new HDRPercentileRanksAggregatorFactory(name, type, config, values, numberOfSignificantValueDigits, keyed, context,
+            return new HDRPercentileRanksAggregatorFactory(name, config, values, numberOfSignificantValueDigits, keyed, context,
                     parent, subFactoriesBuilder, metaData);
         default:
             throw new IllegalStateException("Illegal method [" + method + "]");
@@ -286,7 +284,7 @@ public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Num
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
index ef2512eab9b..767e875c132 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
@@ -25,10 +25,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentilesAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentilesAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -47,7 +46,6 @@ import java.util.Objects;
 
 public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentilesAggregationBuilder> {
     public static final String NAME = Percentiles.TYPE_NAME;
-    public static final Type TYPE = new Type(NAME);
 
     public static final double[] DEFAULT_PERCENTS = new double[] { 1, 5, 25, 50, 75, 95, 99 };
     public static final ParseField PERCENTS_FIELD = new ParseField("percents");
@@ -115,14 +113,14 @@ public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric
     private boolean keyed = true;
 
     public PercentilesAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentilesAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         percents = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
@@ -232,10 +230,10 @@ public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
         switch (method) {
         case TDIGEST:
-            return new TDigestPercentilesAggregatorFactory(name, type, config, percents, compression, keyed, context, parent,
+            return new TDigestPercentilesAggregatorFactory(name, config, percents, compression, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         case HDR:
-            return new HDRPercentilesAggregatorFactory(name, type, config, percents, numberOfSignificantValueDigits, keyed, context, parent,
+            return new HDRPercentilesAggregatorFactory(name, config, percents, numberOfSignificantValueDigits, keyed, context, parent,
                     subFactoriesBuilder, metaData);
         default:
             throw new IllegalStateException("Illegal method [" + method + "]");
@@ -292,7 +290,7 @@ public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
index af609c3bdd0..b10880a13c0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethod.java
@@ -41,7 +41,7 @@ public enum PercentilesMethod implements Writeable {
 
     private final ParseField parseField;
 
-    private PercentilesMethod(String name, String... deprecatedNames) {
+    PercentilesMethod(String name, String... deprecatedNames) {
         this.parseField = new ParseField(name, deprecatedNames);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
index ba76f8b9863..05ae303dc89 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
@@ -40,7 +40,7 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
     protected final DoubleHistogram state;
     private final boolean keyed;
 
-    public AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, DocValueFormat format,
+    AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, DocValueFormat format,
             List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
index 8de009d773b..d89a9a85b28 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentileRanksAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,10 +40,10 @@ public class HDRPercentileRanksAggregatorFactory
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
-    public HDRPercentileRanksAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] values,
+    public HDRPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] values,
             int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.values = values;
         this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
index 852d9175855..1074b6e142d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/HDRPercentilesAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -40,10 +39,10 @@ public class HDRPercentilesAggregatorFactory extends ValuesSourceAggregatorFacto
     private final int numberOfSignificantValueDigits;
     private final boolean keyed;
 
-    public HDRPercentilesAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
+    public HDRPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
             int numberOfSignificantValueDigits, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
index 43bee101f4a..ec9adc45ec9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
@@ -37,7 +37,7 @@ abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetrics
     protected final TDigestState state;
     private final boolean keyed;
 
-    public AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, DocValueFormat formatter,
+    AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
index 9894f9ba8af..223d25216bc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentileRanksAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,10 +40,10 @@ public class TDigestPercentileRanksAggregatorFactory
     private final double compression;
     private final boolean keyed;
 
-    public TDigestPercentileRanksAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
+    public TDigestPercentileRanksAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
             double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.compression = compression;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
index f0a7cc93810..47b17d84f3b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestPercentilesAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,10 +40,10 @@ public class TDigestPercentilesAggregatorFactory
     private final double compression;
     private final boolean keyed;
 
-    public TDigestPercentilesAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double[] percents,
+    public TDigestPercentilesAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double[] percents,
             double compression, boolean keyed, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.percents = percents;
         this.compression = compression;
         this.keyed = keyed;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index c5704d9f2b6..6cb3b626f91 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -88,8 +88,8 @@ public class InternalScriptedMetric extends InternalMetricsAggregation implement
             if (firstAggregation.reduceScript.getParams() != null) {
                 vars.putAll(firstAggregation.reduceScript.getParams());
             }
-            CompiledScript compiledScript = reduceContext.scriptService().compile(firstAggregation.reduceScript,
-                    ScriptContext.Standard.AGGS, Collections.emptyMap());
+            CompiledScript compiledScript = reduceContext.scriptService().compile(
+                firstAggregation.reduceScript, ScriptContext.Standard.AGGS);
             ExecutableScript script = reduceContext.scriptService().executable(compiledScript, vars);
             aggregation = script.run();
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
index e778a9cd369..2acd92395e0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
@@ -34,11 +34,9 @@ import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
@@ -46,9 +44,7 @@ import java.util.Set;
 import java.util.function.Function;
 
 public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder<ScriptedMetricAggregationBuilder> {
-
     public static final String NAME = "scripted_metric";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ParseField INIT_SCRIPT_FIELD = new ParseField("init_script");
     private static final ParseField MAP_SCRIPT_FIELD = new ParseField("map_script");
@@ -63,14 +59,14 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
     private Map<String, Object> params;
 
     public ScriptedMetricAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public ScriptedMetricAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         initScript = in.readOptionalWriteable(Script::new);
         mapScript = in.readOptionalWriteable(Script::new);
         combineScript = in.readOptionalWriteable(Script::new);
@@ -204,7 +200,7 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
         } else {
             executableCombineScript = (p) -> null;
         }
-        return new ScriptedMetricAggregatorFactory(name, type, searchMapScript, executableInitScript, executableCombineScript, reduceScript,
+        return new ScriptedMetricAggregatorFactory(name, searchMapScript, executableInitScript, executableCombineScript, reduceScript,
                 params, context, parent, subfactoriesBuilder, metaData);
     }
 
@@ -254,16 +250,16 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, INIT_SCRIPT_FIELD)) {
-                    initScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MAP_SCRIPT_FIELD)) {
-                    mapScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, COMBINE_SCRIPT_FIELD)) {
-                    combineScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, REDUCE_SCRIPT_FIELD)) {
-                    reduceScript = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                if (INIT_SCRIPT_FIELD.match(currentFieldName)) {
+                    initScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (MAP_SCRIPT_FIELD.match(currentFieldName)) {
+                    mapScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (COMBINE_SCRIPT_FIELD.match(currentFieldName)) {
+                    combineScript = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (REDUCE_SCRIPT_FIELD.match(currentFieldName)) {
+                    reduceScript = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else if (token == XContentParser.Token.START_OBJECT &&
-                        context.getParseFieldMatcher().match(currentFieldName, PARAMS_FIELD)) {
+                        PARAMS_FIELD.match(currentFieldName)) {
                     params = parser.map();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -298,7 +294,7 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
index a8d3b7c3e09..bac2becc8e4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorFactory.java
@@ -26,7 +26,6 @@ import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -45,11 +44,11 @@ public class ScriptedMetricAggregatorFactory extends AggregatorFactory<ScriptedM
     private final Map<String, Object> params;
     private final Function<Map<String, Object>, ExecutableScript> initScript;
 
-    public ScriptedMetricAggregatorFactory(String name, Type type, Function<Map<String, Object>, SearchScript> mapScript,
+    public ScriptedMetricAggregatorFactory(String name, Function<Map<String, Object>, SearchScript> mapScript,
             Function<Map<String, Object>, ExecutableScript> initScript, Function<Map<String, Object>, ExecutableScript> combineScript,
             Script reduceScript, Map<String, Object> params, SearchContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactories, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.mapScript = mapScript;
         this.initScript = initScript;
         this.combineScript = combineScript;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
index 633626dff2b..390be44d747 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,7 +40,6 @@ import java.io.IOException;
 
 public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregationBuilder> {
     public static final String NAME = "stats";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<StatsAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -54,14 +52,14 @@ public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.Leaf
     }
 
     public StatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public StatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.Leaf
     @Override
     protected StatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new StatsAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new StatsAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -91,7 +89,7 @@ public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.Leaf
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
index 996416d2a8e..a6e59d7c75b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.stats;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -36,9 +35,9 @@ import java.util.Map;
 
 public class StatsAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, StatsAggregatorFactory> {
 
-    public StatsAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, SearchContext context,
+    public StatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
index 1b235a6cfec..8a198a5825a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStats.java
@@ -67,7 +67,7 @@ public interface ExtendedStats extends Stats {
     String getVarianceAsString();
 
 
-    public enum Bounds {
+    enum Bounds {
         UPPER, LOWER
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
index 75c011fe715..94857c8753f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -43,7 +42,6 @@ import java.util.Objects;
 public class ExtendedStatsAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregationBuilder> {
     public static final String NAME = "extended_stats";
-    public static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<ExtendedStatsAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -59,14 +57,14 @@ public class ExtendedStatsAggregationBuilder
     private double sigma = 2.0;
 
     public ExtendedStatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public ExtendedStatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         sigma = in.readDouble();
     }
 
@@ -90,7 +88,7 @@ public class ExtendedStatsAggregationBuilder
     @Override
     protected ExtendedStatsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new ExtendedStatsAggregatorFactory(name, type, config, sigma, context, parent, subFactoriesBuilder, metaData);
+        return new ExtendedStatsAggregatorFactory(name, config, sigma, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -111,7 +109,7 @@ public class ExtendedStatsAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
index 6153c108c59..521ea8f68a6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.stats.extended;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -38,10 +37,10 @@ public class ExtendedStatsAggregatorFactory extends ValuesSourceAggregatorFactor
 
     private final double sigma;
 
-    public ExtendedStatsAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, double sigma,
+    public ExtendedStatsAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, double sigma,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
         this.sigma = sigma;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
index 9645610dc8c..7118b14d0cd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
@@ -24,10 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -41,7 +40,6 @@ import java.io.IOException;
 
 public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, SumAggregationBuilder> {
     public static final String NAME = "sum";
-    private static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<SumAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -54,14 +52,14 @@ public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     public SumAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public SumAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     @Override
     protected SumAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<Numeric> config,
             AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new SumAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new SumAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -91,7 +89,7 @@ public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOn
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
index a7dad9cac7f..8b6103214a7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.sum;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
@@ -36,9 +35,9 @@ import java.util.Map;
 
 public class SumAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource.Numeric, SumAggregatorFactory> {
 
-    public SumAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config, SearchContext context,
+    public SumAggregatorFactory(String name, ValuesSourceConfig<Numeric> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
index 8c6758b4f35..3f712f99b35 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHits.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations.metrics.tophits;
 
+import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TopDocs;
@@ -35,6 +36,7 @@ import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -86,6 +88,14 @@ public class InternalTopHits extends InternalMetricsAggregation implements TopHi
         return searchHits;
     }
 
+    TopDocs getTopDocs() {
+        return topDocs;
+    }
+
+    int getSize() {
+        return size;
+    }
+
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         InternalSearchHits[] shardHits = new InternalSearchHits[aggregations.size()];
@@ -145,4 +155,50 @@ public class InternalTopHits extends InternalMetricsAggregation implements TopHi
         searchHits.toXContent(builder, params);
         return builder;
     }
+
+    // Equals and hashcode implemented for testing round trips
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalTopHits other = (InternalTopHits) obj;
+        if (from != other.from) return false;
+        if (size != other.size) return false;
+        if (topDocs.totalHits != other.topDocs.totalHits) return false;
+        if (topDocs.scoreDocs.length != other.topDocs.scoreDocs.length) return false;
+        for (int d = 0; d < topDocs.scoreDocs.length; d++) {
+            ScoreDoc thisDoc = topDocs.scoreDocs[d];
+            ScoreDoc otherDoc = other.topDocs.scoreDocs[d];
+            if (thisDoc.doc != otherDoc.doc) return false;
+            if (thisDoc.score != otherDoc.score) return false;
+            if (thisDoc.shardIndex != otherDoc.shardIndex) return false;
+            if (thisDoc instanceof FieldDoc) {
+                if (false == (otherDoc instanceof FieldDoc)) return false;
+                FieldDoc thisFieldDoc = (FieldDoc) thisDoc;
+                FieldDoc otherFieldDoc = (FieldDoc) otherDoc;
+                if (thisFieldDoc.fields.length != otherFieldDoc.fields.length) return false;
+                for (int f = 0; f < thisFieldDoc.fields.length; f++) {
+                    if (false == thisFieldDoc.fields[f].equals(otherFieldDoc.fields[f])) return false;
+                }
+            }
+        }
+        return searchHits.equals(other.searchHits);
+    }
+
+    @Override
+    protected int doHashCode() {
+        int hashCode = from;
+        hashCode = 31 * hashCode + size;
+        hashCode = 31 * hashCode + topDocs.totalHits;
+        for (int d = 0; d < topDocs.scoreDocs.length; d++) {
+            ScoreDoc doc = topDocs.scoreDocs[d];
+            hashCode = 31 * hashCode + doc.doc;
+            hashCode = 31 * hashCode + Float.floatToIntBits(doc.score);
+            hashCode = 31 * hashCode + doc.shardIndex;
+            if (doc instanceof FieldDoc) {
+                FieldDoc fieldDoc = (FieldDoc) doc;
+                hashCode = 31 * hashCode + Arrays.hashCode(fieldDoc.fields);
+            }
+        }
+        hashCode = 31 * hashCode + searchHits.hashCode();
+        return hashCode;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index d63ca865674..a8ec235c563 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -34,8 +34,6 @@ import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
@@ -60,7 +58,6 @@ import java.util.Set;
 
 public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHitsAggregationBuilder> {
     public static final String NAME = "top_hits";
-    private static final InternalAggregation.Type TYPE = new Type(NAME);
 
     private int from = 0;
     private int size = 3;
@@ -75,14 +72,14 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
     private FetchSourceContext fetchSourceContext;
 
     public TopHitsAggregationBuilder(String name) {
-        super(name, TYPE);
+        super(name);
     }
 
     /**
      * Read from a stream.
      */
     public TopHitsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE);
+        super(in);
         explain = in.readBoolean();
         fetchSourceContext = in.readOptionalWriteable(FetchSourceContext::new);
         if (in.readBoolean()) {
@@ -526,7 +523,8 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
 
     @Override
     public TopHitsAggregationBuilder subAggregations(Builder subFactories) {
-        throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + type + "] cannot accept sub-aggregations");
+        throw new AggregationInitializationException("Aggregator [" + name + "] of type ["
+                + getType() + "] cannot accept sub-aggregations");
     }
 
     @Override
@@ -548,7 +546,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         } else {
             optionalSort = SortBuilder.buildSort(sorts, context.getQueryShardContext());
         }
-        return new TopHitsAggregatorFactory(name, type, from, size, explain, version, trackScores, optionalSort, highlightBuilder,
+        return new TopHitsAggregatorFactory(name, from, size, explain, version, trackScores, optionalSort, highlightBuilder,
                 storedFieldsContext, fieldDataFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData);
     }
 
@@ -605,31 +603,31 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FROM_FIELD)) {
+                if (SearchSourceBuilder.FROM_FIELD.match(currentFieldName)) {
                     factory.from(parser.intValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SIZE_FIELD)) {
+                } else if (SearchSourceBuilder.SIZE_FIELD.match(currentFieldName)) {
                     factory.size(parser.intValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.VERSION_FIELD)) {
+                } else if (SearchSourceBuilder.VERSION_FIELD.match(currentFieldName)) {
                     factory.version(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.EXPLAIN_FIELD)) {
+                } else if (SearchSourceBuilder.EXPLAIN_FIELD.match(currentFieldName)) {
                     factory.explain(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.TRACK_SCORES_FIELD)) {
+                } else if (SearchSourceBuilder.TRACK_SCORES_FIELD.match(currentFieldName)) {
                     factory.trackScores(parser.booleanValue());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context.parser()));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                } else if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
+                } else if (SearchSourceBuilder.STORED_FIELDS_FIELD.match(currentFieldName)) {
                     factory.storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     factory.sort(parser.text());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context.parser()));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELDS_FIELD)) {
+                if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
+                } else if (SearchSourceBuilder.SCRIPT_FIELDS_FIELD.match(currentFieldName)) {
                     List<ScriptField> scriptFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         String scriptFieldName = parser.currentName();
@@ -641,10 +639,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                                 if (token == XContentParser.Token.FIELD_NAME) {
                                     currentFieldName = parser.currentName();
                                 } else if (token.isValue()) {
-                                    if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELD)) {
-                                        script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                                    } else if (context.getParseFieldMatcher().match(currentFieldName,
-                                            SearchSourceBuilder.IGNORE_FAILURE_FIELD)) {
+                                    if (SearchSourceBuilder.SCRIPT_FIELD.match(currentFieldName)) {
+                                        script = Script.parse(parser, context.getDefaultScriptLanguage());
+                                    } else if (SearchSourceBuilder.IGNORE_FAILURE_FIELD.match(currentFieldName)) {
                                         ignoreFailure = parser.booleanValue();
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(),
@@ -652,8 +649,8 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                                                 parser.getTokenLocation());
                                     }
                                 } else if (token == XContentParser.Token.START_OBJECT) {
-                                    if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SCRIPT_FIELD)) {
-                                        script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                                    if (SearchSourceBuilder.SCRIPT_FIELD.match(currentFieldName)) {
+                                        script = Script.parse(parser, context.getDefaultScriptLanguage());
                                     } else {
                                         throw new ParsingException(parser.getTokenLocation(),
                                                 "Unknown key for a " + token + " in [" + currentFieldName + "].",
@@ -671,9 +668,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                         }
                     }
                     factory.scriptFields(scriptFields);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.HIGHLIGHT_FIELD)) {
+                } else if (SearchSourceBuilder.HIGHLIGHT_FIELD.match(currentFieldName)) {
                     factory.highlighter(HighlightBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
                 } else {
@@ -682,10 +679,10 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
 
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                if (SearchSourceBuilder.STORED_FIELDS_FIELD.match(currentFieldName)) {
                     factory.storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD)) {
+                } else if (SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.match(currentFieldName)) {
                     List<String> fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -696,11 +693,11 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                         }
                     }
                     factory.fieldDataFields(fieldDataFields);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.SORT_FIELD)) {
+                } else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName)) {
                     List<SortBuilder<?>> sorts = SortBuilder.fromXContent(context);
                     factory.sorts(sorts);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
-                    factory.fetchSource(FetchSourceContext.parse(context.parser()));
+                } else if (SearchSourceBuilder._SOURCE_FIELD.match(currentFieldName)) {
+                    factory.fetchSource(FetchSourceContext.fromXContent(context.parser()));
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -736,7 +733,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
index d3647f454b2..92041747730 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
@@ -122,7 +122,13 @@ public class TopHitsAggregator extends MetricsAggregator {
                     // In the QueryPhase we don't need this protection, because it is build into the IndexSearcher,
                     // but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
                     topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
-                    TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
+                    TopDocsCollector<?> topLevelCollector;
+                    if (sort == null) {
+                        topLevelCollector = TopScoreDocCollector.create(topN);
+                    } else {
+                        topLevelCollector = TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(),
+                                subSearchContext.trackScores());
+                    }
                     collectors = new TopDocsAndLeafCollector(topLevelCollector);
                     collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx);
                     collectors.leafCollector.setScorer(scorer);
@@ -170,8 +176,8 @@ public class TopHitsAggregator extends MetricsAggregator {
                     searchHitFields.sortValues(fieldDoc.fields, subSearchContext.sort().formats);
                 }
             }
-            topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(),
-                metaData());
+            topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(),
+                    pipelineAggregators(), metaData());
         }
         return topHits;
     }
@@ -184,7 +190,8 @@ public class TopHitsAggregator extends MetricsAggregator {
         } else {
             topDocs = Lucene.EMPTY_TOP_DOCS;
         }
-        return new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, InternalSearchHits.empty(), pipelineAggregators(), metaData());
+        return new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, InternalSearchHits.empty(),
+                pipelineAggregators(), metaData());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
index 511c5104d92..6a41cc97f8e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.tophits;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
@@ -52,12 +51,12 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
     private final List<ScriptFieldsContext.ScriptField> scriptFields;
     private final FetchSourceContext fetchSourceContext;
 
-    public TopHitsAggregatorFactory(String name, Type type, int from, int size, boolean explain, boolean version, boolean trackScores,
+    public TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores,
             Optional<SortAndFormats> sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext,
             List<String> docValueFields, List<ScriptFieldsContext.ScriptField> scriptFields, FetchSourceContext fetchSourceContext,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactories, Map<String, Object> metaData)
             throws IOException {
-        super(name, type, context, parent, subFactories, metaData);
+        super(name, context, parent, subFactories, metaData);
         this.from = from;
         this.size = size;
         this.explain = explain;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
index bfd186df5f2..25dd4e2e123 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
@@ -28,6 +28,7 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * An internal implementation of {@link ValueCount}.
@@ -88,4 +89,15 @@ public class InternalValueCount extends InternalNumericMetricsAggregation.Single
     public String toString() {
         return "count[" + value + "]";
     }
+
+    @Override
+    protected int doHashCode() {
+        return Objects.hash(value);
+    }
+
+    @Override
+    protected boolean doEquals(Object obj) {
+        InternalValueCount that = (InternalValueCount) obj;
+        return Objects.equals(this.value, that.value);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
index fdd83d32aa0..50916b4063c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
@@ -27,7 +27,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
@@ -40,7 +39,6 @@ import java.io.IOException;
 
 public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, ValueCountAggregationBuilder> {
     public static final String NAME = "value_count";
-    public static final Type TYPE = new Type(NAME);
 
     private static final ObjectParser<ValueCountAggregationBuilder, QueryParseContext> PARSER;
     static {
@@ -53,14 +51,14 @@ public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder
     }
 
     public ValueCountAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public ValueCountAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.ANY);
+        super(in, ValuesSourceType.ANY);
     }
 
     @Override
@@ -76,7 +74,7 @@ public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder
     @Override
     protected ValueCountAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig<ValuesSource> config,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new ValueCountAggregatorFactory(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        return new ValueCountAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -95,7 +93,7 @@ public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
index 2dc40e1d8ea..80c8001b93c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.metrics.valuecount;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
@@ -35,9 +34,9 @@ import java.util.Map;
 
 public class ValueCountAggregatorFactory extends ValuesSourceAggregatorFactory<ValuesSource, ValueCountAggregatorFactory> {
 
-    public ValueCountAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, SearchContext context,
+    public ValueCountAggregatorFactory(String name, ValuesSourceConfig<ValuesSource> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        super(name, config, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index 4fb2ed91401..8d28195d551 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -171,4 +171,8 @@ public abstract class AbstractPipelineAggregationBuilder<PAB extends AbstractPip
 
     protected abstract boolean doEquals(Object obj);
 
+    @Override
+    public String getType() {
+        return type;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
index 98b5b67b7cf..9fc1136968e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
@@ -52,7 +52,7 @@ public class BucketHelpers {
      * "insert_zeros": empty buckets will be filled with zeros for all metrics
      * "ignore": empty buckets will simply be ignored
      */
-    public static enum GapPolicy {
+    public enum GapPolicy {
         INSERT_ZEROS((byte) 0, "insert_zeros"), SKIP((byte) 1, "skip");
 
         /**
@@ -65,7 +65,7 @@ public class BucketHelpers {
         public static GapPolicy parse(QueryParseContext context, String text, XContentLocation tokenLocation) {
             GapPolicy result = null;
             for (GapPolicy policy : values()) {
-                if (context.getParseFieldMatcher().match(text, policy.parseField)) {
+                if (policy.parseField.match(text)) {
                     if (result == null) {
                         result = policy;
                     } else {
@@ -87,7 +87,7 @@ public class BucketHelpers {
         private final byte id;
         private final ParseField parseField;
 
-        private GapPolicy(byte id, String name) {
+        GapPolicy(byte id, String name) {
             this.id = id;
             this.parseField = new ParseField(name);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
index 9dee002ca29..e7954174aa3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
@@ -58,17 +58,17 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
                 } else {
                     parseToken(pipelineAggregatorName, parser, context, currentFieldName, token, params);
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
index 81df16f2bf7..cea7d011367 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
@@ -138,7 +138,7 @@ public class PercentilesBucketPipelineAggregationBuilder
         @Override
         protected boolean token(XContentParser parser, QueryParseContext context, String field,
                                 XContentParser.Token token, Map<String, Object> params) throws IOException {
-            if (context.getParseFieldMatcher().match(field, PERCENTS_FIELD) && token == XContentParser.Token.START_ARRAY) {
+            if (PERCENTS_FIELD.match(field) && token == XContentParser.Token.START_ARRAY) {
                 DoubleArrayList percents = new DoubleArrayList(10);
                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
                     percents.add(parser.doubleValue());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
index b7fa49267dc..dfa28c3dd27 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
@@ -46,7 +46,7 @@ public class ExtendedStatsBucketParser extends BucketMetricsParser {
     @Override
     protected boolean token(XContentParser parser, QueryParseContext context, String field,
                             XContentParser.Token token, Map<String, Object> params) throws IOException {
-        if (context.getParseFieldMatcher().match(field, SIGMA) && token == XContentParser.Token.VALUE_NUMBER) {
+        if (SIGMA.match(field) && token == XContentParser.Token.VALUE_NUMBER) {
             params.put(SIGMA.getPreferredName(), parser.doubleValue());
             return true;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
index e04f26654ea..7c96b04b156 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
@@ -170,21 +170,21 @@ public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggr
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPathsMap = new HashMap<>();
                     bucketsPathsMap.put("_value", parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -199,9 +199,9 @@ public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggr
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     Map<String, Object> map = parser.map();
                     bucketsPathsMap = new HashMap<>();
                     for (Map.Entry<String, Object> entry : map.entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
index 008b0217c65..059d2fec037 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
@@ -92,8 +92,7 @@ public class BucketScriptPipelineAggregator extends PipelineAggregator {
         InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg = (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
         List<? extends Bucket> buckets = originalAgg.getBuckets();
 
-        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap());
+        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS);
         List newBuckets = new ArrayList<>();
         for (Bucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
index 877be6ea54f..078f50a978f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
@@ -135,19 +135,19 @@ public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAg
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPathsMap = new HashMap<>();
                     bucketsPathsMap.put("_value", parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -162,9 +162,9 @@ public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAg
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
-                    script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
+                    script = Script.parse(parser, context.getDefaultScriptLanguage());
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     Map<String, Object> map = parser.map();
                     bucketsPathsMap = new HashMap<>();
                     for (Map.Entry<String, Object> entry : map.entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
index eabbad7213a..f45ebd1c5a3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
@@ -86,8 +86,7 @@ public class BucketSelectorPipelineAggregator extends PipelineAggregator {
                 (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
         List<? extends Bucket> buckets = originalAgg.getBuckets();
 
-        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap());
+        CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS);
         List newBuckets = new ArrayList<>();
         for (Bucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
index a83787f1365..5ac185990b4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
@@ -141,16 +141,16 @@ public class CumulativeSumPipelineAggregationBuilder extends AbstractPipelineAgg
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
index ded2d110206..bb2a1c01a50 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
@@ -207,20 +207,20 @@ public class DerivativePipelineAggregationBuilder extends AbstractPipelineAggreg
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) {
+                if (FORMAT_FIELD.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH_FIELD)) {
+                } else if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY_FIELD)) {
+                } else if (GAP_POLICY_FIELD.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentFieldName)) {
                     units = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH_FIELD)) {
+                if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
index f0aa1f81126..bc973ad442f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
@@ -322,13 +322,13 @@ public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregatio
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, WINDOW)) {
+                if (WINDOW.match(currentFieldName)) {
                     window = parser.intValue();
                     if (window <= 0) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive, "
                                 + "non-zero integer.  Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "].");
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, PREDICT)) {
+                } else if (PREDICT.match(currentFieldName)) {
                     predict = parser.intValue();
                     if (predict <= 0) {
                         throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive integer."
@@ -339,20 +339,20 @@ public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregatio
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MODEL)) {
+                } else if (MODEL.match(currentFieldName)) {
                     model = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
@@ -364,14 +364,14 @@ public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregatio
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, SETTINGS)) {
+                if (SETTINGS.match(currentFieldName)) {
                     settings = parser.map();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-                if (context.getParseFieldMatcher().match(currentFieldName, MINIMIZE)) {
+                if (MINIMIZE.match(currentFieldName)) {
                     minimize = parser.booleanValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
@@ -403,11 +403,10 @@ public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregatio
             factory.predict(predict);
         }
         if (model != null) {
-            MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, context.getParseFieldMatcher(),
-                    parser.getTokenLocation());
+            MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, parser.getTokenLocation());
             MovAvgModel movAvgModel;
             try {
-                movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window(), context.getParseFieldMatcher());
+                movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window());
             } catch (ParseException exception) {
                 throw new ParsingException(parser.getTokenLocation(), "Could not parse settings for model [" + model + "].", exception);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
index c7e6b0e8980..26fb0333b18 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -127,8 +126,7 @@ public class EwmaModel extends MovAvgModel {
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             checkUnrecognizedParams(settings);
             return new EwmaModel(alpha);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
index d8a591972ec..18193337385 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -191,8 +190,7 @@ public class HoltLinearModel extends MovAvgModel {
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
 
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
index 2130faf8674..92b2e4d3ea2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
@@ -23,7 +23,6 @@ package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -59,17 +58,16 @@ public class HoltWintersModel extends MovAvgModel {
          * Parse a string SeasonalityType into the byte enum
          *
          * @param text                SeasonalityType in string format (e.g. "add")
-         * @param parseFieldMatcher   Matcher for field names
          * @return                    SeasonalityType enum
          */
         @Nullable
-        public static SeasonalityType parse(String text, ParseFieldMatcher parseFieldMatcher) {
+        public static SeasonalityType parse(String text) {
             if (text == null) {
                 return null;
             }
             SeasonalityType result = null;
             for (SeasonalityType policy : values()) {
-                if (parseFieldMatcher.match(text, policy.parseField)) {
+                if (policy.parseField.match(text)) {
                     result = policy;
                     break;
                 }
@@ -379,8 +377,7 @@ public class HoltWintersModel extends MovAvgModel {
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
 
             double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA);
             double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA);
@@ -399,7 +396,7 @@ public class HoltWintersModel extends MovAvgModel {
                 Object value = settings.get("type");
                 if (value != null) {
                     if (value instanceof String) {
-                        seasonalityType = SeasonalityType.parse((String)value, parseFieldMatcher);
+                        seasonalityType = SeasonalityType.parse((String)value);
                         settings.remove("type");
                     } else {
                         throw new ParseException("Parameter [type] must be a String, type `"
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
index 089f3a430ca..3eed0bf603b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -106,8 +105,7 @@ public class LinearModel extends MovAvgModel {
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             checkUnrecognizedParams(settings);
             return new LinearModel();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
index 0837eca38bd..f64117236d6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -143,11 +142,10 @@ public abstract class MovAvgModel implements NamedWriteable, ToXContent {
          * @param settings           Map of settings, extracted from the request
          * @param pipelineName       Name of the parent pipeline agg
          * @param windowSize         Size of the window for this moving avg
-         * @param parseFieldMatcher  Matcher for field names
          * @return                   A fully built moving average model
          */
         public abstract MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName,
-                                          int windowSize, ParseFieldMatcher parseFieldMatcher) throws ParseException;
+                                          int windowSize) throws ParseException;
 
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
index 14544881883..e30a59d2887 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.pipeline.movavg.models;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -99,8 +98,7 @@ public class SimpleModel extends MovAvgModel {
 
     public static final AbstractModelParser PARSER = new AbstractModelParser() {
         @Override
-        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize,
-                                 ParseFieldMatcher parseFieldMatcher) throws ParseException {
+        public MovAvgModel parse(@Nullable Map<String, Object> settings, String pipelineName, int windowSize) throws ParseException {
             checkUnrecognizedParams(settings);
             return new SimpleModel();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
index f20a4f8da42..0acd4c7f1b7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
@@ -162,18 +162,18 @@ public class SerialDiffPipelineAggregationBuilder extends AbstractPipelineAggreg
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FORMAT)) {
+                if (FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                } else if (BUCKETS_PATH.match(currentFieldName)) {
                     bucketsPaths = new String[] { parser.text() };
-                } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
+                } else if (GAP_POLICY.match(currentFieldName)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                if (context.getParseFieldMatcher().match(currentFieldName, LAG)) {
+                if (LAG.match(currentFieldName)) {
                     lag = parser.intValue(true);
                     if (lag <= 0) {
                         throw new ParsingException(parser.getTokenLocation(),
@@ -186,7 +186,7 @@ public class SerialDiffPipelineAggregationBuilder extends AbstractPipelineAggreg
                             "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
+                if (BUCKETS_PATH.match(currentFieldName)) {
                     List<String> paths = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         String path = parser.text();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
index 28a4bd2567c..4a01b67b780 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java
@@ -80,7 +80,7 @@ public enum MissingValues {
     }
 
     public static ValuesSource.Numeric replaceMissing(final ValuesSource.Numeric valuesSource, final Number missing) {
-        final boolean missingIsFloat = missing.longValue() != (long) missing.doubleValue();
+        final boolean missingIsFloat = missing.doubleValue() % 1 != 0;
         final boolean isFloatingPoint = valuesSource.isFloatingPoint() || missingIsFloat;
         return new ValuesSource.Numeric() {
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
index 4c18103a39c..318540e3e58 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
@@ -95,7 +95,7 @@ public enum ValueType implements Writeable {
     private final byte id;
     private String preferredName;
 
-    private ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType,
+    ValueType(byte id, String description, String preferredName, ValuesSourceType valuesSourceType,
             Class<? extends IndexFieldData> fieldDataType, DocValueFormat defaultFormat) {
         this.id = id;
         this.description = description;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index 124b7ba71f4..f3de52ae653 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -318,7 +318,7 @@ public abstract class ValuesSource {
                 private final SortedNumericDocValues longValues;
                 private final LeafSearchScript script;
 
-                public LongValues(SortedNumericDocValues values, LeafSearchScript script) {
+                LongValues(SortedNumericDocValues values, LeafSearchScript script) {
                     this.longValues = values;
                     this.script = script;
                 }
@@ -346,7 +346,7 @@ public abstract class ValuesSource {
                 private final SortedNumericDoubleValues doubleValues;
                 private final LeafSearchScript script;
 
-                public DoubleValues(SortedNumericDoubleValues values, LeafSearchScript script) {
+                DoubleValues(SortedNumericDoubleValues values, LeafSearchScript script) {
                     this.doubleValues = values;
                     this.script = script;
                 }
@@ -462,7 +462,7 @@ public abstract class ValuesSource {
             private final SortedBinaryDocValues bytesValues;
             private final LeafSearchScript script;
 
-            public BytesValues(SortedBinaryDocValues bytesValues, LeafSearchScript script) {
+            BytesValues(SortedBinaryDocValues bytesValues, LeafSearchScript script) {
                 this.bytesValues = bytesValues;
                 this.script = script;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 859fbb12eb1..45b6afe43ee 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
 
@@ -40,28 +39,29 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
     public abstract static class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
             extends ValuesSourceAggregationBuilder<VS, AB> {
 
-        protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-            super(name, type, valuesSourceType, targetValueType);
+        protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            super(name, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that does not serialize its targetValueType. This should be used by most subclasses.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
-            super(in, type, valuesSourceType, targetValueType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
+            super(in, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
          * {@link #serializeTargetValueType()} to return true.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-            super(in, type, valuesSourceType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+            super(in, valuesSourceType);
         }
 
         @Override
         public AB subAggregations(Builder subFactories) {
-            throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + type + "] cannot accept sub-aggregations");
+            throw new AggregationInitializationException("Aggregator [" + name + "] of type ["
+                    + getType() + "] cannot accept sub-aggregations");
         }
     }
 
@@ -75,8 +75,8 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
     private DateTimeZone timeZone = null;
     protected ValuesSourceConfig<VS> config;
 
-    protected ValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-        super(name, type);
+    protected ValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        super(name);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
         }
@@ -87,9 +87,9 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
     /**
      * Read an aggregation from a stream that does not serialize its targetValueType. This should be used by most subclasses.
      */
-    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
             throws IOException {
-        super(in, type);
+        super(in);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = targetValueType;
@@ -100,8 +100,8 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
      * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
      * {@link #serializeTargetValueType()} to return true.
      */
-    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-        super(in, type);
+    protected ValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+        super(in);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
index b455096aaa9..28d82f4cafd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.support;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 import org.joda.time.DateTimeZone;
@@ -36,9 +35,9 @@ public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource, AF
 
     protected ValuesSourceConfig<VS> config;
 
-    public ValuesSourceAggregatorFactory(String name, Type type, ValuesSourceConfig<VS> config, SearchContext context,
+    public ValuesSourceAggregatorFactory(String name, ValuesSourceConfig<VS> config, SearchContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
+        super(name, context, parent, subFactoriesBuilder, metaData);
         this.config = config;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
index 4116d30a658..7b174d789f4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java
@@ -35,30 +35,30 @@ public final class ValuesSourceParserHelper {
     public static void declareAnyFields(
             ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource, ?>, QueryParseContext> objectParser,
             boolean scriptable, boolean formattable) {
-        declareFields(objectParser, scriptable, formattable, false, ValuesSourceType.ANY, null);
+        declareFields(objectParser, scriptable, formattable, false, null);
     }
 
     public static void declareNumericFields(
             ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, ?>, QueryParseContext> objectParser,
             boolean scriptable, boolean formattable, boolean timezoneAware) {
-        declareFields(objectParser, scriptable, formattable, timezoneAware, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        declareFields(objectParser, scriptable, formattable, timezoneAware, ValueType.NUMERIC);
     }
 
     public static void declareBytesFields(
             ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, ?>, QueryParseContext> objectParser,
             boolean scriptable, boolean formattable) {
-        declareFields(objectParser, scriptable, formattable, false, ValuesSourceType.BYTES, ValueType.STRING);
+        declareFields(objectParser, scriptable, formattable, false, ValueType.STRING);
     }
 
     public static void declareGeoFields(
             ObjectParser<? extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, ?>, QueryParseContext> objectParser,
             boolean scriptable, boolean formattable) {
-        declareFields(objectParser, scriptable, formattable, false, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        declareFields(objectParser, scriptable, formattable, false, ValueType.GEOPOINT);
     }
 
     private static <VS extends ValuesSource> void declareFields(
             ObjectParser<? extends ValuesSourceAggregationBuilder<VS, ?>, QueryParseContext> objectParser,
-            boolean scriptable, boolean formattable, boolean timezoneAware, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            boolean scriptable, boolean formattable, boolean timezoneAware, ValueType targetValueType) {
 
 
         objectParser.declareField(ValuesSourceAggregationBuilder::field, XContentParser::text,
@@ -67,25 +67,26 @@ public final class ValuesSourceParserHelper {
         objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText,
                 new ParseField("missing"), ObjectParser.ValueType.VALUE);
 
+        objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> {
+            ValueType valueType = ValueType.resolveForScript(p.text());
+            if (targetValueType != null && valueType.isNotA(targetValueType)) {
+                throw new ParsingException(p.getTokenLocation(),
+                        "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type ["
+                                + valueType + "]. It can only work on value of type ["
+                                + targetValueType + "]");
+            }
+            return valueType;
+        }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING);
+
         if (formattable) {
             objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text,
                     new ParseField("format"), ObjectParser.ValueType.STRING);
         }
 
         if (scriptable) {
-            objectParser.declareField(ValuesSourceAggregationBuilder::script, org.elasticsearch.script.Script::parse,
+            objectParser.declareField(ValuesSourceAggregationBuilder::script,
+                    (parser, context) -> Script.parse(parser, context.getDefaultScriptLanguage()),
                     Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING);
-
-            objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> {
-                ValueType valueType = ValueType.resolveForScript(p.text());
-                if (targetValueType != null && valueType.isNotA(targetValueType)) {
-                    throw new ParsingException(p.getTokenLocation(),
-                            "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type ["
-                                    + valueType + "]. It can only work on value of type ["
-                                    + targetValueType + "]");
-                }
-                return valueType;
-            }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING);
         }
 
         if (timezoneAware) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
index 06bf998d8da..503cec39b8d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java
@@ -50,15 +50,10 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
             resize(0);
         }
 
-        else if (value instanceof Number) {
-            resize(1);
-            values[0] = ((Number) value).longValue();
-        }
-
         else if (value.getClass().isArray()) {
             resize(Array.getLength(value));
             for (int i = 0; i < count(); ++i) {
-                values[i] = ((Number) Array.get(value, i)).longValue();
+                values[i] = toLongValue(Array.get(value, i));
             }
         }
 
@@ -66,18 +61,33 @@ public class ScriptLongValues extends SortingNumericDocValues implements ScorerA
             resize(((Collection<?>) value).size());
             int i = 0;
             for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) {
-                values[i] = ((Number) it.next()).longValue();
+                values[i] = toLongValue(it.next());
             }
             assert i == count();
         }
 
         else {
-            throw new AggregationExecutionException("Unsupported script value [" + value + "]");
+            resize(1);
+            values[0] = toLongValue(value);
         }
 
         sort();
     }
 
+    private static long toLongValue(Object o) {
+        if (o instanceof Number) {
+            return ((Number) o).longValue();
+        } else if (o instanceof Boolean) {
+            // We do expose boolean fields as boolean in scripts, however aggregations still expect
+            // that scripts return the same internal representation as regular fields, so boolean
+            // values in scripts need to be converted to a number, and the value formatter will
+            // make sure of using true/false in the key_as_string field
+            return ((Boolean) o).booleanValue() ? 1L : 0L;
+        } else {
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number");
+        }
+    }
+
     @Override
     public void setScorer(Scorer scorer) {
         script.setScorer(scorer);
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index f7ffcfea137..41948dce625 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.builder;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -37,12 +38,10 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.SearchExtBuilder;
-import org.elasticsearch.search.SearchExtParser;
-import org.elasticsearch.search.SearchExtRegistry;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -56,7 +55,6 @@ import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -103,13 +101,13 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXT_FIELD = new ParseField("ext");
     public static final ParseField PROFILE_FIELD = new ParseField("profile");
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
+    public static final ParseField COLLAPSE = new ParseField("collapse");
     public static final ParseField SLICE = new ParseField("slice");
     public static final ParseField ALL_FIELDS_FIELDS = new ParseField("all_fields");
 
-    public static SearchSourceBuilder fromXContent(QueryParseContext context, AggregatorParsers aggParsers,
-            Suggesters suggesters, SearchExtRegistry searchExtRegistry) throws IOException {
+    public static SearchSourceBuilder fromXContent(QueryParseContext context) throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
-        builder.parseXContent(context, aggParsers, suggesters, searchExtRegistry);
+        builder.parseXContent(context);
         return builder;
     }
 
@@ -173,6 +171,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     private boolean profile = false;
 
+    private CollapseBuilder collapse = null;
 
     /**
      * Constructs a new search source builder.
@@ -221,6 +220,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         profile = in.readBoolean();
         searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
         sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
+        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            collapse = in.readOptionalWriteable(CollapseBuilder::new);
+        }
     }
 
     @Override
@@ -269,6 +271,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         out.writeBoolean(profile);
         out.writeOptionalWriteable(searchAfterBuilder);
         out.writeOptionalWriteable(sliceBuilder);
+        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            out.writeOptionalWriteable(collapse);
+        }
     }
 
     /**
@@ -518,6 +523,16 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return sliceBuilder;
     }
 
+
+    public CollapseBuilder collapse() {
+        return collapse;
+    }
+
+    public SearchSourceBuilder collapse(CollapseBuilder collapse) {
+        this.collapse = collapse;
+        return this;
+    }
+
     /**
      * Add an aggregation to perform as part of the search.
      */
@@ -878,7 +893,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     }
 
     /**
-     * Create a shallow copy of this source replaced {@link #queryBuilder}, {@link #postQueryBuilder}, and {@linkplain slice}. Used by
+     * Create a shallow copy of this source replaced {@link #queryBuilder}, {@link #postQueryBuilder}, and {@link #sliceBuilder}. Used by
      * {@link #rewrite(QueryShardContext)} and {@link #copyWithNewSlice(SliceBuilder)}.
      */
     private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder postQueryBuilder, SliceBuilder slice) {
@@ -908,18 +923,16 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         rewrittenBuilder.timeout = timeout;
         rewrittenBuilder.trackScores = trackScores;
         rewrittenBuilder.version = version;
+        rewrittenBuilder.collapse = collapse;
         return rewrittenBuilder;
     }
 
     /**
      * Parse some xContent into this SearchSourceBuilder, overwriting any values specified in the xContent. Use this if you need to set up
      * different defaults than a regular SearchSourceBuilder would have and use
-     * {@link #fromXContent(QueryParseContext, AggregatorParsers, Suggesters, SearchExtRegistry)} if you have normal defaults.
+     * {@link #fromXContent(QueryParseContext)} if you have normal defaults.
      */
-    public void parseXContent(QueryParseContext context, AggregatorParsers aggParsers,
-                              Suggesters suggesters, SearchExtRegistry searchExtRegistry)
-        throws IOException {
-
+    public void parseXContent(QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token = parser.currentToken();
         String currentFieldName = null;
@@ -931,32 +944,32 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (context.getParseFieldMatcher().match(currentFieldName, FROM_FIELD)) {
+                if (FROM_FIELD.match(currentFieldName)) {
                     from = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SIZE_FIELD)) {
+                } else if (SIZE_FIELD.match(currentFieldName)) {
                     size = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TIMEOUT_FIELD)) {
+                } else if (TIMEOUT_FIELD.match(currentFieldName)) {
                     timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TERMINATE_AFTER_FIELD)) {
+                } else if (TERMINATE_AFTER_FIELD.match(currentFieldName)) {
                     terminateAfter = parser.intValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
+                } else if (MIN_SCORE_FIELD.match(currentFieldName)) {
                     minScore = parser.floatValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, VERSION_FIELD)) {
+                } else if (VERSION_FIELD.match(currentFieldName)) {
                     version = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, EXPLAIN_FIELD)) {
+                } else if (EXPLAIN_FIELD.match(currentFieldName)) {
                     explain = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, TRACK_SCORES_FIELD)) {
+                } else if (TRACK_SCORES_FIELD.match(currentFieldName)) {
                     trackScores = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context.parser());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (STORED_FIELDS_FIELD.match(currentFieldName)) {
                     storedFieldsContext =
                         StoredFieldsContext.fromXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sort(parser.text());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, PROFILE_FIELD)) {
+                } else if (PROFILE_FIELD.match(currentFieldName)) {
                     profile = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                } else if (FIELDS_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(), "Deprecated field [" +
                         SearchSourceBuilder.FIELDS_FIELD + "] used, expected [" +
                         SearchSourceBuilder.STORED_FIELDS_FIELD + "] instead");
@@ -965,18 +978,18 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (context.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     queryBuilder = context.parseInnerQueryBuilder();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, POST_FILTER_FIELD)) {
+                } else if (POST_FILTER_FIELD.match(currentFieldName)) {
                     postQueryBuilder = context.parseInnerQueryBuilder();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context.parser());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (SCRIPT_FIELDS_FIELD.match(currentFieldName)) {
                     scriptFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         scriptFields.add(new ScriptField(context));
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, INDICES_BOOST_FIELD)) {
+                } else if (INDICES_BOOST_FIELD.match(currentFieldName)) {
                     DEPRECATION_LOGGER.deprecated(
                         "Object format in indices_boost is deprecated, please use array format instead");
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -989,28 +1002,26 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                                 " in [" + currentFieldName + "].", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, AGGREGATIONS_FIELD)
-                        || context.getParseFieldMatcher().match(currentFieldName, AGGS_FIELD)) {
-                    aggregations = aggParsers.parseAggregators(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, HIGHLIGHT_FIELD)) {
+                } else if (AGGREGATIONS_FIELD.match(currentFieldName)
+                        || AGGS_FIELD.match(currentFieldName)) {
+                    aggregations = AggregatorFactories.parseAggregators(context);
+                } else if (HIGHLIGHT_FIELD.match(currentFieldName)) {
                     highlightBuilder = HighlightBuilder.fromXContent(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SUGGEST_FIELD)) {
-                    suggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (SUGGEST_FIELD.match(currentFieldName)) {
+                    suggestBuilder = SuggestBuilder.fromXContent(context.parser());
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sorts = new ArrayList<>(SortBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
+                } else if (RESCORE_FIELD.match(currentFieldName)) {
                     rescoreBuilders = new ArrayList<>();
                     rescoreBuilders.add(RescoreBuilder.parseFromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, EXT_FIELD)) {
+                } else if (EXT_FIELD.match(currentFieldName)) {
                     extBuilders = new ArrayList<>();
                     String extSectionName = null;
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             extSectionName = parser.currentName();
                         } else {
-                            SearchExtParser searchExtParser = searchExtRegistry.lookup(extSectionName,
-                                    context.getParseFieldMatcher(), parser.getTokenLocation());
-                            SearchExtBuilder searchExtBuilder = searchExtParser.fromXContent(parser);
+                            SearchExtBuilder searchExtBuilder = parser.namedObject(SearchExtBuilder.class, extSectionName, null);
                             if (searchExtBuilder.getWriteableName().equals(extSectionName) == false) {
                                 throw new IllegalStateException("The parsed [" + searchExtBuilder.getClass().getName() + "] object has a "
                                         + "different writeable name compared to the name of the section that it was parsed from: found ["
@@ -1019,16 +1030,18 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                             extBuilders.add(searchExtBuilder);
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SLICE)) {
+                } else if (SLICE.match(currentFieldName)) {
                     sliceBuilder = SliceBuilder.fromXContent(context);
+                } else if (COLLAPSE.match(currentFieldName)) {
+                    collapse = CollapseBuilder.fromXContent(context);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                if (STORED_FIELDS_FIELD.match(currentFieldName)) {
                     storedFieldsContext = StoredFieldsContext.fromXContent(STORED_FIELDS_FIELD.getPreferredName(), context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, DOCVALUE_FIELDS_FIELD)) {
+                } else if (DOCVALUE_FIELDS_FIELD.match(currentFieldName)) {
                     docValueFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -1038,18 +1051,18 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                                 "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, INDICES_BOOST_FIELD)) {
+                } else if (INDICES_BOOST_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         indexBoosts.add(new IndexBoost(context));
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     sorts = new ArrayList<>(SortBuilder.fromXContent(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
+                } else if (RESCORE_FIELD.match(currentFieldName)) {
                     rescoreBuilders = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         rescoreBuilders.add(RescoreBuilder.parseFromXContent(context));
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, STATS_FIELD)) {
+                } else if (STATS_FIELD.match(currentFieldName)) {
                     stats = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -1059,11 +1072,11 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                                     "] in [" + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
-                    fetchSourceContext = FetchSourceContext.parse(context.parser());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SEARCH_AFTER)) {
-                    searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                } else if (_SOURCE_FIELD.match(currentFieldName)) {
+                    fetchSourceContext = FetchSourceContext.fromXContent(context.parser());
+                } else if (SEARCH_AFTER.match(currentFieldName)) {
+                    searchAfterBuilder = SearchAfterBuilder.fromXContent(parser);
+                } else if (FIELDS_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(), "The field [" +
                         SearchSourceBuilder.FIELDS_FIELD + "] is no longer supported, please use [" +
                         SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
@@ -1212,6 +1225,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             }
             builder.endObject();
         }
+
+        if (collapse != null) {
+            builder.field(COLLAPSE.getPreferredName(), collapse);
+        }
     }
 
     public static class IndexBoost implements Writeable, ToXContent {
@@ -1341,17 +1358,17 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     if (token == XContentParser.Token.FIELD_NAME) {
                         currentFieldName = parser.currentName();
                     } else if (token.isValue()) {
-                        if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) {
-                            script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
-                        } else if (context.getParseFieldMatcher().match(currentFieldName, IGNORE_FAILURE_FIELD)) {
+                        if (SCRIPT_FIELD.match(currentFieldName)) {
+                            script = Script.parse(parser, context.getDefaultScriptLanguage());
+                        } else if (IGNORE_FAILURE_FIELD.match(currentFieldName)) {
                             ignoreFailure = parser.booleanValue();
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName
                                     + "].", parser.getTokenLocation());
                         }
                     } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELD)) {
-                            script = Script.parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage());
+                        if (SCRIPT_FIELD.match(currentFieldName)) {
+                            script = Script.parse(parser, context.getDefaultScriptLanguage());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName
                                     + "].", parser.getTokenLocation());
@@ -1416,7 +1433,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder,
                 indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size,
                 sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version,
-                profile, extBuilders);
+                profile, extBuilders, collapse);
     }
 
     @Override
@@ -1452,6 +1469,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 && Objects.equals(trackScores, other.trackScores)
                 && Objects.equals(version, other.version)
                 && Objects.equals(profile, other.profile)
-                && Objects.equals(extBuilders, other.extBuilders);
+                && Objects.equals(extBuilders, other.extBuilders)
+                && Objects.equals(collapse, other.collapse);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
new file mode 100644
index 00000000000..75a7d6dadf3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.SearchContextException;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A builder that enables field collapsing on search request.
+ */
+public class CollapseBuilder extends ToXContentToBytes implements Writeable {
+    public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    private static final ObjectParser<CollapseBuilder, QueryParseContext> PARSER =
+        new ObjectParser<>("collapse", CollapseBuilder::new);
+
+    static {
+        PARSER.declareString(CollapseBuilder::setField, FIELD_FIELD);
+        PARSER.declareObject(CollapseBuilder::setInnerHits,
+            (p, c) -> InnerHitBuilder.fromXContent(c), INNER_HITS_FIELD);
+    }
+
+    private String field;
+    private InnerHitBuilder innerHit;
+
+    private CollapseBuilder() {}
+
+    /**
+     * Public constructor
+     * @param field The name of the field to collapse on
+     */
+    public CollapseBuilder(String field) {
+        Objects.requireNonNull(field, "field must be non-null");
+        this.field = field;
+    }
+
+    public CollapseBuilder(StreamInput in) throws IOException {
+        this.field = in.readString();
+        this.innerHit = in.readOptionalWriteable(InnerHitBuilder::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(field);
+        out.writeOptionalWriteable(innerHit);
+    }
+
+    public static CollapseBuilder fromXContent(QueryParseContext context) throws IOException {
+        CollapseBuilder builder = PARSER.parse(context.parser(), new CollapseBuilder(), context);
+        return builder;
+    }
+
+    private CollapseBuilder setField(String field) {
+        if (Strings.isEmpty(field)) {
+            throw new IllegalArgumentException("field name is null or empty");
+        }
+        this.field = field;
+        return this;
+    }
+
+    public CollapseBuilder setInnerHits(InnerHitBuilder innerHit) {
+        this.innerHit = innerHit;
+        return this;
+    }
+
+    /**
+     * The name of the field to collapse against
+     */
+    public String getField() {
+        return this.field;
+    }
+
+    /**
+     * The inner hit options to expand the collapsed results
+     */
+    public InnerHitBuilder getInnerHit() {
+        return this.innerHit;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder);
+        builder.endObject();
+        return builder;
+    }
+
+    private void innerToXContent(XContentBuilder builder) throws IOException {
+        builder.field(FIELD_FIELD.getPreferredName(), field);
+        if (innerHit != null) {
+            builder.field(INNER_HITS_FIELD.getPreferredName(), innerHit);
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        CollapseBuilder that = (CollapseBuilder) o;
+
+        if (field != null ? !field.equals(that.field) : that.field != null) return false;
+        return innerHit != null ? innerHit.equals(that.innerHit) : that.innerHit == null;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.field, this.innerHit);
+    }
+
+    public CollapseContext build(SearchContext context) {
+        if (context.scrollContext() != null) {
+            throw new SearchContextException(context, "cannot use `collapse` in a scroll context");
+        }
+        if (context.searchAfter() != null) {
+            throw new SearchContextException(context, "cannot use `collapse` in conjunction with `search_after`");
+        }
+        if (context.rescore() != null && context.rescore().isEmpty() == false) {
+            throw new SearchContextException(context, "cannot use `collapse` in conjunction with `rescore`");
+        }
+
+        MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(field);
+        if (fieldType == null) {
+            throw new SearchContextException(context, "no mapping found for `" + field + "` in order to collapse on");
+        }
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType == false &&
+            fieldType instanceof NumberFieldMapper.NumberFieldType == false) {
+            throw new SearchContextException(context, "unknown type for collapse field `" + field +
+                "`, only keywords and numbers are accepted");
+        }
+
+        if (fieldType.hasDocValues() == false) {
+            throw new SearchContextException(context, "cannot collapse on field `" + field + "` without `doc_values`");
+        }
+        if (fieldType.indexOptions() == IndexOptions.NONE && innerHit != null) {
+            throw new SearchContextException(context, "cannot expand `inner_hits` for collapse field `"
+                + field + "`, " + "only indexed field can retrieve `inner_hits`");
+        }
+        return new CollapseContext(fieldType, innerHit);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
new file mode 100644
index 00000000000..d0ea2154ab3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/collapse/CollapseContext.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+
+import java.io.IOException;
+
+/**
+ * Context used for field collapsing
+ */
+public class CollapseContext {
+    private final MappedFieldType fieldType;
+    private final InnerHitBuilder innerHit;
+
+    public CollapseContext(MappedFieldType fieldType, InnerHitBuilder innerHit) {
+        this.fieldType = fieldType;
+        this.innerHit = innerHit;
+    }
+
+    /** The field type used for collapsing **/
+    public MappedFieldType getFieldType() {
+        return fieldType;
+    }
+
+
+    /** The inner hit options to expand the collapsed results **/
+    public InnerHitBuilder getInnerHit() {
+        return innerHit;
+    }
+
+    public CollapsingTopDocsCollector<?> createTopDocs(Sort sort, int topN, boolean trackMaxScore) throws IOException {
+        if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) {
+            return CollapsingTopDocsCollector.createKeyword(fieldType.name(), sort, topN, trackMaxScore);
+        } else if (fieldType instanceof NumberFieldMapper.NumberFieldType) {
+            return CollapsingTopDocsCollector.createNumeric(fieldType.name(), sort, topN, trackMaxScore);
+        } else {
+            throw new IllegalStateException("unknown type for collapse field " + fieldType.name() +
+                ", only keywords and numbers are accepted");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/collapse/ExpandCollapseSearchResponseListener.java b/core/src/main/java/org/elasticsearch/search/collapse/ExpandCollapseSearchResponseListener.java
new file mode 100644
index 00000000000..9a45b7a9fec
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/collapse/ExpandCollapseSearchResponseListener.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.collapse;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.internal.InternalSearchHit;
+
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.function.BiConsumer;
+
+/**
+ * A search response listener that intercepts the search response and expands collapsed hits
+ * using the {@link CollapseBuilder#innerHit} options.
+ */
+public class ExpandCollapseSearchResponseListener implements BiConsumer<SearchRequest, SearchResponse> {
+    private final Client client;
+
+    public ExpandCollapseSearchResponseListener(Client client) {
+        this.client = Objects.requireNonNull(client);
+    }
+
+    @Override
+    public void accept(SearchRequest searchRequest, SearchResponse searchResponse) {
+        if (searchRequest.source() == null) {
+            return ;
+        }
+        CollapseBuilder collapseBuilder = searchRequest.source().collapse();
+        if (collapseBuilder == null || collapseBuilder.getInnerHit() == null) {
+            return ;
+        }
+        for (SearchHit hit : searchResponse.getHits()) {
+            InternalSearchHit internalHit = (InternalSearchHit) hit;
+            BoolQueryBuilder groupQuery = new BoolQueryBuilder();
+            Object collapseValue = internalHit.field(collapseBuilder.getField()).getValue();
+            if (collapseValue != null) {
+                groupQuery.filter(QueryBuilders.matchQuery(collapseBuilder.getField(), collapseValue));
+            } else {
+                groupQuery.mustNot(QueryBuilders.existsQuery(collapseBuilder.getField()));
+            }
+            QueryBuilder origQuery = searchRequest.source().query();
+            if (origQuery != null) {
+                groupQuery.must(origQuery);
+            }
+            SearchSourceBuilder sourceBuilder = createGroupSearchBuilder(collapseBuilder.getInnerHit())
+                .query(groupQuery);
+            SearchRequest groupRequest = new SearchRequest(searchRequest.indices())
+                .types(searchRequest.types())
+                .source(sourceBuilder);
+            SearchResponse groupResponse = client.search(groupRequest).actionGet();
+            SearchHits innerHits = groupResponse.getHits();
+            if (internalHit.getInnerHits() == null) {
+                internalHit.setInnerHits(new HashMap<>(1));
+            }
+            internalHit.getInnerHits().put(collapseBuilder.getInnerHit().getName(), innerHits);
+        }
+    }
+
+    private SearchSourceBuilder createGroupSearchBuilder(InnerHitBuilder options) {
+        SearchSourceBuilder groupSource = new SearchSourceBuilder();
+        groupSource.from(options.getFrom());
+        groupSource.size(options.getSize());
+        if (options.getSorts() != null) {
+            options.getSorts().forEach(groupSource::sort);
+        }
+        if (options.getFetchSourceContext() != null) {
+            if (options.getFetchSourceContext().includes() == null && options.getFetchSourceContext().excludes() == null) {
+                groupSource.fetchSource(options.getFetchSourceContext().fetchSource());
+            } else {
+                groupSource.fetchSource(options.getFetchSourceContext().includes(),
+                    options.getFetchSourceContext().excludes());
+            }
+        }
+        if (options.getDocValueFields() != null) {
+            options.getDocValueFields().forEach(groupSource::docValueField);
+        }
+        if (options.getStoredFieldsContext() != null && options.getStoredFieldsContext().fieldNames() != null) {
+            options.getStoredFieldsContext().fieldNames().forEach(groupSource::storedField);
+        }
+        if (options.getScriptFields() != null) {
+            for (SearchSourceBuilder.ScriptField field : options.getScriptFields()) {
+                groupSource.scriptField(field.fieldName(), field.script());
+            }
+        }
+        if (options.getHighlightBuilder() != null) {
+            groupSource.highlighter(options.getHighlightBuilder());
+        }
+        groupSource.explain(options.isExplain());
+        groupSource.trackScores(options.isTrackScores());
+        return groupSource;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java b/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
index d762540caab..f7bfb6a8052 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/AggregatedDfs.java
@@ -85,10 +85,10 @@ public class AggregatedDfs implements Streamable {
         out.writeVInt(termStatistics.size());
 
         for (ObjectObjectCursor<Term, TermStatistics> c : termStatistics()) {
-            Term term = (Term) c.key;
+            Term term = c.key;
             out.writeString(term.field());
             out.writeBytesRef(term.bytes());
-            TermStatistics stats = (TermStatistics) c.value;
+            TermStatistics stats = c.value;
             out.writeBytesRef(stats.term());
             out.writeVLong(stats.docFreq());
             out.writeVLong(DfsSearchResult.addOne(stats.totalTermFreq()));
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
index befce94a9e4..9444f38d80e 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java
@@ -27,6 +27,7 @@ import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 
 /**
@@ -38,6 +39,15 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.collapse() != null) {
+            // retrieve the `doc_value` associated with the collapse field
+            String name = context.collapse().getFieldType().name();
+            if (context.docValueFieldsContext() == null) {
+                context.docValueFieldsContext(new DocValueFieldsContext(Collections.singletonList(name)));
+            } else if (context.docValueFieldsContext().fields().contains(name) == false) {
+                context.docValueFieldsContext().fields().add(name);
+            }
+        }
         if (context.docValueFieldsContext() == null) {
             return;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
index 1eec405502e..c86342f690f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.fetch.subphase;
 
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -55,10 +54,6 @@ public class FetchSourceContext implements Writeable, ToXContent {
     private final String[] excludes;
     private Function<Map<String, ?>, Map<String, Object>> filter;
 
-    public static FetchSourceContext parse(XContentParser parser) throws IOException {
-        return fromXContent(parser, ParseFieldMatcher.STRICT);
-    }
-
     public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) {
         this.fetchSource = fetchSource;
         this.includes = includes == null ? Strings.EMPTY_ARRAY : includes;
@@ -101,9 +96,9 @@ public class FetchSourceContext implements Writeable, ToXContent {
 
         String source = request.param("_source");
         if (source != null) {
-            if (Booleans.isExplicitTrue(source)) {
+            if (Booleans.isTrue(source)) {
                 fetchSource = true;
-            } else if (Booleans.isExplicitFalse(source)) {
+            } else if (Booleans.isFalse(source)) {
                 fetchSource = false;
             } else {
                 source_includes = Strings.splitStringByCommaToArray(source);
@@ -127,7 +122,7 @@ public class FetchSourceContext implements Writeable, ToXContent {
         return null;
     }
 
-    public static FetchSourceContext fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static FetchSourceContext fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token = parser.currentToken();
         boolean fetchSource = true;
         String[] includes = Strings.EMPTY_ARRAY;
@@ -148,7 +143,7 @@ public class FetchSourceContext implements Writeable, ToXContent {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token == XContentParser.Token.START_ARRAY) {
-                    if (parseFieldMatcher.match(currentFieldName, INCLUDES_FIELD)) {
+                    if (INCLUDES_FIELD.match(currentFieldName)) {
                         List<String> includesList = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_STRING) {
@@ -159,7 +154,7 @@ public class FetchSourceContext implements Writeable, ToXContent {
                             }
                         }
                         includes = includesList.toArray(new String[includesList.size()]);
-                    } else if (parseFieldMatcher.match(currentFieldName, EXCLUDES_FIELD)) {
+                    } else if (EXCLUDES_FIELD.match(currentFieldName)) {
                         List<String> excludesList = new ArrayList<>();
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             if (token == XContentParser.Token.VALUE_STRING) {
@@ -175,9 +170,9 @@ public class FetchSourceContext implements Writeable, ToXContent {
                                 + " in [" + currentFieldName + "].", parser.getTokenLocation());
                     }
                 } else if (token == XContentParser.Token.VALUE_STRING) {
-                    if (parseFieldMatcher.match(currentFieldName, INCLUDES_FIELD)) {
+                    if (INCLUDES_FIELD.match(currentFieldName)) {
                         includes = new String[] {parser.text()};
-                    } else if (parseFieldMatcher.match(currentFieldName, EXCLUDES_FIELD)) {
+                    } else if (EXCLUDES_FIELD.match(currentFieldName)) {
                         excludes = new String[] {parser.text()};
                     } else {
                         throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
index cb8310975fc..5cd19096beb 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhase.java
@@ -63,6 +63,10 @@ public final class ParentFieldSubFetchPhase implements FetchSubPhase {
     public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) {
         try {
             SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name());
+            if (docValues == null) {
+                // hit has no _parent field.
+                return null;
+            }
             BytesRef parentId = docValues.get(docId);
             return parentId.length > 0 ? parentId.utf8ToString() : null;
         } catch (IOException e) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
index 8a16d4c6eb0..e25e7b74941 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/CustomQueryScorer.java
@@ -66,11 +66,11 @@ public final class CustomQueryScorer extends QueryScorer {
 
     private static class CustomWeightedSpanTermExtractor extends WeightedSpanTermExtractor {
 
-        public CustomWeightedSpanTermExtractor() {
+        CustomWeightedSpanTermExtractor() {
             super();
         }
 
-        public CustomWeightedSpanTermExtractor(String defaultField) {
+        CustomWeightedSpanTermExtractor(String defaultField) {
             super(defaultField);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
index 2015ac2b6e1..7ff2147868a 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightField.java
@@ -34,6 +34,8 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A field highlighted with its highlighted fragments.
  */
@@ -121,13 +123,16 @@ public class HighlightField implements ToXContent, Streamable {
     }
 
     public static HighlightField fromXContent(XContentParser parser) throws IOException {
-        XContentParser.Token token = parser.nextToken();
-        assert token == XContentParser.Token.FIELD_NAME;
+        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
         String fieldName = parser.currentName();
         Text[] fragments = null;
-        token = parser.nextToken();
+        XContentParser.Token token = parser.nextToken();
         if (token == XContentParser.Token.START_ARRAY) {
-            fragments = parseValues(parser);
+            List<Text> values = new ArrayList<>();
+            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                values.add(new Text(parser.text()));
+            }
+            fragments = values.toArray(new Text[values.size()]);
         } else if (token == XContentParser.Token.VALUE_NULL) {
             fragments = null;
         } else {
@@ -137,14 +142,6 @@ public class HighlightField implements ToXContent, Streamable {
         return new HighlightField(fieldName, fragments);
     }
 
-    private static Text[] parseValues(XContentParser parser) throws IOException {
-        List<Text> values = new ArrayList<>();
-        while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-            values.add(new Text(parser.text()));
-        }
-        return values.toArray(new Text[values.size()]);
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field(name);
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
index 7ed50c7a1dd..330fb908cc5 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PostingsHighlighter.java
@@ -25,7 +25,7 @@ import org.apache.lucene.search.highlight.Encoder;
 import org.apache.lucene.search.postingshighlight.CustomPassageFormatter;
 import org.apache.lucene.search.postingshighlight.CustomPostingsHighlighter;
 import org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator;
-import org.apache.lucene.search.postingshighlight.Snippet;
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.text.Text;
@@ -139,14 +139,14 @@ public class PostingsHighlighter implements Highlighter {
         return fieldMapper.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
     }
 
-    private static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
+    static String mergeFieldValues(List<Object> fieldValues, char valuesSeparator) {
         //postings highlighter accepts all values in a single string, as offsets etc. need to match with content
         //loaded from stored fields, we merge all values using a proper separator
         String rawValue = Strings.collectionToDelimitedString(fieldValues, String.valueOf(valuesSeparator));
         return rawValue.substring(0, Math.min(rawValue.length(), Integer.MAX_VALUE - 1));
     }
 
-    private static List<Snippet> filterSnippets(List<Snippet> snippets, int numberOfFragments) {
+    static List<Snippet> filterSnippets(List<Snippet> snippets, int numberOfFragments) {
 
         //We need to filter the snippets as due to no_match_size we could have
         //either highlighted snippets or non highlighted ones and we don't want to mix those up
@@ -181,11 +181,11 @@ public class PostingsHighlighter implements Highlighter {
         return filteredSnippets;
     }
 
-    private static class HighlighterEntry {
+    static class HighlighterEntry {
         Map<FieldMapper, MapperHighlighterEntry> mappers = new HashMap<>();
     }
 
-    private static class MapperHighlighterEntry {
+    static class MapperHighlighterEntry {
         final CustomPassageFormatter passageFormatter;
 
         private MapperHighlighterEntry(CustomPassageFormatter passageFormatter) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
new file mode 100644
index 00000000000..4afce9700e5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.subphase.highlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.highlight.Encoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.uhighlight.CustomPassageFormatter;
+import org.apache.lucene.search.uhighlight.CustomUnifiedHighlighter;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter.filterSnippets;
+import static org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter.mergeFieldValues;
+
+public class UnifiedHighlighter implements Highlighter {
+    private static final String CACHE_KEY = "highlight-unified";
+
+    @Override
+    public boolean canHighlight(FieldMapper fieldMapper) {
+        return true;
+    }
+
+    @Override
+    public HighlightField highlight(HighlighterContext highlighterContext) {
+        FieldMapper fieldMapper = highlighterContext.mapper;
+        SearchContextHighlight.Field field = highlighterContext.field;
+        SearchContext context = highlighterContext.context;
+        FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
+
+        if (!hitContext.cache().containsKey(CACHE_KEY)) {
+            hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
+        }
+
+        HighlighterEntry highlighterEntry = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
+        MapperHighlighterEntry mapperHighlighterEntry = highlighterEntry.mappers.get(fieldMapper);
+
+        if (mapperHighlighterEntry == null) {
+            Encoder encoder = field.fieldOptions().encoder().equals("html") ?
+                HighlightUtils.Encoders.HTML : HighlightUtils.Encoders.DEFAULT;
+            CustomPassageFormatter passageFormatter =
+                new CustomPassageFormatter(field.fieldOptions().preTags()[0],
+                    field.fieldOptions().postTags()[0], encoder);
+            mapperHighlighterEntry = new MapperHighlighterEntry(passageFormatter);
+        }
+
+        List<Snippet> snippets = new ArrayList<>();
+        int numberOfFragments;
+        try {
+            Analyzer analyzer =
+                context.mapperService().documentMapper(hitContext.hit().type()).mappers().indexAnalyzer();
+            List<Object> fieldValues = HighlightUtils.loadFieldValues(field, fieldMapper, context, hitContext);
+            fieldValues = fieldValues.stream().map(obj -> {
+                if (obj instanceof BytesRef) {
+                    return fieldMapper.fieldType().valueForDisplay(obj).toString();
+                } else {
+                    return obj;
+                }
+            }).collect(Collectors.toList());
+            IndexSearcher searcher = new IndexSearcher(hitContext.reader());
+            CustomUnifiedHighlighter highlighter;
+            if (field.fieldOptions().numberOfFragments() == 0) {
+                // we use a control char to separate values, which is the only char that the custom break iterator
+                // breaks the text on, so we don't lose the distinction between the different values of a field and we
+                // get back a snippet per value
+                String fieldValue = mergeFieldValues(fieldValues, HighlightUtils.NULL_SEPARATOR);
+                org.apache.lucene.search.postingshighlight.CustomSeparatorBreakIterator breakIterator =
+                    new org.apache.lucene.search.postingshighlight
+                        .CustomSeparatorBreakIterator(HighlightUtils.NULL_SEPARATOR);
+                highlighter =
+                    new CustomUnifiedHighlighter(searcher, analyzer, mapperHighlighterEntry.passageFormatter,
+                        breakIterator, fieldValue, field.fieldOptions().noMatchSize() > 0);
+                numberOfFragments = fieldValues.size(); // we are highlighting the whole content, one snippet per value
+            } else {
+                //using paragraph separator we make sure that each field value holds a discrete passage for highlighting
+                String fieldValue = mergeFieldValues(fieldValues, HighlightUtils.PARAGRAPH_SEPARATOR);
+                highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
+                    mapperHighlighterEntry.passageFormatter, null, fieldValue, field.fieldOptions().noMatchSize() > 0);
+                numberOfFragments = field.fieldOptions().numberOfFragments();
+            }
+            if (field.fieldOptions().requireFieldMatch()) {
+                final String fieldName = highlighterContext.fieldName;
+                highlighter.setFieldMatcher((name) -> fieldName.equals(name));
+            } else {
+                highlighter.setFieldMatcher((name) -> true);
+            }
+            Snippet[] fieldSnippets = highlighter.highlightField(highlighterContext.fieldName,
+                highlighterContext.query, hitContext.docId(), numberOfFragments);
+            for (Snippet fieldSnippet : fieldSnippets) {
+                if (Strings.hasText(fieldSnippet.getText())) {
+                    snippets.add(fieldSnippet);
+                }
+            }
+        } catch (IOException e) {
+            throw new FetchPhaseExecutionException(context,
+                "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
+        }
+
+        snippets = filterSnippets(snippets, field.fieldOptions().numberOfFragments());
+
+        if (field.fieldOptions().scoreOrdered()) {
+            //let's sort the snippets by score if needed
+            CollectionUtil.introSort(snippets, (o1, o2) -> Double.compare(o2.getScore(), o1.getScore()));
+        }
+
+        String[] fragments = new String[snippets.size()];
+        for (int i = 0; i < fragments.length; i++) {
+            fragments[i] = snippets.get(i).getText();
+        }
+
+        if (fragments.length > 0) {
+            return new HighlightField(highlighterContext.fieldName, Text.convertFromStringArray(fragments));
+        }
+        return null;
+    }
+
+    static class HighlighterEntry {
+        Map<FieldMapper, MapperHighlighterEntry> mappers = new HashMap<>();
+    }
+
+    static class MapperHighlighterEntry {
+        final CustomPassageFormatter passageFormatter;
+
+        private MapperHighlighterEntry(CustomPassageFormatter passageFormatter) {
+            this.passageFormatter = passageFormatter;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java b/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java
index fd684380526..915e745ddf4 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/AliasFilter.java
@@ -21,10 +21,13 @@ package org.elasticsearch.search.internal;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 
@@ -62,7 +65,14 @@ public final class AliasFilter implements Writeable {
         if (reparseAliases) {
             // we are processing a filter received from a 5.0 node - we need to reparse this on the executing node
             final IndexMetaData indexMetaData = context.getIndexSettings().getIndexMetaData();
-            return ShardSearchRequest.parseAliasFilter(context::newParseContext, indexMetaData, aliases);
+            /* Being static, parseAliasFilter doesn't have access to whatever guts it needs to parse a query. Instead of passing in a bunch
+             * of dependencies we pass in a function that can perform the parsing. */
+            CheckedFunction<byte[], QueryBuilder, IOException> filterParser = bytes -> {
+                try (XContentParser parser = XContentFactory.xContent(bytes).createParser(context.getXContentRegistry(), bytes)) {
+                    return context.newParseContext(parser).parseInnerQueryBuilder();
+                }
+            };
+            return ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliases);
         }
         return filter;
     }
@@ -118,4 +128,13 @@ public final class AliasFilter implements Writeable {
     public int hashCode() {
         return Objects.hash(aliases, filter, reparseAliases);
     }
+
+    @Override
+    public String toString() {
+        return "AliasFilter{" +
+            "aliases=" + Arrays.toString(aliases) +
+            ", filter=" + filter +
+            ", reparseAliases=" + reparseAliases +
+            '}';
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index ad86b0b69fa..adc528728e0 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -25,7 +25,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -40,6 +39,7 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
@@ -63,8 +63,6 @@ public abstract class FilteredSearchContext extends SearchContext {
     private final SearchContext in;
 
     public FilteredSearchContext(SearchContext in) {
-        //inner_hits in percolator ends up with null inner search context
-        super(in == null ? ParseFieldMatcher.EMPTY : in.parseFieldMatcher());
         this.in = in;
     }
 
@@ -531,4 +529,14 @@ public abstract class FilteredSearchContext extends SearchContext {
     public boolean isCancelled() {
         return in.isCancelled();
     }
+
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        return in.collapse(collapse);
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return in.collapse();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index 82324e3edf7..3487312b727 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -20,12 +20,10 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.search.Explanation;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -38,6 +36,9 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
@@ -49,6 +50,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -62,16 +64,19 @@ import static org.elasticsearch.common.lucene.Lucene.readExplanation;
 import static org.elasticsearch.common.lucene.Lucene.writeExplanation;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.parseStoredFieldsValue;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 import static org.elasticsearch.search.fetch.subphase.highlight.HighlightField.readHighlightField;
 import static org.elasticsearch.search.internal.InternalSearchHitField.readSearchHitField;
 
 public class InternalSearchHit implements SearchHit {
 
-    private static final Object[] EMPTY_SORT_VALUES = new Object[0];
-
     private transient int docId;
 
-    private float score = Float.NEGATIVE_INFINITY;
+    private static final float DEFAULT_SCORE = Float.NEGATIVE_INFINITY;
+    private float score = DEFAULT_SCORE;
 
     private Text id;
     private Text type;
@@ -86,7 +91,7 @@ public class InternalSearchHit implements SearchHit {
 
     private Map<String, HighlightField> highlightFields = null;
 
-    private Object[] sortValues = EMPTY_SORT_VALUES;
+    private SearchSortValues sortValues = SearchSortValues.EMPTY;
 
     private String[] matchedQueries = Strings.EMPTY_ARRAY;
 
@@ -95,6 +100,8 @@ public class InternalSearchHit implements SearchHit {
     @Nullable
     private SearchShardTarget shard;
 
+    private transient String index;
+
     private Map<String, Object> sourceAsMap;
     private byte[] sourceAsBytes;
 
@@ -109,20 +116,17 @@ public class InternalSearchHit implements SearchHit {
     }
 
     public InternalSearchHit(int docId, String id, Text type, Map<String, SearchHitField> fields) {
-        this.docId = docId;
+        this(docId, id, type, null, fields);
+    }
+
+    public InternalSearchHit(int nestedTopDocId, String id, Text type, InternalNestedIdentity nestedIdentity, Map<String, SearchHitField> fields) {
+        this.docId = nestedTopDocId;
         if (id != null) {
             this.id = new Text(id);
         } else {
             this.id = null;
         }
         this.type = type;
-        this.fields = fields;
-    }
-
-    public InternalSearchHit(int nestedTopDocId, String id, Text type, InternalNestedIdentity nestedIdentity, Map<String, SearchHitField> fields) {
-        this.docId = nestedTopDocId;
-        this.id = new Text(id);
-        this.type = type;
         this.nestedIdentity = nestedIdentity;
         this.fields = fields;
     }
@@ -131,15 +135,6 @@ public class InternalSearchHit implements SearchHit {
         return this.docId;
     }
 
-    public void shardTarget(SearchShardTarget shardTarget) {
-        this.shard = shardTarget;
-        if (innerHits != null) {
-            for (InternalSearchHits searchHits : innerHits.values()) {
-                searchHits.shardTarget(shardTarget);
-            }
-        }
-    }
-
     public void score(float score) {
         this.score = score;
     }
@@ -170,7 +165,7 @@ public class InternalSearchHit implements SearchHit {
 
     @Override
     public String index() {
-        return shard.index();
+        return this.index;
     }
 
     @Override
@@ -235,14 +230,6 @@ public class InternalSearchHit implements SearchHit {
         return sourceRef();
     }
 
-    /**
-     * Internal source representation, might be compressed....
-     */
-    public BytesReference internalSourceRef() {
-        return source;
-    }
-
-
     @Override
     public byte[] source() {
         if (source == null) {
@@ -324,10 +311,6 @@ public class InternalSearchHit implements SearchHit {
         this.fields = fields;
     }
 
-    public Map<String, HighlightField> internalHighlightFields() {
-        return highlightFields;
-    }
-
     @Override
     public Map<String, HighlightField> highlightFields() {
         return highlightFields == null ? emptyMap() : highlightFields;
@@ -343,17 +326,16 @@ public class InternalSearchHit implements SearchHit {
     }
 
     public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
-        this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
-        for (int i = 0; i < sortValues.length; ++i) {
-            if (this.sortValues[i] instanceof BytesRef) {
-                this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
-            }
-        }
+        sortValues(new SearchSortValues(sortValues, sortValueFormats));
+    }
+
+    public void sortValues(SearchSortValues sortValues) {
+        this.sortValues = sortValues;
     }
 
     @Override
     public Object[] sortValues() {
-        return sortValues;
+        return sortValues.sortValues();
     }
 
     @Override
@@ -387,6 +369,9 @@ public class InternalSearchHit implements SearchHit {
 
     public void shard(SearchShardTarget target) {
         this.shard = target;
+        if (target != null) {
+            this.index = target.getIndex();
+        }
     }
 
     public void matchedQueries(String[] matchedQueries) {
@@ -428,6 +413,8 @@ public class InternalSearchHit implements SearchHit {
         static final String DESCRIPTION = "description";
         static final String DETAILS = "details";
         static final String INNER_HITS = "inner_hits";
+        static final String _SHARD = "_shard";
+        static final String _NODE = "_node";
     }
 
     // public because we render hit as part of completion suggestion option
@@ -450,14 +437,14 @@ public class InternalSearchHit implements SearchHit {
         // For inner_hit hits shard is null and that is ok, because the parent search hit has all this information.
         // Even if this was included in the inner_hit hits this would be the same, so better leave it out.
         if (explanation() != null && shard != null) {
-            builder.field("_shard", shard.shardId());
-            builder.field("_node", shard.nodeIdText());
+            builder.field(Fields._SHARD, shard.getShardId());
+            builder.field(Fields._NODE, shard.getNodeIdText());
         }
         if (nestedIdentity != null) {
             nestedIdentity.toXContent(builder, params);
         } else {
-            if (shard != null) {
-                builder.field(Fields._INDEX, shard.indexText());
+            if (index != null) {
+                builder.field(Fields._INDEX, index);
             }
             if (type != null) {
                 builder.field(Fields._TYPE, type);
@@ -479,7 +466,7 @@ public class InternalSearchHit implements SearchHit {
             builder.field(field.name(), value);
         }
         if (source != null) {
-            XContentHelper.writeRawField("_source", source, builder, params);
+            XContentHelper.writeRawField(SourceFieldMapper.NAME, source, builder, params);
         }
         if (!otherFields.isEmpty()) {
             builder.startObject(Fields.FIELDS);
@@ -499,13 +486,7 @@ public class InternalSearchHit implements SearchHit {
             }
             builder.endObject();
         }
-        if (sortValues != null && sortValues.length > 0) {
-            builder.startArray(Fields.SORT);
-            for (Object sortValue : sortValues) {
-                builder.value(sortValue);
-            }
-            builder.endArray();
-        }
+        sortValues.toXContent(builder, params);
         if (matchedQueries.length > 0) {
             builder.startArray(Fields.MATCHED_QUERIES);
             for (String matchedFilter : matchedQueries) {
@@ -529,6 +510,160 @@ public class InternalSearchHit implements SearchHit {
         return builder;
     }
 
+    public static InternalSearchHit fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token;
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        String currentFieldName = null;
+        String type = null, id = null;
+        String index = null;
+        float score = DEFAULT_SCORE;
+        long version = -1;
+        SearchSortValues sortValues = SearchSortValues.EMPTY;
+        InternalNestedIdentity nestedIdentity = null;
+        Map<String, HighlightField> highlightFields = new HashMap<>();
+        BytesReference parsedSource = null;
+        List<String> matchedQueries = new ArrayList<>();
+        Map<String, SearchHitField> fields = new HashMap<>();
+        Explanation explanation = null;
+        ShardId shardId = null;
+        String nodeId = null;
+        Map<String, InternalSearchHits> innerHits = null;
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (Fields._TYPE.equals(currentFieldName)) {
+                    type = parser.text();
+                } else if (Fields._INDEX.equals(currentFieldName)) {
+                    index = parser.text();
+                } else if (Fields._ID.equals(currentFieldName)) {
+                    id = parser.text();
+                } else if (Fields._SCORE.equals(currentFieldName)) {
+                     score = parser.floatValue();
+                } else if (Fields._VERSION.equals(currentFieldName)) {
+                    version = parser.longValue();
+                } else if (Fields._SHARD.equals(currentFieldName)) {
+                    shardId = ShardId.fromString(parser.text());
+                } else if (Fields._NODE.equals(currentFieldName)) {
+                   nodeId = parser.text();
+                } else if (MapperService.isMetadataField(currentFieldName)) {
+                    List<Object> values = new ArrayList<>();
+                    values.add(parseStoredFieldsValue(parser));
+                    fields.put(currentFieldName, new InternalSearchHitField(currentFieldName, values));
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.VALUE_NULL) {
+                if (Fields._SCORE.equals(currentFieldName)) {
+                     score = Float.NaN;
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (SourceFieldMapper.NAME.equals(currentFieldName)) {
+                    try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) {
+                        //the original document gets slightly modified: whitespaces or pretty printing are not preserved,
+                        //it all depends on the current builder settings
+                        builder.copyCurrentStructure(parser);
+                        parsedSource = builder.bytes();
+                    }
+                } else if (Fields.HIGHLIGHT.equals(currentFieldName)) {
+                    while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        HighlightField highlightField = HighlightField.fromXContent(parser);
+                        highlightFields.put(highlightField.getName(), highlightField);
+                    }
+                } else if (Fields.FIELDS.equals(currentFieldName)) {
+                    while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        String fieldName = parser.currentName();
+                        List<Object> values = new ArrayList<>();
+                        ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+                        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                            values.add(parseStoredFieldsValue(parser));
+                        }
+                        fields.put(fieldName, new InternalSearchHitField(fieldName, values));
+                    }
+                } else if (Fields._EXPLANATION.equals(currentFieldName)) {
+                    explanation = parseExplanation(parser);
+                } else if (Fields.INNER_HITS.equals(currentFieldName)) {
+                    innerHits = new HashMap<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        // parse the key
+                        ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+                        String name = parser.currentName();
+                        innerHits.put(name, InternalSearchHits.fromXContent(parser));
+                        parser.nextToken();
+                        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation);
+                    }
+                } else if (InternalNestedIdentity.Fields._NESTED.equals(currentFieldName)) {
+                    nestedIdentity = InternalNestedIdentity.fromXContent(parser);
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (Fields.SORT.equals(currentFieldName)) {
+                    sortValues = SearchSortValues.fromXContent(parser);
+                } else if (Fields.MATCHED_QUERIES.equals(currentFieldName)) {
+                    while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        matchedQueries.add(parser.text());
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        InternalSearchHit internalSearchHit = new InternalSearchHit(-1, id, new Text(type), nestedIdentity, Collections.emptyMap());
+        internalSearchHit.index = index;
+        internalSearchHit.score(score);
+        internalSearchHit.version(version);
+        internalSearchHit.sortValues(sortValues);
+        internalSearchHit.highlightFields(highlightFields);
+        internalSearchHit.sourceRef(parsedSource);
+        internalSearchHit.explanation(explanation);
+        internalSearchHit.setInnerHits(innerHits);
+        if (matchedQueries.size() > 0) {
+            internalSearchHit.matchedQueries(matchedQueries.toArray(new String[matchedQueries.size()]));
+        }
+        if (shardId != null && nodeId != null) {
+            internalSearchHit.shard(new SearchShardTarget(nodeId, shardId));
+        }
+        internalSearchHit.fields(fields);
+        return internalSearchHit;
+    }
+
+    private static Explanation parseExplanation(XContentParser parser) throws IOException {
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        XContentParser.Token token;
+        Float value = null;
+        String description = null;
+        List<Explanation> details = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, () -> parser.getTokenLocation());
+            String currentFieldName = parser.currentName();
+            token = parser.nextToken();
+            if (Fields.VALUE.equals(currentFieldName)) {
+                value = parser.floatValue();
+            } else if (Fields.DESCRIPTION.equals(currentFieldName)) {
+                description = parser.textOrNull();
+            } else if (Fields.DETAILS.equals(currentFieldName)) {
+                ensureExpectedToken(XContentParser.Token.START_ARRAY, token, () -> parser.getTokenLocation());
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    details.add(parseExplanation(parser));
+                }
+            } else {
+                throwUnknownField(currentFieldName, parser.getTokenLocation());
+            }
+        }
+        if (value == null) {
+            throw new ParsingException(parser.getTokenLocation(), "missing explanation value");
+        }
+        if (description == null) {
+            throw new ParsingException(parser.getTokenLocation(), "missing explanation description");
+        }
+        return Explanation.match(value, description, details);
+    }
+
     private void buildExplanation(XContentBuilder builder, Explanation explanation) throws IOException {
         builder.startObject();
         builder.field(Fields.VALUE, explanation.getValue());
@@ -603,34 +738,7 @@ public class InternalSearchHit implements SearchHit {
             this.highlightFields = unmodifiableMap(highlightFields);
         }
 
-        size = in.readVInt();
-        if (size > 0) {
-            sortValues = new Object[size];
-            for (int i = 0; i < sortValues.length; i++) {
-                byte type = in.readByte();
-                if (type == 0) {
-                    sortValues[i] = null;
-                } else if (type == 1) {
-                    sortValues[i] = in.readString();
-                } else if (type == 2) {
-                    sortValues[i] = in.readInt();
-                } else if (type == 3) {
-                    sortValues[i] = in.readLong();
-                } else if (type == 4) {
-                    sortValues[i] = in.readFloat();
-                } else if (type == 5) {
-                    sortValues[i] = in.readDouble();
-                } else if (type == 6) {
-                    sortValues[i] = in.readByte();
-                } else if (type == 7) {
-                    sortValues[i] = in.readShort();
-                } else if (type == 8) {
-                    sortValues[i] = in.readBoolean();
-                } else {
-                    throw new IOException("Can't match type [" + type + "]");
-                }
-            }
-        }
+        sortValues = new SearchSortValues(in);
 
         size = in.readVInt();
         if (size > 0) {
@@ -639,7 +747,8 @@ public class InternalSearchHit implements SearchHit {
                 matchedQueries[i] = in.readString();
             }
         }
-        shard = in.readOptionalWriteable(SearchShardTarget::new);
+        // we call the setter here because that also sets the local index parameter
+        shard(in.readOptionalWriteable(SearchShardTarget::new));
         size = in.readVInt();
         if (size > 0) {
             innerHits = new HashMap<>(size);
@@ -681,46 +790,7 @@ public class InternalSearchHit implements SearchHit {
                 highlightField.writeTo(out);
             }
         }
-
-        if (sortValues.length == 0) {
-            out.writeVInt(0);
-        } else {
-            out.writeVInt(sortValues.length);
-            for (Object sortValue : sortValues) {
-                if (sortValue == null) {
-                    out.writeByte((byte) 0);
-                } else {
-                    Class type = sortValue.getClass();
-                    if (type == String.class) {
-                        out.writeByte((byte) 1);
-                        out.writeString((String) sortValue);
-                    } else if (type == Integer.class) {
-                        out.writeByte((byte) 2);
-                        out.writeInt((Integer) sortValue);
-                    } else if (type == Long.class) {
-                        out.writeByte((byte) 3);
-                        out.writeLong((Long) sortValue);
-                    } else if (type == Float.class) {
-                        out.writeByte((byte) 4);
-                        out.writeFloat((Float) sortValue);
-                    } else if (type == Double.class) {
-                        out.writeByte((byte) 5);
-                        out.writeDouble((Double) sortValue);
-                    } else if (type == Byte.class) {
-                        out.writeByte((byte) 6);
-                        out.writeByte((Byte) sortValue);
-                    } else if (type == Short.class) {
-                        out.writeByte((byte) 7);
-                        out.writeShort((Short) sortValue);
-                    } else if (type == Boolean.class) {
-                        out.writeByte((byte) 8);
-                        out.writeBoolean((Boolean) sortValue);
-                    } else {
-                        throw new IOException("Can't handle sort field value of type [" + type + "]");
-                    }
-                }
-            }
-        }
+        sortValues.writeTo(out);
 
         if (matchedQueries.length == 0) {
             out.writeVInt(0);
@@ -742,6 +812,31 @@ public class InternalSearchHit implements SearchHit {
         }
     }
 
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        InternalSearchHit other = (InternalSearchHit) obj;
+        return Objects.equals(id, other.id)
+                && Objects.equals(type, other.type)
+                && Objects.equals(nestedIdentity, other.nestedIdentity)
+                && Objects.equals(version, other.version)
+                && Objects.equals(source, other.source)
+                && Objects.equals(fields, other.fields)
+                && Objects.equals(highlightFields(), other.highlightFields())
+                && Arrays.equals(matchedQueries, other.matchedQueries)
+                && Objects.equals(explanation, other.explanation)
+                && Objects.equals(shard, other.shard)
+                && Objects.equals(innerHits, other.innerHits);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, type, nestedIdentity, version, source, fields, highlightFields(), Arrays.hashCode(matchedQueries),
+                explanation, shard, innerHits);
+    }
+
     public static final class InternalNestedIdentity implements NestedIdentity, Writeable, ToXContent {
 
         private Text field;
@@ -807,7 +902,7 @@ public class InternalSearchHit implements SearchHit {
             return builder;
         }
 
-        private static final ConstructingObjectParser<InternalNestedIdentity, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+        private static final ConstructingObjectParser<InternalNestedIdentity, Void> PARSER = new ConstructingObjectParser<>(
                 "nested_identity",
                 ctorArgs -> new InternalNestedIdentity((String) ctorArgs[0], (int) ctorArgs[1], (InternalNestedIdentity) ctorArgs[2]));
         static {
@@ -817,7 +912,7 @@ public class InternalSearchHit implements SearchHit {
         }
 
         public static InternalNestedIdentity fromXContent(XContentParser parser) {
-            return PARSER.apply(parser, () -> ParseFieldMatcher.EMPTY);
+            return PARSER.apply(parser, null);
         }
 
         @Override
@@ -845,5 +940,4 @@ public class InternalSearchHit implements SearchHit {
             static final String _NESTED_OFFSET = "offset";
         }
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java
index 9214127ff06..d1f94969bd3 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHitField.java
@@ -28,6 +28,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 public class InternalSearchHitField implements SearchHitField {
 
@@ -109,4 +110,19 @@ public class InternalSearchHitField implements SearchHitField {
             out.writeGenericValue(value);
         }
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        InternalSearchHitField other = (InternalSearchHitField) obj;
+        return Objects.equals(name, other.name)
+                && Objects.equals(values, other.values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, values);
+    }
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
index 9b82c8783a1..834476b543a 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHits.java
@@ -22,14 +22,20 @@ package org.elasticsearch.search.internal;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.SearchShardTarget;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
 import static org.elasticsearch.search.internal.InternalSearchHit.readSearchHit;
 
 public class InternalSearchHits implements SearchHits {
@@ -132,6 +138,44 @@ public class InternalSearchHits implements SearchHits {
         return builder;
     }
 
+    public static InternalSearchHits fromXContent(XContentParser parser) throws IOException {
+        if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+            parser.nextToken();
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        }
+        XContentParser.Token token = parser.currentToken();
+        String currentFieldName = null;
+        List<InternalSearchHit> hits = new ArrayList<>();
+        long totalHits = 0;
+        float maxScore = 0f;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (Fields.TOTAL.equals(currentFieldName)) {
+                    totalHits = parser.longValue();
+                } else if (Fields.MAX_SCORE.equals(currentFieldName)) {
+                    maxScore = parser.floatValue();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.VALUE_NULL) {
+                if (Fields.MAX_SCORE.equals(currentFieldName)) {
+                    maxScore = Float.NaN; // NaN gets rendered as null-field
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    hits.add(InternalSearchHit.fromXContent(parser));
+                }
+            }
+        }
+        InternalSearchHits internalSearchHits = new InternalSearchHits(hits.toArray(new InternalSearchHit[hits.size()]), totalHits,
+                maxScore);
+        return internalSearchHits;
+    }
+
 
     public static InternalSearchHits readSearchHits(StreamInput in) throws IOException {
         InternalSearchHits hits = new InternalSearchHits();
@@ -165,4 +209,20 @@ public class InternalSearchHits implements SearchHits {
             }
         }
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        InternalSearchHits other = (InternalSearchHits) obj;
+        return Objects.equals(totalHits, other.totalHits)
+                && Objects.equals(maxScore, other.maxScore)
+                && Arrays.equals(hits, other.hits);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(totalHits, maxScore, Arrays.hashCode(hits));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index c89a4a4892b..4201dc0180b 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -26,7 +26,6 @@ import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.unit.TimeValue;
@@ -39,6 +38,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ObjectMapper;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -86,15 +86,8 @@ public abstract class SearchContext extends AbstractRefCounted implements Releas
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private InnerHitsContext innerHitsContext;
 
-    protected final ParseFieldMatcher parseFieldMatcher;
-
-    protected SearchContext(ParseFieldMatcher parseFieldMatcher) {
+    protected SearchContext() {
         super("search_context");
-        this.parseFieldMatcher = parseFieldMatcher;
-    }
-
-    public ParseFieldMatcher parseFieldMatcher() {
-        return parseFieldMatcher;
     }
 
     public abstract void setTask(SearchTask task);
@@ -249,6 +242,10 @@ public abstract class SearchContext extends AbstractRefCounted implements Releas
 
     public abstract FieldDoc searchAfter();
 
+    public abstract SearchContext collapse(CollapseContext collapse);
+
+    public abstract CollapseContext collapse();
+
     public abstract SearchContext parsedPostFilter(ParsedQuery postFilter);
 
     public abstract ParsedQuery parsedPostFilter();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchSortValues.java b/core/src/main/java/org/elasticsearch/search/internal/SearchSortValues.java
new file mode 100644
index 00000000000..9aa29d7768f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchSortValues.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.internal.InternalSearchHit.Fields;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+public class SearchSortValues implements ToXContent, Writeable {
+
+    static final SearchSortValues EMPTY = new SearchSortValues(new Object[0]);
+    private final Object[] sortValues;
+
+    SearchSortValues(Object[] sortValues) {
+        this.sortValues = Objects.requireNonNull(sortValues, "sort values must not be empty");
+    }
+
+    public SearchSortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
+        Objects.requireNonNull(sortValues);
+        Objects.requireNonNull(sortValueFormats);
+        this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
+        for (int i = 0; i < sortValues.length; ++i) {
+            if (this.sortValues[i] instanceof BytesRef) {
+                this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
+            }
+        }
+    }
+
+    public SearchSortValues(StreamInput in) throws IOException {
+        int size = in.readVInt();
+        if (size > 0) {
+            sortValues = new Object[size];
+            for (int i = 0; i < sortValues.length; i++) {
+                byte type = in.readByte();
+                if (type == 0) {
+                    sortValues[i] = null;
+                } else if (type == 1) {
+                    sortValues[i] = in.readString();
+                } else if (type == 2) {
+                    sortValues[i] = in.readInt();
+                } else if (type == 3) {
+                    sortValues[i] = in.readLong();
+                } else if (type == 4) {
+                    sortValues[i] = in.readFloat();
+                } else if (type == 5) {
+                    sortValues[i] = in.readDouble();
+                } else if (type == 6) {
+                    sortValues[i] = in.readByte();
+                } else if (type == 7) {
+                    sortValues[i] = in.readShort();
+                } else if (type == 8) {
+                    sortValues[i] = in.readBoolean();
+                } else {
+                    throw new IOException("Can't match type [" + type + "]");
+                }
+            }
+        } else {
+            sortValues = new Object[0];
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(sortValues.length);
+        for (Object sortValue : sortValues) {
+            if (sortValue == null) {
+                out.writeByte((byte) 0);
+            } else {
+                Class type = sortValue.getClass();
+                if (type == String.class) {
+                    out.writeByte((byte) 1);
+                    out.writeString((String) sortValue);
+                } else if (type == Integer.class) {
+                    out.writeByte((byte) 2);
+                    out.writeInt((Integer) sortValue);
+                } else if (type == Long.class) {
+                    out.writeByte((byte) 3);
+                    out.writeLong((Long) sortValue);
+                } else if (type == Float.class) {
+                    out.writeByte((byte) 4);
+                    out.writeFloat((Float) sortValue);
+                } else if (type == Double.class) {
+                    out.writeByte((byte) 5);
+                    out.writeDouble((Double) sortValue);
+                } else if (type == Byte.class) {
+                    out.writeByte((byte) 6);
+                    out.writeByte((Byte) sortValue);
+                } else if (type == Short.class) {
+                    out.writeByte((byte) 7);
+                    out.writeShort((Short) sortValue);
+                } else if (type == Boolean.class) {
+                    out.writeByte((byte) 8);
+                    out.writeBoolean((Boolean) sortValue);
+                } else {
+                    throw new IOException("Can't handle sort field value of type [" + type + "]");
+                }
+            }
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (sortValues.length > 0) {
+            builder.startArray(Fields.SORT);
+            for (Object sortValue : sortValues) {
+                builder.value(sortValue);
+            }
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    public static SearchSortValues fromXContent(XContentParser parser) throws IOException {
+        XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation);
+        return new SearchSortValues(parser.list().toArray());
+    }
+
+    public Object[] sortValues() {
+        return sortValues;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        SearchSortValues other = (SearchSortValues) obj;
+        return Arrays.equals(sortValues, other.sortValues);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(sortValues);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index cc7dc3eb5fb..dd08cb49353 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -22,14 +22,12 @@ package org.elasticsearch.search.internal;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.AliasFilterParsingException;
@@ -97,7 +95,7 @@ public interface ShardSearchRequest {
      * The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
      * Returns <tt>null</tt> if no filtering is required.</p>
      */
-    static QueryBuilder parseAliasFilter(Function<XContentParser, QueryParseContext> contextFactory,
+    static QueryBuilder parseAliasFilter(CheckedFunction<byte[], QueryBuilder, IOException> filterParser,
                                          IndexMetaData metaData, String... aliasNames) {
         if (aliasNames == null || aliasNames.length == 0) {
             return null;
@@ -109,10 +107,7 @@ public interface ShardSearchRequest {
                 return null;
             }
             try {
-                byte[] filterSource = alias.filter().uncompressed();
-                try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) {
-                    return contextFactory.apply(parser).parseInnerQueryBuilder();
-                }
+                return filterParser.apply(alias.filter().uncompressed());
             } catch (IOException ex) {
                 throw new AliasFilterParsingException(index, alias.getAlias(), "Invalid alias filter", ex);
             }
@@ -128,19 +123,19 @@ public interface ShardSearchRequest {
             // we need to bench here a bit, to see maybe it makes sense to use OrFilter
             BoolQueryBuilder combined = new BoolQueryBuilder();
             for (String aliasName : aliasNames) {
-            AliasMetaData alias = aliases.get(aliasName);
-            if (alias == null) {
-                // This shouldn't happen unless alias disappeared after filteringAliases was called.
-                throw new InvalidAliasNameException(index, aliasNames[0],
-                    "Unknown alias name was passed to alias Filter");
-            }
-            QueryBuilder parsedFilter = parserFunction.apply(alias);
-            if (parsedFilter != null) {
-                combined.should(parsedFilter);
-            } else {
-                // The filter might be null only if filter was removed after filteringAliases was called
-                return null;
-            }
+                AliasMetaData alias = aliases.get(aliasName);
+                if (alias == null) {
+                    // This shouldn't happen unless alias disappeared after filteringAliases was called.
+                    throw new InvalidAliasNameException(index, aliasNames[0],
+                        "Unknown alias name was passed to alias Filter");
+                }
+                QueryBuilder parsedFilter = parserFunction.apply(alias);
+                if (parsedFilter != null) {
+                    combined.should(parsedFilter);
+                } else {
+                    // The filter might be null only if filter was removed after filteringAliases was called
+                    return null;
+                }
             }
             return combined;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 26d47a44f80..ebdcd2d72b7 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -23,6 +23,7 @@ import org.apache.lucene.util.Counter;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
@@ -316,6 +317,11 @@ public class SubSearchContext extends FilteredSearchContext {
         return this;
     }
 
+    @Override
+    public CollapseContext collapse() {
+        return null;
+    }
+
     @Override
     public void accessed(long accessTime) {
         throw new UnsupportedOperationException("Not supported");
diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
index 5d20080408b..a93d3c419ad 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
@@ -162,10 +162,9 @@ public abstract class AbstractInternalProfileTree<PB extends AbstractProfileBrea
 
         // TODO this would be better done bottom-up instead of top-down to avoid
         // calculating the same times over and over...but worth the effort?
-        long nodeTime = getNodeTime(timings);
         String type = getTypeFromElement(element);
         String description = getDescriptionFromElement(element);
-        return new ProfileResult(type, description, timings, childrenProfileResults, nodeTime);
+        return new ProfileResult(type, description, timings, childrenProfileResults);
     }
 
     protected abstract String getTypeFromElement(E element);
@@ -184,19 +183,4 @@ public abstract class AbstractInternalProfileTree<PB extends AbstractProfileBrea
         tree.set(parent, parentNode);
     }
 
-    /**
-     * Internal helper to calculate the time of a node, inclusive of children
-     *
-     * @param timings
-     *            A map of breakdown timing for the node
-     * @return The total time at this node, inclusive of children
-     */
-    private static long getNodeTime(Map<String, Long> timings) {
-        long nodeTime = 0;
-        for (long time : timings.values()) {
-            nodeTime += time;
-        }
-        return nodeTime;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
index 4c745a5127d..3b1bfe3c27a 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
@@ -23,16 +23,21 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
 
 /**
  * This class is the internal representation of a profiled Query, corresponding
@@ -43,11 +48,12 @@ import java.util.Map;
  * Each InternalProfileResult has a List of InternalProfileResults, which will contain
  * "children" queries if applicable
  */
-public final class ProfileResult implements Writeable, ToXContent {
+public final class ProfileResult implements Writeable, ToXContentObject {
 
     private static final ParseField TYPE = new ParseField("type");
     private static final ParseField DESCRIPTION = new ParseField("description");
     private static final ParseField NODE_TIME = new ParseField("time");
+    private static final ParseField NODE_TIME_RAW = new ParseField("time_in_nanos");
     private static final ParseField CHILDREN = new ParseField("children");
     private static final ParseField BREAKDOWN = new ParseField("breakdown");
 
@@ -57,13 +63,12 @@ public final class ProfileResult implements Writeable, ToXContent {
     private final long nodeTime;
     private final List<ProfileResult> children;
 
-    public ProfileResult(String type, String description, Map<String, Long> timings, List<ProfileResult> children,
-            long nodeTime) {
+    public ProfileResult(String type, String description, Map<String, Long> timings, List<ProfileResult> children) {
         this.type = type;
         this.description = description;
-        this.timings = timings;
+        this.timings = Objects.requireNonNull(timings, "required timings argument missing");
         this.children = children;
-        this.nodeTime = nodeTime;
+        this.nodeTime = getTotalTime(timings);
     }
 
     /**
@@ -146,7 +151,7 @@ public final class ProfileResult implements Writeable, ToXContent {
         builder = builder.startObject()
                 .field(TYPE.getPreferredName(), type)
                 .field(DESCRIPTION.getPreferredName(), description)
-                .field(NODE_TIME.getPreferredName(), String.format(Locale.US, "%.10gms", getTime() / 1000000.0))
+                .timeValueField(NODE_TIME_RAW.getPreferredName(), NODE_TIME.getPreferredName(), getTime(), TimeUnit.NANOSECONDS)
                 .field(BREAKDOWN.getPreferredName(), timings);
 
         if (!children.isEmpty()) {
@@ -161,4 +166,65 @@ public final class ProfileResult implements Writeable, ToXContent {
         return builder;
     }
 
+    public static ProfileResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        String type = null, description = null;
+        Map<String, Long> timings =  new HashMap<>();
+        List<ProfileResult> children = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (TYPE.match(currentFieldName)) {
+                    type = parser.text();
+                } else if (DESCRIPTION.match(currentFieldName)) {
+                    description = parser.text();
+                } else if (NODE_TIME.match(currentFieldName)) {
+                    // skip, total time is calculate by adding up 'timings' values in ProfileResult ctor
+                    parser.text();
+                } else if (NODE_TIME_RAW.match(currentFieldName)) {
+                    // skip, total time is calculate by adding up 'timings' values in ProfileResult ctor
+                    parser.longValue();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (BREAKDOWN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        ensureExpectedToken(parser.currentToken(), XContentParser.Token.FIELD_NAME, parser::getTokenLocation);
+                        String name = parser.currentName();
+                        ensureExpectedToken(parser.nextToken(), XContentParser.Token.VALUE_NUMBER, parser::getTokenLocation);
+                        long value = parser.longValue();
+                        timings.put(name, value);
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (CHILDREN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        children.add(ProfileResult.fromXContent(parser));
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            }
+        }
+        return new ProfileResult(type, description, timings, children);
+    }
+
+    /**
+     * @param timings a map of breakdown timing for the node
+     * @return The total time at this node
+     */
+    private static long getTotalTime(Map<String, Long> timings) {
+        long nodeTime = 0;
+        for (long time : timings.values()) {
+            nodeTime += time;
+        }
+        return nodeTime;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
index 6794aa49399..eb3017bd1e7 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
 import org.elasticsearch.search.profile.aggregation.AggregationProfiler;
 import org.elasticsearch.search.profile.query.QueryProfileShardResult;
@@ -35,6 +36,12 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeSet;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 
 /**
  * A container class to hold all the profile results across all shards.  Internally
@@ -42,6 +49,11 @@ import java.util.Map;
  */
 public final class SearchProfileShardResults implements Writeable, ToXContent{
 
+    private static final String SEARCHES_FIELD = "searches";
+    private static final String ID_FIELD = "id";
+    private static final String SHARDS_FIELD = "shards";
+    public static final String PROFILE_FIELD = "profile";
+
     private Map<String, ProfileShardResult> shardResults;
 
     public SearchProfileShardResults(Map<String, ProfileShardResult> shardResults) {
@@ -75,26 +87,73 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject("profile").startArray("shards");
-
-        for (Map.Entry<String, ProfileShardResult> entry : shardResults.entrySet()) {
+        builder.startObject(PROFILE_FIELD).startArray(SHARDS_FIELD);
+        // shardResults is a map, but we print entries in a json array, which is ordered.
+        // we sort the keys of the map, so that toXContent always prints out the same array order
+        TreeSet<String> sortedKeys = new TreeSet<>(shardResults.keySet());
+        for (String key : sortedKeys) {
             builder.startObject();
-            builder.field("id", entry.getKey());
-            builder.startArray("searches");
-            for (QueryProfileShardResult result : entry.getValue().getQueryProfileResults()) {
-                builder.startObject();
+            builder.field(ID_FIELD, key);
+            builder.startArray(SEARCHES_FIELD);
+            ProfileShardResult profileShardResult = shardResults.get(key);
+            for (QueryProfileShardResult result : profileShardResult.getQueryProfileResults()) {
                 result.toXContent(builder, params);
-                builder.endObject();
             }
             builder.endArray();
-            entry.getValue().getAggregationProfileResults().toXContent(builder, params);
+            profileShardResult.getAggregationProfileResults().toXContent(builder, params);
             builder.endObject();
         }
-
         builder.endArray().endObject();
         return builder;
     }
 
+    public static SearchProfileShardResults fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        Map<String, ProfileShardResult> searchProfileResults = new HashMap<>();
+        ensureFieldName(parser, parser.nextToken(), SHARDS_FIELD);
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation);
+        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            parseSearchProfileResultsEntry(parser, searchProfileResults);
+        }
+        ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        return new SearchProfileShardResults(searchProfileResults);
+    }
+
+    private static void parseSearchProfileResultsEntry(XContentParser parser,
+            Map<String, ProfileShardResult> searchProfileResults) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
+        AggregationProfileShardResult aggProfileShardResult = null;
+        String id = null;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (ID_FIELD.equals(currentFieldName)) {
+                    id = parser.text();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (SEARCHES_FIELD.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        queryProfileResults.add(QueryProfileShardResult.fromXContent(parser));
+                    }
+                } else if (AggregationProfileShardResult.AGGREGATIONS.equals(currentFieldName)) {
+                    aggProfileShardResult = AggregationProfileShardResult.fromXContent(parser);
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        searchProfileResults.put(id, new ProfileShardResult(queryProfileResults, aggProfileShardResult));
+    }
+
     /**
      * Helper method to convert Profiler into InternalProfileShardResults, which
      * can be serialized to other nodes, emitted as JSON, etc.
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
index df55c5592d6..95a41401d11 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.ProfileResult;
 
 import java.io.IOException;
@@ -31,12 +32,15 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
 /**
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  */
 public final class AggregationProfileShardResult implements Writeable, ToXContent {
 
+    public static final String AGGREGATIONS = "aggregations";
     private final List<ProfileResult> aggProfileResults;
 
     public AggregationProfileShardResult(List<ProfileResult> aggProfileResults) {
@@ -69,11 +73,21 @@ public final class AggregationProfileShardResult implements Writeable, ToXConten
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("aggregations");
+        builder.startArray(AGGREGATIONS);
         for (ProfileResult p : aggProfileResults) {
             p.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
     }
+
+    public static AggregationProfileShardResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation);
+        List<ProfileResult> aggProfileResults = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            aggProfileResults.add(ProfileResult.fromXContent(parser));
+        }
+        return new AggregationProfileShardResult(aggProfileResults);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
index 6b4d7c0e842..1fa56bde7fe 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
@@ -24,19 +24,25 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
 
 /**
  * Public interface and serialization container for profiled timings of the
  * Collectors used in the search.  Children CollectorResult's may be
  * embedded inside of a parent CollectorResult
  */
-public class CollectorResult implements ToXContent, Writeable {
+public class CollectorResult implements ToXContentObject, Writeable {
 
     public static final String REASON_SEARCH_COUNT = "search_count";
     public static final String REASON_SEARCH_TOP_HITS = "search_top_hits";
@@ -52,6 +58,7 @@ public class CollectorResult implements ToXContent, Writeable {
     private static final ParseField NAME = new ParseField("name");
     private static final ParseField REASON = new ParseField("reason");
     private static final ParseField TIME = new ParseField("time");
+    private static final ParseField TIME_NANOS = new ParseField("time_in_nanos");
     private static final ParseField CHILDREN = new ParseField("children");
 
     /**
@@ -140,7 +147,7 @@ public class CollectorResult implements ToXContent, Writeable {
         builder = builder.startObject()
                 .field(NAME.getPreferredName(), getName())
                 .field(REASON.getPreferredName(), getReason())
-                .field(TIME.getPreferredName(), String.format(Locale.US, "%.10gms", (double) (getTime() / 1000000.0)));
+                .timeValueField(TIME_NANOS.getPreferredName(), TIME.getPreferredName(), getTime(), TimeUnit.NANOSECONDS);
 
         if (!children.isEmpty()) {
             builder = builder.startArray(CHILDREN.getPreferredName());
@@ -152,4 +159,42 @@ public class CollectorResult implements ToXContent, Writeable {
         builder = builder.endObject();
         return builder;
     }
+
+    public static CollectorResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        String name = null, reason = null;
+        long time = -1;
+        List<CollectorResult> children = new ArrayList<>();
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (NAME.match(currentFieldName)) {
+                    name = parser.text();
+                } else if (REASON.match(currentFieldName)) {
+                    reason = parser.text();
+                } else if (TIME.match(currentFieldName)) {
+                    // we need to consume this value, but we use the raw nanosecond value
+                    parser.text();
+                } else if (TIME_NANOS.match(currentFieldName)) {
+                    time = parser.longValue();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (CHILDREN.match(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        children.add(CollectorResult.fromXContent(parser));
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        return new CollectorResult(name, reason, time, children);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
index c65d0978906..ea8dbb2f335 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
@@ -34,7 +34,7 @@ final class ProfileCollector extends FilterCollector {
     private long time;
 
     /** Sole constructor. */
-    public ProfileCollector(Collector in) {
+    ProfileCollector(Collector in) {
         super(in);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
index d5e00aca336..362826e860c 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
@@ -22,8 +22,9 @@ package org.elasticsearch.search.profile.query;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.profile.ProfileResult;
 
 import java.io.IOException;
@@ -31,11 +32,19 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
+
 /**
  * A container class to hold the profile results for a single shard in the request.
  * Contains a list of query profiles, a collector tree and a total rewrite tree.
  */
-public final class QueryProfileShardResult implements Writeable, ToXContent {
+public final class QueryProfileShardResult implements Writeable, ToXContentObject {
+
+    public static final String COLLECTOR = "collector";
+    public static final String REWRITE_TIME = "rewrite_time";
+    public static final String QUERY_ARRAY = "query";
 
     private final List<ProfileResult> queryProfileResults;
 
@@ -90,15 +99,52 @@ public final class QueryProfileShardResult implements Writeable, ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("query");
+        builder.startObject();
+        builder.startArray(QUERY_ARRAY);
         for (ProfileResult p : queryProfileResults) {
             p.toXContent(builder, params);
         }
         builder.endArray();
-        builder.field("rewrite_time", rewriteTime);
-        builder.startArray("collector");
+        builder.field(REWRITE_TIME, rewriteTime);
+        builder.startArray(COLLECTOR);
         profileCollector.toXContent(builder, params);
         builder.endArray();
+        builder.endObject();
         return builder;
     }
+
+    public static QueryProfileShardResult fromXContent(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation);
+        String currentFieldName = null;
+        List<ProfileResult> queryProfileResults = new ArrayList<>();
+        long rewriteTime = 0;
+        CollectorResult collector = null;
+        while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if (REWRITE_TIME.equals(currentFieldName)) {
+                    rewriteTime = parser.longValue();
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (QUERY_ARRAY.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        queryProfileResults.add(ProfileResult.fromXContent(parser));
+                    }
+                } else if (COLLECTOR.equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        collector = CollectorResult.fromXContent(parser);
+                    }
+                } else {
+                    throwUnknownField(currentFieldName, parser.getTokenLocation());
+                }
+            } else {
+                throwUnknownToken(token, parser.getTokenLocation());
+            }
+        }
+        return new QueryProfileShardResult(queryProfileResults, rewriteTime, collector);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 5579e55826e..cc873edc566 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -41,11 +41,13 @@ import org.apache.lucene.search.TopFieldCollector;
 import org.apache.lucene.search.TopScoreDocCollector;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.MinimumScoreCollector;
 import org.elasticsearch.common.lucene.search.FilteredCollector;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.SearchService;
@@ -173,7 +175,7 @@ public class QueryPhase implements SearchPhase {
                 // Perhaps have a dedicated scroll phase?
                 final ScrollContext scrollContext = searchContext.scrollContext();
                 assert (scrollContext != null) == (searchContext.request().scroll() != null);
-                final TopDocsCollector<?> topDocsCollector;
+                final Collector topDocsCollector;
                 ScoreDoc after = null;
                 if (searchContext.request().scroll() != null) {
                     numDocs = Math.min(searchContext.size(), totalNumDocs);
@@ -206,17 +208,31 @@ public class QueryPhase implements SearchPhase {
                     numDocs = 1;
                 }
                 assert numDocs > 0;
-                if (searchContext.sort() != null) {
-                    SortAndFormats sf = searchContext.sort();
-                    topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
+                if (searchContext.collapse() == null) {
+                    if (searchContext.sort() != null) {
+                        SortAndFormats sf = searchContext.sort();
+                        topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
                             (FieldDoc) after, true, searchContext.trackScores(), searchContext.trackScores());
-                    sortValueFormats = sf.formats;
-                } else {
-                    rescore = !searchContext.rescore().isEmpty();
-                    for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
-                        numDocs = Math.max(rescoreContext.window(), numDocs);
+                        sortValueFormats = sf.formats;
+                    } else {
+                        rescore = !searchContext.rescore().isEmpty();
+                        for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
+                            numDocs = Math.max(rescoreContext.window(), numDocs);
+                        }
+                        topDocsCollector = TopScoreDocCollector.create(numDocs, after);
+                    }
+                } else {
+                    Sort sort = Sort.RELEVANCE;
+                    if (searchContext.sort() != null) {
+                        sort = searchContext.sort().sort;
+                    }
+                    CollapseContext collapse = searchContext.collapse();
+                    topDocsCollector = collapse.createTopDocs(sort, numDocs, searchContext.trackScores());
+                    if (searchContext.sort() == null) {
+                        sortValueFormats = new DocValueFormat[] {DocValueFormat.RAW};
+                    } else {
+                        sortValueFormats = searchContext.sort().formats;
                     }
-                    topDocsCollector = TopScoreDocCollector.create(numDocs, after);
                 }
                 collector = topDocsCollector;
                 if (doProfile) {
@@ -225,7 +241,14 @@ public class QueryPhase implements SearchPhase {
                 topDocsCallable = new Callable<TopDocs>() {
                     @Override
                     public TopDocs call() throws Exception {
-                        TopDocs topDocs = topDocsCollector.topDocs();
+                        final TopDocs topDocs;
+                        if (topDocsCollector instanceof TopDocsCollector) {
+                            topDocs = ((TopDocsCollector<?>) topDocsCollector).topDocs();
+                        } else if (topDocsCollector instanceof CollapsingTopDocsCollector) {
+                            topDocs = ((CollapsingTopDocsCollector) topDocsCollector).getTopDocs();
+                        } else {
+                            throw new IllegalStateException("Unknown top docs collector " + topDocsCollector.getClass().getName());
+                        }
                         if (scrollContext != null) {
                             if (scrollContext.totalHits == -1) {
                                 // first round
@@ -239,8 +262,7 @@ public class QueryPhase implements SearchPhase {
                             }
                             switch (searchType) {
                             case QUERY_AND_FETCH:
-                            case DFS_QUERY_AND_FETCH:
-                                // for (DFS_)QUERY_AND_FETCH, we already know the last emitted doc
+                                // for QUERY_AND_FETCH, we already know the last emitted doc
                                 if (topDocs.scoreDocs.length > 0) {
                                     // set the last emitted doc
                                     scrollContext.lastEmittedDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1];
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
index 16c9c9ba8c7..ac136a7601e 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
@@ -85,7 +85,7 @@ public abstract class RescoreBuilder<RB extends RescoreBuilder<RB>> extends ToXC
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(fieldName, WINDOW_SIZE_FIELD)) {
+                if (WINDOW_SIZE_FIELD.match(fieldName)) {
                     windowSize = parser.intValue();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support [" + fieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
index bf15e06bcb3..43a9dd9c641 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
@@ -66,7 +66,6 @@ public interface Rescorer {
     /**
      * Extracts all terms needed to execute this {@link Rescorer}. This method
      * is executed in a distributed frequency collection roundtrip for
-     * {@link SearchType#DFS_QUERY_AND_FETCH} and
      * {@link SearchType#DFS_QUERY_THEN_FETCH}
      */
     void extractTerms(SearchContext context, RescoreSearchContext rescoreContext, Set<Term> termsSet);
diff --git a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
index b9a6ca9be57..bc73ad7925b 100644
--- a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -202,7 +201,7 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
         builder.array(SEARCH_AFTER.getPreferredName(), sortValues);
     }
 
-    public static SearchAfterBuilder fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static SearchAfterBuilder fromXContent(XContentParser parser) throws IOException {
         SearchAfterBuilder builder = new SearchAfterBuilder();
         XContentParser.Token token = parser.currentToken();
         List<Object> values = new ArrayList<> ();
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 9defd05f471..3362a34ab0b 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -29,10 +29,8 @@ import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -74,16 +72,14 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private static final ParseField UNIT_FIELD = new ParseField("unit");
     private static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
-            .withAllDeprecated("use validation_method instead");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
-            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed").withAllDeprecated("validation_method");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize").withAllDeprecated("validation_method");
     private static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
 
     private final String fieldName;
     private final List<GeoPoint> points = new ArrayList<>();
 
-    private GeoDistance geoDistance = GeoDistance.DEFAULT;
+    private GeoDistance geoDistance = GeoDistance.ARC;
     private DistanceUnit unit = DistanceUnit.DEFAULT;
 
     private SortMode sortMode = null;
@@ -393,18 +389,18 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
      * Creates a new {@link GeoDistanceSortBuilder} from the query held by the {@link QueryParseContext} in
      * {@link org.elasticsearch.common.xcontent.XContent} format.
      *
-     * @param context the input parse context. The state on the parser contained in this context will be changed as a side effect of this
-     *        method call
-     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies further parameters, e.g.
-     *        in '{ "foo": { "order" : "asc"} }'. When parsing the inner object, the field name can be passed in via this argument
+     * @param context the input parse context. The state on the parser contained in this context will be changed as a
+     *                side effect of this method call
+     * @param elementName in some sort syntax variations the field name precedes the xContent object that specifies
+     *                    further parameters, e.g. in '{ "foo": { "order" : "asc"} }'. When parsing the inner object,
+     *                    the field name can be passed in via this argument
      */
     public static GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
         XContentParser parser = context.parser();
-        ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
         String fieldName = null;
         List<GeoPoint> geoPoints = new ArrayList<>();
         DistanceUnit unit = DistanceUnit.DEFAULT;
-        GeoDistance geoDistance = GeoDistance.DEFAULT;
+        GeoDistance geoDistance = GeoDistance.ARC;
         SortOrder order = SortOrder.ASC;
         SortMode sortMode = null;
         QueryBuilder nestedFilter = null;
@@ -424,7 +420,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
 
                 fieldName = currentName;
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentName, NESTED_FILTER_FIELD)) {
+                if (NESTED_FILTER_FIELD.match(currentName)) {
                     nestedFilter = context.parseInnerQueryBuilder();
                 } else {
                     // the json in the format of -> field : { lat : 30, lon : 12 }
@@ -441,27 +437,27 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
                     geoPoints.add(point);
                 }
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentName, ORDER_FIELD)) {
+                if (ORDER_FIELD.match(currentName)) {
                     order = SortOrder.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, UNIT_FIELD)) {
+                } else if (UNIT_FIELD.match(currentName)) {
                     unit = DistanceUnit.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, DISTANCE_TYPE_FIELD)) {
+                } else if (DISTANCE_TYPE_FIELD.match(currentName)) {
                     geoDistance = GeoDistance.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, COERCE_FIELD)) {
+                } else if (COERCE_FIELD.match(currentName)) {
                     coerce = parser.booleanValue();
                     if (coerce) {
                         ignoreMalformed = true;
                     }
-                } else if (parseFieldMatcher.match(currentName, IGNORE_MALFORMED_FIELD)) {
+                } else if (IGNORE_MALFORMED_FIELD.match(currentName)) {
                     boolean ignore_malformed_value = parser.booleanValue();
                     if (coerce == false) {
                         ignoreMalformed = ignore_malformed_value;
                     }
-                } else if (parseFieldMatcher.match(currentName, VALIDATION_METHOD_FIELD)) {
+                } else if (VALIDATION_METHOD_FIELD.match(currentName)) {
                     validation = GeoValidationMethod.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) {
+                } else if (SORTMODE_FIELD.match(currentName)) {
                     sortMode = SortMode.fromString(parser.text());
-                } else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) {
+                } else if (NESTED_PATH_FIELD.match(currentName)) {
                     nestedPath = parser.text();
                 } else if (token == Token.VALUE_STRING){
                     if (fieldName != null && fieldName.equals(currentName) == false) {
@@ -509,12 +505,10 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
-        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
-        List<GeoPoint> localPoints = new ArrayList<>();
-        for (GeoPoint geoPoint : this.points) {
-            localPoints.add(new GeoPoint(geoPoint));
-        }
 
+        // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed
+        // on 2.x created indexes
+        GeoPoint[] localPoints =  points.toArray(new GeoPoint[points.size()]);
         if (!indexCreatedBeforeV2_0 && !GeoValidationMethod.isIgnoreMalformed(validation)) {
             for (GeoPoint point : localPoints) {
                 if (GeoUtils.isValidLatitude(point.lat()) == false) {
@@ -548,7 +542,8 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
 
         MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType == null) {
-            throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
+            throw new IllegalArgumentException("failed to find mapper for [" + fieldName
+                + "] for geo distance based sort");
         }
         final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
         final Nested nested = resolveNested(context, nestedPath, nestedFilter);
@@ -558,17 +553,12 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
                 && finalSortMode == MultiValueMode.MIN // LatLonDocValuesField internally picks the closest point
                 && unit == DistanceUnit.METERS
                 && reverse == false
-                && localPoints.size() == 1) {
+                && localPoints.length == 1) {
             return new SortFieldAndFormat(
-                    LatLonDocValuesField.newDistanceSort(fieldName, localPoints.get(0).lat(), localPoints.get(0).lon()),
+                    LatLonDocValuesField.newDistanceSort(fieldName, localPoints[0].lat(), localPoints[0].lon()),
                     DocValueFormat.RAW);
         }
 
-        final FixedSourceDistance[] distances = new FixedSourceDistance[localPoints.size()];
-        for (int i = 0; i < localPoints.size(); i++) {
-            distances[i] = geoDistance.fixedSourceDistance(localPoints.get(i).lat(), localPoints.get(i).lon(), unit);
-        }
-
         IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {
 
             @Override
@@ -577,12 +567,15 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
             }
 
             @Override
-            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
+            public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed)
+                throws IOException {
                 return new FieldComparator.DoubleComparator(numHits, null, null) {
                     @Override
-                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
+                    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field)
+                        throws IOException {
                         final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
-                        final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances);
+                        final SortedNumericDoubleValues distanceValues = GeoUtils.distanceValues(geoDistance, unit,
+                            geoPointValues, localPoints);
                         final NumericDoubleValues selectedValues;
                         if (nested == null) {
                             selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY);
@@ -599,14 +592,16 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
 
         };
 
-        return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW);
+        return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse),
+            DocValueFormat.RAW);
     }
 
     static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {
         while (!parser.nextToken().equals(XContentParser.Token.END_ARRAY)) {
             if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                // we might get here if the geo point is " number, number] " and the parser already moved over the opening bracket
-                // in this case we cannot use GeoUtils.parseGeoPoint(..) because this expects an opening bracket
+                // we might get here if the geo point is " number, number] " and the parser already moved over the
+                // opening bracket in this case we cannot use GeoUtils.parseGeoPoint(..) because this expects an opening
+                // bracket
                 double lon = parser.doubleValue();
                 parser.nextToken();
                 if (!parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER)) {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index 6da93b26b8c..f284597e782 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -216,7 +216,8 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
             a -> new ScriptSortBuilder((Script) a[0], (ScriptSortType) a[1]));
 
     static {
-        PARSER.declareField(constructorArg(), Script::parse, Script.SCRIPT_PARSE_FIELD, ValueType.OBJECT_OR_STRING);
+        PARSER.declareField(constructorArg(), (parser, context) -> Script.parse(parser, context.getDefaultScriptLanguage()),
+                Script.SCRIPT_PARSE_FIELD, ValueType.OBJECT_OR_STRING);
         PARSER.declareField(constructorArg(), p -> ScriptSortType.fromString(p.text()), TYPE_FIELD, ValueType.STRING);
         PARSER.declareString((b, v) -> b.order(SortOrder.fromString(v)), ORDER_FIELD);
         PARSER.declareString((b, v) -> b.sortMode(SortMode.fromString(v)), SORTMODE_FIELD);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
index 332d18f256d..fec2b0f3bf3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.suggest;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,7 +28,6 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
@@ -138,9 +136,7 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable {
         return builder;
     }
 
-    public static SuggestBuilder fromXContent(QueryParseContext parseContext, Suggesters suggesters) throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+    public static SuggestBuilder fromXContent(XContentParser parser) throws IOException {
         SuggestBuilder suggestBuilder = new SuggestBuilder();
         String fieldName = null;
 
@@ -154,7 +150,7 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(fieldName, GLOBAL_TEXT_FIELD)) {
+                if (GLOBAL_TEXT_FIELD.match(fieldName)) {
                     suggestBuilder.setGlobalText(parser.text());
                 } else {
                     throw new IllegalArgumentException("[suggest] does not support [" + fieldName + "]");
@@ -164,7 +160,7 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable {
                 if (suggestionName == null) {
                     throw new IllegalArgumentException("suggestion must have name");
                 }
-                suggestBuilder.addSuggestion(suggestionName, SuggestionBuilder.fromXContent(parseContext, suggesters));
+                suggestBuilder.addSuggestion(suggestionName, SuggestionBuilder.fromXContent(parser));
             } else {
                 throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "] after [" + fieldName + "]");
             }
@@ -195,8 +191,7 @@ public class SuggestBuilder extends ToXContentToBytes implements Writeable {
         if (other == null || getClass() != other.getClass()) {
             return false;
         }
-        @SuppressWarnings("unchecked")
-        SuggestBuilder o = (SuggestBuilder)other;
+        SuggestBuilder o = (SuggestBuilder) other;
         return Objects.equals(globalText, o.globalText) &&
                Objects.equals(suggestions, o.suggestions);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
index 711e3d9db7f..6c007bf4cb2 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/Suggester.java
@@ -21,23 +21,16 @@ package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.index.query.QueryParseContext;
 
 import java.io.IOException;
 
-public abstract class Suggester<T extends SuggestionSearchContext.SuggestionContext> implements Writeable.Reader<SuggestionBuilder<?>> {
+public abstract class Suggester<T extends SuggestionSearchContext.SuggestionContext> {
 
     protected abstract Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
         innerExecute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException;
 
-    /**
-     * Read the SuggestionBuilder paired with this Suggester XContent.
-     */
-    public abstract SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException;
-
     public Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>
-        execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
+                execute(String name, T suggestion, IndexSearcher searcher, CharsRefBuilder spare) throws IOException {
         // #3469 We want to ignore empty shards
 
         if (searcher.getIndexReader().numDocs() == 0) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
index ca1d7a2306f..6714778c1b3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestionBuilder.java
@@ -21,19 +21,17 @@ package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 
@@ -43,7 +41,7 @@ import java.util.Objects;
 /**
  * Base class for the different suggestion implementations.
  */
-public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends ToXContentToBytes implements NamedWriteable {
+public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> implements NamedWriteable, ToXContent {
 
     protected final String field;
     protected String text;
@@ -254,10 +252,7 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
 
     protected abstract XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException;
 
-    static SuggestionBuilder<?> fromXContent(QueryParseContext parseContext, Suggesters suggesters)
-            throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parsefieldMatcher = parseContext.getParseFieldMatcher();
+    static SuggestionBuilder<?> fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String currentFieldName = null;
         String suggestText = null;
@@ -269,17 +264,17 @@ public abstract class SuggestionBuilder<T extends SuggestionBuilder<T>> extends
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parsefieldMatcher.match(currentFieldName, TEXT_FIELD)) {
+                if (TEXT_FIELD.match(currentFieldName)) {
                     suggestText = parser.text();
-                } else if (parsefieldMatcher.match(currentFieldName, PREFIX_FIELD)) {
+                } else if (PREFIX_FIELD.match(currentFieldName)) {
                     prefix = parser.text();
-                } else if (parsefieldMatcher.match(currentFieldName, REGEX_FIELD)) {
+                } else if (REGEX_FIELD.match(currentFieldName)) {
                     regex = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "suggestion does not support [" + currentFieldName + "]");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                suggestionBuilder = suggesters.getSuggester(currentFieldName).innerFromXContent(parseContext);
+                suggestionBuilder = parser.namedObject(SuggestionBuilder.class, currentFieldName, null);
             }
         }
         if (suggestionBuilder == null) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index 049883822de..b048977741f 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -29,18 +29,14 @@ import org.apache.lucene.search.suggest.document.TopSuggestDocs;
 import org.apache.lucene.search.suggest.document.TopSuggestDocsCollector;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.apache.lucene.util.PriorityQueue;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -112,7 +108,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
 
             private List<TopSuggestDocs.SuggestScoreDoc> suggestScoreDocs;
 
-            public SuggestDoc(int doc, CharSequence key, CharSequence context, float score) {
+            SuggestDoc(int doc, CharSequence key, CharSequence context, float score) {
                 super(doc, key, context, score);
             }
 
@@ -156,7 +152,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
 
         private static final class SuggestDocPriorityQueue extends PriorityQueue<SuggestDoc> {
 
-            public SuggestDocPriorityQueue(int maxSize) {
+            SuggestDocPriorityQueue(int maxSize) {
                 super(maxSize);
             }
 
@@ -188,7 +184,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
         private final SuggestDocPriorityQueue pq;
         private final Map<Integer, SuggestDoc> scoreDocMap;
 
-        public TopDocumentsCollector(int num) {
+        TopDocumentsCollector(int num) {
             super(1); // TODO hack, we don't use the underlying pq, so we allocate a size of 1
             this.num = num;
             this.scoreDocMap = new LinkedHashMap<>(num);
@@ -241,14 +237,4 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             }
         }
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return CompletionSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new CompletionSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
index 09382d9aaff..91dfd510995 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.suggest.completion;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
@@ -42,7 +40,6 @@ import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextMappings;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -68,27 +65,26 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
      *     "payload" : STRING_ARRAY
      * }
      */
-    private static ObjectParser<CompletionSuggestionBuilder.InnerBuilder, ParseFieldMatcherSupplier> TLP_PARSER =
-        new ObjectParser<>(SUGGESTION_NAME, null);
+    private static final ObjectParser<CompletionSuggestionBuilder.InnerBuilder, Void> PARSER = new ObjectParser<>(SUGGESTION_NAME, null);
     static {
-        TLP_PARSER.declareField((parser, completionSuggestionContext, context) -> {
+        PARSER.declareField((parser, completionSuggestionContext, context) -> {
                 if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) {
                     if (parser.booleanValue()) {
                         completionSuggestionContext.fuzzyOptions = new FuzzyOptions.Builder().build();
                     }
                 } else {
-                    completionSuggestionContext.fuzzyOptions = FuzzyOptions.parse(parser, context);
+                    completionSuggestionContext.fuzzyOptions = FuzzyOptions.parse(parser);
                 }
             },
             FuzzyOptions.FUZZY_OPTIONS, ObjectParser.ValueType.OBJECT_OR_BOOLEAN);
-        TLP_PARSER.declareField((parser, completionSuggestionContext, context) ->
-            completionSuggestionContext.regexOptions = RegexOptions.parse(parser, context),
+        PARSER.declareField((parser, completionSuggestionContext, context) ->
+            completionSuggestionContext.regexOptions = RegexOptions.parse(parser),
             RegexOptions.REGEX_OPTIONS, ObjectParser.ValueType.OBJECT);
-        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, FIELDNAME_FIELD);
-        TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, ANALYZER_FIELD);
-        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SIZE_FIELD);
-        TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SHARDSIZE_FIELD);
-        TLP_PARSER.declareField((p, v, c) -> {
+        PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, FIELDNAME_FIELD);
+        PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, ANALYZER_FIELD);
+        PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SIZE_FIELD);
+        PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SHARDSIZE_FIELD);
+        PARSER.declareField((p, v, c) -> {
             // Copy the current structure. We will parse, once the mapping is provided
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.copyCurrentStructure(p);
@@ -214,7 +210,7 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
     private static class InnerBuilder extends CompletionSuggestionBuilder {
         private String field;
 
-        public InnerBuilder() {
+        InnerBuilder() {
             super("_na_");
         }
 
@@ -233,17 +229,14 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
             regexOptions.toXContent(builder, params);
         }
         if (contextBytes != null) {
-            try (XContentParser contextParser = XContentFactory.xContent(XContentType.JSON).createParser(contextBytes)) {
-                builder.field(CONTEXTS_FIELD.getPreferredName());
-                builder.copyCurrentStructure(contextParser);
-            }
+            builder.rawField(CONTEXTS_FIELD.getPreferredName(), contextBytes);
         }
         return builder;
     }
 
-    static CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
+    public static CompletionSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         CompletionSuggestionBuilder.InnerBuilder builder = new CompletionSuggestionBuilder.InnerBuilder();
-        TLP_PARSER.parse(parseContext.parser(), builder, parseContext);
+        PARSER.parse(parser, builder, null);
         String field = builder.field;
         // now we should have field name, check and copy fields over to the suggestion builder we return
         if (field == null) {
@@ -270,7 +263,8 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
             CompletionFieldMapper.CompletionFieldType type = (CompletionFieldMapper.CompletionFieldType) mappedFieldType;
             suggestionContext.setFieldType(type);
             if (type.hasContextMappings() && contextBytes != null) {
-                try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(contextBytes)) {
+                try (XContentParser contextParser = XContentFactory.xContent(contextBytes).createParser(context.getXContentRegistry(),
+                        contextBytes)) {
                     if (type.hasContextMappings() && contextParser != null) {
                         ContextMappings contextMappings = type.getContextMappings();
                         contextParser.nextToken();
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
index 3ae05f9ecf5..8d121b24f12 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/FuzzyOptions.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.suggest.completion;
 import org.apache.lucene.search.suggest.document.FuzzyCompletionQuery;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -56,7 +55,7 @@ public class FuzzyOptions implements ToXContent, Writeable {
      *     "max_determinized_states" : INT
      * }
      */
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>(FUZZY_OPTIONS.getPreferredName(),
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(FUZZY_OPTIONS.getPreferredName(),
             Builder::new);
     static {
         PARSER.declareInt(Builder::setFuzzyMinLength, MIN_LENGTH_FIELD);
@@ -67,6 +66,14 @@ public class FuzzyOptions implements ToXContent, Writeable {
         PARSER.declareField(Builder::setFuzziness, Fuzziness::parse, Fuzziness.FIELD, ObjectParser.ValueType.VALUE);
     }
 
+    static FuzzyOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null).build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
     private int editDistance;
     private boolean transpositions;
     private int fuzzyMinLength;
@@ -106,14 +113,6 @@ public class FuzzyOptions implements ToXContent, Writeable {
         out.writeVInt(maxDeterminizedStates);
     }
 
-    static FuzzyOptions parse(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
-        return PARSER.parse(parser, context).build();
-    }
-
-    public static Builder builder() {
-        return new Builder();
-    }
-
     /**
      * Returns the maximum number of edits
      */
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
index a9209044c8b..f330322a9f0 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/RegexOptions.java
@@ -23,7 +23,6 @@ import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -49,7 +48,7 @@ public class RegexOptions implements ToXContent, Writeable {
      *     "max_determinized_states" : INT
      * }
      */
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>(REGEX_OPTIONS.getPreferredName(),
+    private static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(REGEX_OPTIONS.getPreferredName(),
             Builder::new);
     static {
         PARSER.declareInt(Builder::setMaxDeterminizedStates, MAX_DETERMINIZED_STATES);
@@ -66,6 +65,14 @@ public class RegexOptions implements ToXContent, Writeable {
         PARSER.declareStringOrNull(Builder::setFlags, FLAGS_VALUE);
     }
 
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    static RegexOptions parse(XContentParser parser) throws IOException {
+        return PARSER.parse(parser, null).build();
+    }
+
     private int flagsValue;
     private int maxDeterminizedStates;
 
@@ -103,14 +110,6 @@ public class RegexOptions implements ToXContent, Writeable {
         return maxDeterminizedStates;
     }
 
-    public static Builder builder() {
-        return new Builder();
-    }
-
-    static RegexOptions parse(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
-        return PARSER.parse(parser, context).build();
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
index a17b7a87b43..59f59075bd3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/CategoryQueryContext.java
@@ -21,8 +21,6 @@ package org.elasticsearch.search.suggest.completion.context;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -98,7 +96,7 @@ public final class CategoryQueryContext implements ToXContent {
         return result;
     }
 
-    private static ObjectParser<Builder, ParseFieldMatcherSupplier> CATEGORY_PARSER = new ObjectParser<>(NAME, null);
+    private static ObjectParser<Builder, Void> CATEGORY_PARSER = new ObjectParser<>(NAME, null);
     static {
         CATEGORY_PARSER.declareString(Builder::setCategory, new ParseField(CONTEXT_VALUE));
         CATEGORY_PARSER.declareInt(Builder::setBoost, new ParseField(CONTEXT_BOOST));
@@ -110,7 +108,7 @@ public final class CategoryQueryContext implements ToXContent {
         XContentParser.Token token = parser.currentToken();
         Builder builder = builder();
         if (token == XContentParser.Token.START_OBJECT) {
-            CATEGORY_PARSER.parse(parser, builder, () -> ParseFieldMatcher.STRICT);
+            CATEGORY_PARSER.parse(parser, builder, null);
         } else if (token == XContentParser.Token.VALUE_STRING) {
             builder.setCategory(parser.text());
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
index 5a7cfccf315..0cac17dbb81 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/ContextMappings.java
@@ -115,7 +115,7 @@ public class ContextMappings implements ToXContent {
         private final Map<String, Set<CharSequence>> contexts;
         private final ParseContext.Document document;
 
-        public TypedContextField(String name, String value, int weight, Map<String, Set<CharSequence>> contexts,
+        TypedContextField(String name, String value, int weight, Map<String, Set<CharSequence>> contexts,
                                  ParseContext.Document document) {
             super(name, value, weight);
             this.contexts = contexts;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
index 6ed8754c978..e2f00c0001c 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/context/GeoQueryContext.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.suggest.completion.context;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -113,7 +112,7 @@ public final class GeoQueryContext implements ToXContent {
         return new Builder();
     }
 
-    private static ObjectParser<GeoQueryContext.Builder, ParseFieldMatcherSupplier> GEO_CONTEXT_PARSER = new ObjectParser<>(NAME, null);
+    private static ObjectParser<GeoQueryContext.Builder, Void> GEO_CONTEXT_PARSER = new ObjectParser<>(NAME, null);
     static {
         GEO_CONTEXT_PARSER.declareField((parser, geoQueryContext, geoContextMapping) -> geoQueryContext.setGeoPoint(GeoUtils.parseGeoPoint(parser)), new ParseField(CONTEXT_VALUE), ObjectParser.ValueType.OBJECT);
         GEO_CONTEXT_PARSER.declareInt(GeoQueryContext.Builder::setBoost, new ParseField(CONTEXT_BOOST));
@@ -130,7 +129,7 @@ public final class GeoQueryContext implements ToXContent {
         XContentParser.Token token = parser.currentToken();
         GeoQueryContext.Builder builder = new Builder();
         if (token == XContentParser.Token.START_OBJECT) {
-            GEO_CONTEXT_PARSER.parse(parser, builder, context);
+            GEO_CONTEXT_PARSER.parse(parser, builder, null);
         } else if (token == XContentParser.Token.VALUE_STRING) {
             builder.setGeoPoint(GeoPoint.fromGeohash(parser.text()));
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
index b3dad43ad34..d24ce6b3c29 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java
@@ -29,7 +29,7 @@ final class CandidateScorer {
     private final int maxNumCorrections;
     private final int gramSize;
 
-    public CandidateScorer(WordScorer scorer, int maxNumCorrections, int gramSize) {
+    CandidateScorer(WordScorer scorer, int maxNumCorrections, int gramSize) {
         this.scorer = scorer;
         this.maxNumCorrections = maxNumCorrections;
         this.gramSize = gramSize;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index 4132247363a..d9d06783e47 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -35,7 +35,6 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder.CandidateGenerator;
 
@@ -387,7 +386,7 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
         }
     }
 
-    private static ConstructingObjectParser<DirectCandidateGeneratorBuilder, QueryParseContext> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<DirectCandidateGeneratorBuilder, Void> PARSER = new ConstructingObjectParser<>(
             TYPE, args -> new DirectCandidateGeneratorBuilder((String) args[0]));
 
     static {
@@ -407,10 +406,6 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
         PARSER.declareInt(DirectCandidateGeneratorBuilder::prefixLength, PREFIX_LENGTH_FIELD);
     }
 
-    public static DirectCandidateGeneratorBuilder fromXContent(QueryParseContext parseContext) throws IOException {
-        return PARSER.apply(parseContext.parser(), parseContext);
-    }
-
     @Override
     public PhraseSuggestionContext.DirectCandidateGenerator build(MapperService mapperService) throws IOException {
         PhraseSuggestionContext.DirectCandidateGenerator generator = new PhraseSuggestionContext.DirectCandidateGenerator();
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
index 6c781a8ee91..222f6901916 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -103,8 +102,7 @@ public final class Laplace extends SmoothingModel {
         return Objects.hash(alpha);
     }
 
-    public static SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double alpha = DEFAULT_LAPLACE_ALPHA;
@@ -112,7 +110,7 @@ public final class Laplace extends SmoothingModel {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             }
-            if (token.isValue() && parseContext.getParseFieldMatcher().match(fieldName, ALPHA_FIELD)) {
+            if (token.isValue() && ALPHA_FIELD.match(fieldName)) {
                 alpha = parser.doubleValue();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
index 72802e66eaa..10a170d766d 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
@@ -23,14 +23,12 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -131,29 +129,27 @@ public final class LinearInterpolation extends SmoothingModel {
         return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
     }
 
-    public static LinearInterpolation innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static LinearInterpolation fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double trigramLambda = 0.0;
         double bigramLambda = 0.0;
         double unigramLambda = 0.0;
-        ParseFieldMatcher matcher = parseContext.getParseFieldMatcher();
         while ((token = parser.nextToken()) != Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (matcher.match(fieldName, TRIGRAM_FIELD)) {
+                if (TRIGRAM_FIELD.match(fieldName)) {
                     trigramLambda = parser.doubleValue();
                     if (trigramLambda < 0) {
                         throw new IllegalArgumentException("trigram_lambda must be positive");
                     }
-                } else if (matcher.match(fieldName, BIGRAM_FIELD)) {
+                } else if (BIGRAM_FIELD.match(fieldName)) {
                     bigramLambda = parser.doubleValue();
                     if (bigramLambda < 0) {
                         throw new IllegalArgumentException("bigram_lambda must be positive");
                     }
-                } else if (matcher.match(fieldName, UNIGRAM_FIELD)) {
+                } else if (UNIGRAM_FIELD.match(fieldName)) {
                     unigramLambda = parser.doubleValue();
                     if (unigramLambda < 0) {
                         throw new IllegalArgumentException("unigram_lambda must be positive");
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index 8eb0aa0529a..fc862f63638 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -29,21 +29,18 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry;
 import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker.Result;
 
@@ -120,7 +117,8 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
                     QueryShardContext shardContext = suggestion.getShardContext();
                     final ExecutableScript executable = collateScript.apply(vars);
                     final BytesReference querySource = (BytesReference) executable.run();
-                    try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) {
+                    try (XContentParser parser = XContentFactory.xContent(querySource).createParser(shardContext.getXContentRegistry(),
+                            querySource)) {
                         QueryBuilder innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
                         final ParsedQuery parsedQuery = shardContext.toQuery(innerQueryBuilder);
                         collateMatch = Lucene.exists(searcher, parsedQuery.query());
@@ -151,14 +149,4 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return PhraseSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new PhraseSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 969b1c24d5c..0a5596adf16 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.suggest.phrase;
 import org.apache.lucene.analysis.Analyzer;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,7 +36,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -126,7 +124,7 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
     /**
      * Read from a stream.
      */
-    PhraseSuggestionBuilder(StreamInput in) throws IOException {
+    public PhraseSuggestionBuilder(StreamInput in) throws IOException {
         super(in);
         maxErrors = in.readFloat();
         realWordErrorLikelihood = in.readFloat();
@@ -488,10 +486,8 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
         return builder;
     }
 
-    static PhraseSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static PhraseSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         PhraseSuggestionBuilder tmpSuggestion = new PhraseSuggestionBuilder("_na_");
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
         XContentParser.Token token;
         String currentFieldName = null;
         String fieldname = null;
@@ -499,56 +495,56 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                     tmpSuggestion.analyzer(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                     fieldname = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.size(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.shardSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD)) {
+                } else if (PhraseSuggestionBuilder.RWE_LIKELIHOOD_FIELD.match(currentFieldName)) {
                     tmpSuggestion.realWordErrorLikelihood(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.CONFIDENCE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.CONFIDENCE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.confidence(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SEPARATOR_FIELD)) {
+                } else if (PhraseSuggestionBuilder.SEPARATOR_FIELD.match(currentFieldName)) {
                     tmpSuggestion.separator(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.MAXERRORS_FIELD)) {
+                } else if (PhraseSuggestionBuilder.MAXERRORS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxErrors(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.GRAMSIZE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.GRAMSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.gramSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD)) {
+                } else if (PhraseSuggestionBuilder.FORCE_UNIGRAM_FIELD.match(currentFieldName)) {
                     tmpSuggestion.forceUnigrams(parser.booleanValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD)) {
+                } else if (PhraseSuggestionBuilder.TOKEN_LIMIT_FIELD.match(currentFieldName)) {
                     tmpSuggestion.tokenLimit(parser.intValue());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "suggester[phrase] doesn't support field [" + currentFieldName + "]");
                 }
             } else if (token == Token.START_ARRAY) {
-                if (parseFieldMatcher.match(currentFieldName, DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD)) {
+                if (DirectCandidateGeneratorBuilder.DIRECT_GENERATOR_FIELD.match(currentFieldName)) {
                     // for now we only have a single type of generators
                     while ((token = parser.nextToken()) == Token.START_OBJECT) {
-                        tmpSuggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.fromXContent(parseContext));
+                        tmpSuggestion.addCandidateGenerator(DirectCandidateGeneratorBuilder.PARSER.apply(parser, null));
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "suggester[phrase]  doesn't support array field [" + currentFieldName + "]");
                 }
             } else if (token == Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD)) {
+                if (PhraseSuggestionBuilder.SMOOTHING_MODEL_FIELD.match(currentFieldName)) {
                     ensureNoSmoothing(tmpSuggestion);
-                    tmpSuggestion.smoothingModel(SmoothingModel.fromXContent(parseContext));
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.HIGHLIGHT_FIELD)) {
+                    tmpSuggestion.smoothingModel(SmoothingModel.fromXContent(parser));
+                } else if (PhraseSuggestionBuilder.HIGHLIGHT_FIELD.match(currentFieldName)) {
                     String preTag = null;
                     String postTag = null;
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
                         } else if (token.isValue()) {
-                            if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.PRE_TAG_FIELD)) {
+                            if (PhraseSuggestionBuilder.PRE_TAG_FIELD.match(currentFieldName)) {
                                 preTag = parser.text();
-                            } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.POST_TAG_FIELD)) {
+                            } else if (PhraseSuggestionBuilder.POST_TAG_FIELD.match(currentFieldName)) {
                                 postTag = parser.text();
                             } else {
                                 throw new ParsingException(parser.getTokenLocation(),
@@ -557,21 +553,21 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
                         }
                     }
                     tmpSuggestion.highlight(preTag, postTag);
-                } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_FIELD)) {
+                } else if (PhraseSuggestionBuilder.COLLATE_FIELD.match(currentFieldName)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                         if (token == XContentParser.Token.FIELD_NAME) {
                             currentFieldName = parser.currentName();
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_FIELD)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_FIELD.match(currentFieldName)) {
                             if (tmpSuggestion.collateQuery() != null) {
                                 throw new ParsingException(parser.getTokenLocation(),
                                         "suggester[phrase][collate] query already set, doesn't support additional ["
                                         + currentFieldName + "]");
                             }
-                            Script template = Script.parse(parser, parseFieldMatcher, "mustache");
+                            Script template = Script.parse(parser, "mustache");
                             tmpSuggestion.collateQuery(template);
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_PARAMS.match(currentFieldName)) {
                             tmpSuggestion.collateParams(parser.map());
-                        } else if (parseFieldMatcher.match(currentFieldName, PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE)) {
+                        } else if (PhraseSuggestionBuilder.COLLATE_QUERY_PRUNE.match(currentFieldName)) {
                             if (parser.isBooleanValue()) {
                                 tmpSuggestion.collatePrune(parser.booleanValue());
                             } else {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
index 960dca419f7..99e2e18496b 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionContext.java
@@ -60,7 +60,7 @@ class PhraseSuggestionContext extends SuggestionContext {
     private Map<String, Object> collateScriptParams = new HashMap<>(1);
     private WordScorer.WordScorerFactory scorer = DEFAULT_SCORER;
 
-    public PhraseSuggestionContext(QueryShardContext shardContext) {
+    PhraseSuggestionContext(QueryShardContext shardContext) {
         super(PhraseSuggester.INSTANCE, shardContext);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
index 0f55c9ed8fe..6fd7d6ea539 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/SmoothingModel.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -65,9 +63,7 @@ public abstract class SmoothingModel implements NamedWriteable, ToXContent {
 
     protected abstract int doHashCode();
 
-    public static SmoothingModel fromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         SmoothingModel model = null;
@@ -75,12 +71,12 @@ public abstract class SmoothingModel implements NamedWriteable, ToXContent {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(fieldName, LinearInterpolation.PARSE_FIELD)) {
-                    model = LinearInterpolation.innerFromXContent(parseContext);
-                } else if (parseFieldMatcher.match(fieldName, Laplace.PARSE_FIELD)) {
-                    model = Laplace.innerFromXContent(parseContext);
-                } else if (parseFieldMatcher.match(fieldName, StupidBackoff.PARSE_FIELD)) {
-                    model = StupidBackoff.innerFromXContent(parseContext);
+                if (LinearInterpolation.PARSE_FIELD.match(fieldName)) {
+                    model = LinearInterpolation.fromXContent(parser);
+                } else if (Laplace.PARSE_FIELD.match(fieldName)) {
+                    model = Laplace.fromXContent(parser);
+                } else if (StupidBackoff.PARSE_FIELD.match(fieldName)) {
+                    model = StupidBackoff.fromXContent(parser);
                 } else {
                     throw new IllegalArgumentException("suggester[phrase] doesn't support object field [" + fieldName + "]");
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
index 3e67110ce7d..ebdd1c9ec28 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.phrase.WordScorer.WordScorerFactory;
 
 import java.io.IOException;
@@ -106,8 +105,7 @@ public final class StupidBackoff extends SmoothingModel {
         return Objects.hash(discount);
     }
 
-    public static SmoothingModel innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static SmoothingModel fromXContent(XContentParser parser) throws IOException {
         XContentParser.Token token;
         String fieldName = null;
         double discount = DEFAULT_BACKOFF_DISCOUNT;
@@ -115,7 +113,7 @@ public final class StupidBackoff extends SmoothingModel {
             if (token == XContentParser.Token.FIELD_NAME) {
                 fieldName = parser.currentName();
             }
-            if (token.isValue() && parseContext.getParseFieldMatcher().match(fieldName, DISCOUNT_FIELD)) {
+            if (token.isValue() && DISCOUNT_FIELD.match(fieldName)) {
                 discount = parser.doubleValue();
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
index 8eb08ef0688..166a4182c8f 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoffScorer.java
@@ -28,7 +28,7 @@ import java.io.IOException;
 class StupidBackoffScorer extends WordScorer {
     private final double discount;
 
-    public StupidBackoffScorer(IndexReader reader, Terms terms,String field, double realWordLikelyhood, BytesRef separator, double discount)
+    StupidBackoffScorer(IndexReader reader, Terms terms,String field, double realWordLikelyhood, BytesRef separator, double discount)
             throws IOException {
         super(reader, terms, field, realWordLikelyhood, separator);
         this.discount = discount;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
index 0d58e0f5cac..d6edf352d19 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggester.java
@@ -27,11 +27,8 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.phrase.DirectCandidateGenerator;
 
@@ -84,16 +81,6 @@ public final class TermSuggester extends Suggester<TermSuggestionContext> {
        return result;
     }
 
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return TermSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new TermSuggestionBuilder(in);
-    }
-
     private static class Token {
 
         public final Term term;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
index 31e6c3718e8..72fd41dc5b4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilder.java
@@ -26,14 +26,12 @@ import org.apache.lucene.search.spell.LuceneLevenshteinDistance;
 import org.apache.lucene.search.spell.NGramDistance;
 import org.apache.lucene.search.spell.StringDistance;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SortBy;
@@ -105,7 +103,7 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
     /**
      * Read from a stream.
      */
-    TermSuggestionBuilder(StreamInput in) throws IOException {
+    public TermSuggestionBuilder(StreamInput in) throws IOException {
         super(in);
         suggestMode = SuggestMode.readFromStream(in);
         accuracy = in.readFloat();
@@ -388,10 +386,8 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
         return builder;
     }
 
-    static TermSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-        XContentParser parser = parseContext.parser();
+    public static TermSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
         TermSuggestionBuilder tmpSuggestion = new TermSuggestionBuilder("_na_");
-        ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
         XContentParser.Token token;
         String currentFieldName = null;
         String fieldname = null;
@@ -399,33 +395,33 @@ public class TermSuggestionBuilder extends SuggestionBuilder<TermSuggestionBuild
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
-                if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                     tmpSuggestion.analyzer(parser.text());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                     fieldname = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.size(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.shardSize(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SUGGESTMODE_FIELD)) {
+                } else if (SUGGESTMODE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.suggestMode(SuggestMode.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, ACCURACY_FIELD)) {
+                } else if (ACCURACY_FIELD.match(currentFieldName)) {
                     tmpSuggestion.accuracy(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, SORT_FIELD)) {
+                } else if (SORT_FIELD.match(currentFieldName)) {
                     tmpSuggestion.sort(SortBy.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, STRING_DISTANCE_FIELD)) {
+                } else if (STRING_DISTANCE_FIELD.match(currentFieldName)) {
                     tmpSuggestion.stringDistance(StringDistanceImpl.resolve(parser.text()));
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_EDITS_FIELD)) {
+                } else if (MAX_EDITS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxEdits(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_INSPECTIONS_FIELD)) {
+                } else if (MAX_INSPECTIONS_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxInspections(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MAX_TERM_FREQ_FIELD)) {
+                } else if (MAX_TERM_FREQ_FIELD.match(currentFieldName)) {
                     tmpSuggestion.maxTermFreq(parser.floatValue());
-                } else if (parseFieldMatcher.match(currentFieldName, PREFIX_LENGTH_FIELD)) {
+                } else if (PREFIX_LENGTH_FIELD.match(currentFieldName)) {
                     tmpSuggestion.prefixLength(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MIN_WORD_LENGTH_FIELD)) {
+                } else if (MIN_WORD_LENGTH_FIELD.match(currentFieldName)) {
                     tmpSuggestion.minWordLength(parser.intValue());
-                } else if (parseFieldMatcher.match(currentFieldName, MIN_DOC_FREQ_FIELD)) {
+                } else if (MIN_DOC_FREQ_FIELD.match(currentFieldName)) {
                     tmpSuggestion.minDocFreq(parser.floatValue());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
index c950e5729da..19ad03e1b47 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/term/TermSuggestionContext.java
@@ -26,7 +26,7 @@ final class TermSuggestionContext extends SuggestionContext {
 
     private final DirectSpellcheckerSettings settings = new DirectSpellcheckerSettings();
 
-    public TermSuggestionContext(QueryShardContext shardContext) {
+    TermSuggestionContext(QueryShardContext shardContext) {
         super(TermSuggester.INSTANCE, shardContext);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 7f84f2d66c8..6afefec2c94 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -37,6 +37,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.RestoreInProgress.ShardRestoreStatus;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -175,6 +176,11 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
             // Read snapshot info and metadata from the repository
             Repository repository = repositoriesService.repository(request.repositoryName);
             final RepositoryData repositoryData = repository.getRepositoryData();
+            final Optional<SnapshotId> incompatibleSnapshotId =
+                repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> request.snapshotName.equals(s.getName())).findFirst();
+            if (incompatibleSnapshotId.isPresent()) {
+                throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "cannot restore incompatible snapshot");
+            }
             final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds().stream()
                 .filter(s -> request.snapshotName.equals(s.getName())).findFirst();
             if (matchingSnapshotId.isPresent() == false) {
@@ -205,6 +211,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
                     if (restoreInProgress != null && !restoreInProgress.entries().isEmpty()) {
                         throw new ConcurrentSnapshotExecutionException(snapshot, "Restore process is already running in this cluster");
                     }
+                    // Check if the snapshot to restore is currently being deleted
+                    SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                    if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                        throw new ConcurrentSnapshotExecutionException(snapshot,
+                            "cannot restore a snapshot while a snapshot deletion is in-progress [" +
+                                deletionsInProgress.getEntries().get(0).getSnapshot() + "]");
+                    }
 
                     // Updating cluster state
                     ClusterState.Builder builder = ClusterState.builder(currentState);
@@ -623,13 +636,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp
 
         private final Logger logger;
 
-        public CleanRestoreStateTaskExecutor(Logger logger) {
+        CleanRestoreStateTaskExecutor(Logger logger) {
             this.logger = logger;
         }
 
         @Override
-        public BatchResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
-            final BatchResult.Builder<Task> resultBuilder = BatchResult.<Task>builder().successes(tasks);
+        public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
+            final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
             Set<Snapshot> completedSnapshots = tasks.stream().map(e -> e.snapshot).collect(Collectors.toSet());
             final List<RestoreInProgress.Entry> entries = new ArrayList<>();
             final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index ddcee4b0353..519393d49e1 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -21,13 +21,12 @@ package org.elasticsearch.snapshots;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -42,9 +41,8 @@ import java.util.Objects;
 /**
  * Information about a snapshot
  */
-public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, FromXContentBuilder<SnapshotInfo>, Writeable {
+public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, Writeable {
 
-    public static final SnapshotInfo PROTO = new SnapshotInfo(new SnapshotId("", ""), Collections.emptyList(), 0);
     public static final String CONTEXT_MODE_PARAM = "context_mode";
     public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT";
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime");
@@ -70,6 +68,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
 
+    private static final Version VERSION_INCOMPATIBLE_INTRODUCED = Version.V_5_2_0_UNRELEASED;
+
     private final SnapshotId snapshotId;
 
     private final SnapshotState state;
@@ -86,6 +86,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     private final int successfulShards;
 
+    @Nullable
     private final Version version;
 
     private final List<SnapshotShardFailure> shardFailures;
@@ -141,7 +142,21 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         } else {
             shardFailures = Collections.emptyList();
         }
-        version = Version.readVersion(in);
+        if (in.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            version = Version.readVersion(in);
+        } else {
+            version = in.readBoolean() ? Version.readVersion(in) : null;
+        }
+    }
+
+    /**
+     * Gets a new {@link SnapshotInfo} instance for a snapshot that is incompatible with the
+     * current version of the cluster.
+     */
+    public static SnapshotInfo incompatible(SnapshotId snapshotId) {
+        return new SnapshotInfo(snapshotId, Collections.emptyList(), SnapshotState.INCOMPATIBLE,
+                                "the snapshot is incompatible with the current version of Elasticsearch and its exact version is unknown",
+                                null, 0L, 0L, 0, 0, Collections.emptyList());
     }
 
     /**
@@ -237,10 +252,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     }
 
     /**
-     * Returns the version of elasticsearch that the snapshot was created with
+     * Returns the version of elasticsearch that the snapshot was created with.  Will only
+     * return {@code null} if {@link #state()} returns {@link SnapshotState#INCOMPATIBLE}.
      *
      * @return version of elasticsearch that the snapshot was created with
      */
+    @Nullable
     public Version version() {
         return version;
     }
@@ -308,8 +325,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         builder.startObject();
         builder.field(SNAPSHOT, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
-        builder.field(VERSION_ID, version.id);
-        builder.field(VERSION, version.toString());
+        if (version != null) {
+            builder.field(VERSION_ID, version.id);
+            builder.field(VERSION, version.toString());
+        } else {
+            builder.field(VERSION, "unknown");
+        }
         builder.startArray(INDICES);
         for (String index : indices) {
             builder.value(index);
@@ -348,6 +369,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         builder.startObject(SNAPSHOT);
         builder.field(NAME, snapshotId.getName());
         builder.field(UUID, snapshotId.getUUID());
+        assert version != null : "version must always be known when writing a snapshot metadata blob";
         builder.field(VERSION_ID, version.id);
         builder.startArray(INDICES);
         for (String index : indices) {
@@ -373,11 +395,6 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         return builder;
     }
 
-    @Override
-    public SnapshotInfo fromXContent(final XContentParser parser, final ParseFieldMatcher matcher) throws IOException {
-        return fromXContent(parser);
-    }
-
     /**
      * This method creates a SnapshotInfo from internal x-content.  It does not
      * handle x-content written with the external version as external x-content
@@ -479,7 +496,11 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         for (String index : indices) {
             out.writeString(index);
         }
-        out.writeByte(state.value());
+        if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED) && state == SnapshotState.INCOMPATIBLE) {
+            out.writeByte(SnapshotState.FAILED.value());
+        } else {
+            out.writeByte(state.value());
+        }
         out.writeOptionalString(reason);
         out.writeVLong(startTime);
         out.writeVLong(endTime);
@@ -489,7 +510,20 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         for (SnapshotShardFailure failure : shardFailures) {
             failure.writeTo(out);
         }
-        Version.writeVersion(version, out);
+        if (out.getVersion().before(VERSION_INCOMPATIBLE_INTRODUCED)) {
+            Version versionToWrite = version;
+            if (versionToWrite == null) {
+                versionToWrite = Version.CURRENT;
+            }
+            Version.writeVersion(versionToWrite, out);
+        } else {
+            if (version != null) {
+                out.writeBoolean(true);
+                Version.writeVersion(version, out);
+            } else {
+                out.writeBoolean(false);
+            }
+        }
     }
 
     private static SnapshotState snapshotState(final String reason, final List<SnapshotShardFailure> shardFailures) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index d6598eb3a12..6b42d178694 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -27,13 +27,15 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -43,7 +45,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.engine.SnapshotFailedEngineException;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
@@ -68,7 +69,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Condition;
@@ -105,8 +105,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
 
     private volatile Map<Snapshot, SnapshotShards> shardSnapshots = emptyMap();
 
-    private final BlockingQueue<UpdateIndexShardSnapshotStatusRequest> updatedSnapshotStateQueue = ConcurrentCollections.newBlockingQueue();
-
+    private final SnapshotStateExecutor snapshotStateExecutor = new SnapshotStateExecutor();
 
     @Inject
     public SnapshotShardsService(Settings settings, ClusterService clusterService, SnapshotsService snapshotsService, ThreadPool threadPool,
@@ -458,8 +457,6 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
         private ShardId shardId;
         private ShardSnapshotStatus status;
 
-        private volatile boolean processed; // state field, no need to serialize
-
         public UpdateIndexShardSnapshotStatusRequest() {
 
         }
@@ -502,14 +499,6 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
         public String toString() {
             return "" + snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]";
         }
-
-        public void markAsProcessed() {
-            processed = true;
-        }
-
-        public boolean isProcessed() {
-            return processed;
-        }
     }
 
     /**
@@ -531,83 +520,65 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
      */
     private void innerUpdateSnapshotState(final UpdateIndexShardSnapshotStatusRequest request) {
         logger.trace("received updated snapshot restore state [{}]", request);
-        updatedSnapshotStateQueue.add(request);
+        clusterService.submitStateUpdateTask(
+            "update snapshot state",
+            request,
+            ClusterStateTaskConfig.build(Priority.NORMAL),
+            snapshotStateExecutor,
+            (source, e) -> logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}][{}] failed to update snapshot status to [{}]",
+                request.snapshot(), request.shardId(), request.status()), e));
+    }
 
-        clusterService.submitStateUpdateTask("update snapshot state", new ClusterStateUpdateTask() {
-            private final List<UpdateIndexShardSnapshotStatusRequest> drainedRequests = new ArrayList<>();
+    class SnapshotStateExecutor implements ClusterStateTaskExecutor<UpdateIndexShardSnapshotStatusRequest> {
 
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                // The request was already processed as a part of an early batch - skipping
-                if (request.isProcessed()) {
-                    return currentState;
-                }
+        @Override
+        public ClusterTasksResult<UpdateIndexShardSnapshotStatusRequest> execute(ClusterState currentState, List<UpdateIndexShardSnapshotStatusRequest> tasks) throws Exception {
+            final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
+            if (snapshots != null) {
+                int changedCount = 0;
+                final List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
+                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
+                    ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
+                    boolean updated = false;
 
-                updatedSnapshotStateQueue.drainTo(drainedRequests);
-
-                final int batchSize = drainedRequests.size();
-
-                // nothing to process (a previous event has processed it already)
-                if (batchSize == 0) {
-                    return currentState;
-                }
-
-                final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
-                if (snapshots != null) {
-                    int changedCount = 0;
-                    final List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
-                    for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
-                        ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
-                        boolean updated = false;
-
-                        for (int i = 0; i < batchSize; i++) {
-                            final UpdateIndexShardSnapshotStatusRequest updateSnapshotState = drainedRequests.get(i);
-                            updateSnapshotState.markAsProcessed();
-
-                            if (entry.snapshot().equals(updateSnapshotState.snapshot())) {
-                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
-                                if (updated == false) {
-                                    shards.putAll(entry.shards());
-                                    updated = true;
-                                }
-                                shards.put(updateSnapshotState.shardId(), updateSnapshotState.status());
-                                changedCount++;
+                    for (UpdateIndexShardSnapshotStatusRequest updateSnapshotState : tasks) {
+                        if (entry.snapshot().equals(updateSnapshotState.snapshot())) {
+                            logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
+                            if (updated == false) {
+                                shards.putAll(entry.shards());
+                                updated = true;
                             }
+                            shards.put(updateSnapshotState.shardId(), updateSnapshotState.status());
+                            changedCount++;
                         }
+                    }
 
-                        if (updated) {
-                            if (completed(shards.values()) == false) {
-                                entries.add(new SnapshotsInProgress.Entry(entry, shards.build()));
-                            } else {
-                                // Snapshot is finished - mark it as done
-                                // TODO: Add PARTIAL_SUCCESS status?
-                                SnapshotsInProgress.Entry updatedEntry = new SnapshotsInProgress.Entry(entry, State.SUCCESS, shards.build());
-                                entries.add(updatedEntry);
-                                // Finalize snapshot in the repository
-                                snapshotsService.endSnapshot(updatedEntry);
-                                logger.info("snapshot [{}] is done", updatedEntry.snapshot());
-                            }
+                    if (updated) {
+                        if (completed(shards.values()) == false) {
+                            entries.add(new SnapshotsInProgress.Entry(entry, shards.build()));
                         } else {
-                            entries.add(entry);
+                            // Snapshot is finished - mark it as done
+                            // TODO: Add PARTIAL_SUCCESS status?
+                            SnapshotsInProgress.Entry updatedEntry = new SnapshotsInProgress.Entry(entry, State.SUCCESS, shards.build());
+                            entries.add(updatedEntry);
+                            // Finalize snapshot in the repository
+                            snapshotsService.endSnapshot(updatedEntry);
+                            logger.info("snapshot [{}] is done", updatedEntry.snapshot());
                         }
-                    }
-                    if (changedCount > 0) {
-                        logger.trace("changed cluster state triggered by {} snapshot state updates", changedCount);
-
-                        final SnapshotsInProgress updatedSnapshots = new SnapshotsInProgress(entries.toArray(new SnapshotsInProgress.Entry[entries.size()]));
-                        return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, updatedSnapshots).build();
+                    } else {
+                        entries.add(entry);
                     }
                 }
-                return currentState;
-            }
+                if (changedCount > 0) {
+                    logger.trace("changed cluster state triggered by {} snapshot state updates", changedCount);
 
-            @Override
-            public void onFailure(String source, Exception e) {
-                for (UpdateIndexShardSnapshotStatusRequest request : drainedRequests) {
-                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}][{}] failed to update snapshot status to [{}]", request.snapshot(), request.shardId(), request.status()), e);
+                    final SnapshotsInProgress updatedSnapshots = new SnapshotsInProgress(entries.toArray(new SnapshotsInProgress.Entry[entries.size()]));
+                    return ClusterTasksResult.<UpdateIndexShardSnapshotStatusRequest>builder().successes(tasks).build(
+                        ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, updatedSnapshots).build());
                 }
             }
-        });
+            return ClusterTasksResult.<UpdateIndexShardSnapshotStatusRequest>builder().successes(tasks).build(currentState);
+        }
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
index b893a372d13..3df5f8fff04 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotState.java
@@ -39,7 +39,11 @@ public enum SnapshotState {
     /**
      * Snapshot was partial successful
      */
-    PARTIAL((byte) 3, true, true);
+    PARTIAL((byte) 3, true, true),
+    /**
+     * Snapshot is incompatible with the current version of the cluster
+     */
+    INCOMPATIBLE((byte) 4, true, false);
 
     private byte value;
 
@@ -47,7 +51,7 @@ public enum SnapshotState {
 
     private boolean restorable;
 
-    private SnapshotState(byte value, boolean completed, boolean restorable) {
+    SnapshotState(byte value, boolean completed, boolean restorable) {
         this.value = value;
         this.completed = completed;
         this.restorable = restorable;
@@ -97,6 +101,8 @@ public enum SnapshotState {
                 return FAILED;
             case 3:
                 return PARTIAL;
+            case 4:
+                return INCOMPATIBLE;
             default:
                 throw new IllegalArgumentException("No snapshot state for value [" + value + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 056b2e7b10d..d6f8cf8b586 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -33,6 +33,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
@@ -48,6 +49,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -129,15 +131,15 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     }
 
     /**
-     * Retrieves list of snapshot ids that are present in a repository
+     * Gets the {@link RepositoryData} for the given repository.
      *
      * @param repositoryName repository name
-     * @return list of snapshot ids
+     * @return repository data
      */
-    public List<SnapshotId> snapshotIds(final String repositoryName) {
+    public RepositoryData getRepositoryData(final String repositoryName) {
         Repository repository = repositoriesService.repository(repositoryName);
         assert repository != null; // should only be called once we've validated the repository exists
-        return repository.getRepositoryData().getSnapshotIds();
+        return repository.getRepositoryData();
     }
 
     /**
@@ -232,7 +234,11 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
             @Override
             public ClusterState execute(ClusterState currentState) {
                 validate(request, currentState);
-
+                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName,
+                        "cannot snapshot while a snapshot deletion is in-progress");
+                }
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                 if (snapshots == null || snapshots.entries().isEmpty()) {
                     // Store newSnapshot here to be processed in clusterStateProcessed
@@ -245,10 +251,10 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                                 State.INIT,
                                                                 snapshotIndices,
                                                                 System.currentTimeMillis(),
+                                                                repositoryData.getGenId(),
                                                                 null);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
-                    // TODO: What should we do if a snapshot is already running?
                     throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "a snapshot is already running");
                 }
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
@@ -441,7 +447,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         private final CreateSnapshotListener userCreateSnapshotListener;
         private final Exception e;
 
-        public CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Exception e) {
+        CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Exception e) {
             this.snapshot = snapshot;
             this.snapshotCreated = snapshotCreated;
             this.userCreateSnapshotListener = userCreateSnapshotListener;
@@ -468,7 +474,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                          snapshot.startTime(),
                                                          ExceptionsHelper.detailedMessage(exception),
                                                          0,
-                                                         Collections.emptyList());
+                                                         Collections.emptyList(),
+                                                         snapshot.getRepositoryStateId());
                 } catch (Exception inner) {
                     inner.addSuppressed(exception);
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to close snapshot in repository", snapshot.snapshot()), inner);
@@ -601,12 +608,35 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 if (event.routingTableChanged()) {
                     processStartedShards(event);
                 }
+                finalizeSnapshotDeletionFromPreviousMaster(event);
             }
         } catch (Exception e) {
             logger.warn("Failed to update snapshot state ", e);
         }
     }
 
+    /**
+     * Finalizes a snapshot deletion in progress if the current node is the master but it
+     * was not master in the previous cluster state and there is still a lingering snapshot
+     * deletion in progress in the cluster state.  This means that the old master failed
+     * before it could clean up an in-progress snapshot deletion.  We attempt to delete the
+     * snapshot files and remove the deletion from the cluster state.  It is possible that the
+     * old master was in a state of long GC and then it resumes and tries to delete the snapshot
+     * that has already been deleted by the current master.  This is acceptable however, since
+     * the old master's snapshot deletion will just respond with an error but in actuality, the
+     * snapshot was deleted and a call to GET snapshots would reveal that the snapshot no longer exists.
+     */
+    private void finalizeSnapshotDeletionFromPreviousMaster(ClusterChangedEvent event) {
+        if (event.localNodeMaster() && event.previousState().nodes().isLocalNodeElectedMaster() == false) {
+            SnapshotDeletionsInProgress deletionsInProgress = event.state().custom(SnapshotDeletionsInProgress.TYPE);
+            if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster";
+                SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0);
+                deleteSnapshotFromRepository(entry.getSnapshot(), null, entry.getRepositoryStateId());
+            }
+        }
+    }
+
     /**
      * Cleans up shard snapshots that were running on removed nodes
      *
@@ -667,7 +697,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                 public void onFailure(Exception e) {
                                     logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                                 }
-                            });
+                            }, updatedSnapshot.getRepositoryStateId(), false);
                         } else if (snapshot.state() == State.SUCCESS && newMaster) {
                             // Finalize the snapshot
                             endSnapshot(snapshot);
@@ -875,7 +905,14 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                             shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                         }
                     }
-                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
+                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(
+                        snapshot.getSnapshotId(),
+                        entry.indices(),
+                        entry.startTime(),
+                        failure,
+                        entry.shards().size(),
+                        Collections.unmodifiableList(shardFailures),
+                        entry.getRepositoryStateId());
                     removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
                 } catch (Exception e) {
                     logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
@@ -904,6 +941,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Exception failure,
                                                 @Nullable ActionListener<SnapshotInfo> listener) {
         clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() {
+
             @Override
             public ClusterState execute(ClusterState currentState) {
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
@@ -961,10 +999,17 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
      * @param snapshotName    snapshotName
      * @param listener        listener
      */
-    public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener) {
+    public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener,
+                               final boolean immediatePriority) {
         // First, look for the snapshot in the repository
         final Repository repository = repositoriesService.repository(repositoryName);
-        Optional<SnapshotId> matchedEntry = repository.getRepositoryData().getSnapshotIds()
+        final RepositoryData repositoryData = repository.getRepositoryData();
+        final Optional<SnapshotId> incompatibleSnapshotId =
+            repositoryData.getIncompatibleSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
+        if (incompatibleSnapshotId.isPresent()) {
+            throw new SnapshotException(repositoryName, snapshotName, "cannot delete incompatible snapshot");
+        }
+        Optional<SnapshotId> matchedEntry = repositoryData.getSnapshotIds()
                                                 .stream()
                                                 .filter(s -> s.getName().equals(snapshotName))
                                                 .findFirst();
@@ -976,7 +1021,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         if (matchedEntry.isPresent() == false) {
             throw new SnapshotMissingException(repositoryName, snapshotName);
         }
-        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener);
+        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener, repositoryData.getGenId(), immediatePriority);
     }
 
     /**
@@ -984,16 +1029,23 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
      * <p>
      * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
      *
-     * @param snapshot  snapshot
-     * @param listener  listener
+     * @param snapshot snapshot
+     * @param listener listener
+     * @param repositoryStateId the unique id for the state of the repository
      */
-    public void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener) {
-        clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask() {
+    private void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener, final long repositoryStateId,
+                                final boolean immediatePriority) {
+        Priority priority = immediatePriority ? Priority.IMMEDIATE : Priority.NORMAL;
+        clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(priority) {
 
             boolean waitForSnapshot = false;
 
             @Override
             public ClusterState execute(ClusterState currentState) throws Exception {
+                SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
+                    throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete - another snapshot is currently being deleted");
+                }
                 RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
                 if (restoreInProgress != null) {
                     // don't allow snapshot deletions while a restore is taking place,
@@ -1003,19 +1055,27 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete snapshot during a restore");
                     }
                 }
+                ClusterState.Builder clusterStateBuilder = ClusterState.builder(currentState);
                 SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
-                if (snapshots == null) {
-                    // No snapshots running - we can continue
-                    return currentState;
-                }
-                SnapshotsInProgress.Entry snapshotEntry = snapshots.snapshot(snapshot);
+                SnapshotsInProgress.Entry snapshotEntry = snapshots != null ? snapshots.snapshot(snapshot) : null;
                 if (snapshotEntry == null) {
-                    // This snapshot is not running - continue
-                    if (!snapshots.entries().isEmpty()) {
+                    // This snapshot is not running - delete
+                    if (snapshots != null && !snapshots.entries().isEmpty()) {
                         // However other snapshots are running - cannot continue
                         throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete");
                     }
-                    return currentState;
+                    // add the snapshot deletion to the cluster state
+                    SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(
+                        snapshot,
+                        System.currentTimeMillis(),
+                        repositoryStateId
+                    );
+                    if (deletionsInProgress != null) {
+                        deletionsInProgress = deletionsInProgress.withAddedEntry(entry);
+                    } else {
+                        deletionsInProgress = SnapshotDeletionsInProgress.newInstance(entry);
+                    }
+                    clusterStateBuilder.putCustom(SnapshotDeletionsInProgress.TYPE, deletionsInProgress);
                 } else {
                     // This snapshot is currently running - stopping shards first
                     waitForSnapshot = true;
@@ -1060,8 +1120,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                     }
                     SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards);
                     snapshots = new SnapshotsInProgress(newSnapshot);
-                    return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
+                    clusterStateBuilder.putCustom(SnapshotsInProgress.TYPE, snapshots);
                 }
+                return clusterStateBuilder.build();
             }
 
             @Override
@@ -1079,7 +1140,10 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                             if (completedSnapshot.equals(snapshot)) {
                                 logger.trace("deleted snapshot completed - deleting files");
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshot, listener);
+                                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() ->
+                                    deleteSnapshot(completedSnapshot.getRepository(), completedSnapshot.getSnapshotId().getName(),
+                                        listener, true)
+                                );
                             }
                         }
 
@@ -1088,13 +1152,15 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                             if (failedSnapshot.equals(snapshot)) {
                                 logger.trace("deleted snapshot failed - deleting files", e);
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshot, listener);
+                                threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() ->
+                                    deleteSnapshot(failedSnapshot.getRepository(), failedSnapshot.getSnapshotId().getName(), listener, true)
+                                );
                             }
                         }
                     });
                 } else {
                     logger.trace("deleted snapshot is not running - deleting files");
-                    deleteSnapshotFromRepository(snapshot, listener);
+                    deleteSnapshotFromRepository(snapshot, listener, repositoryStateId);
                 }
             }
         });
@@ -1116,6 +1182,14 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 }
             }
         }
+        SnapshotDeletionsInProgress deletionsInProgress = clusterState.custom(SnapshotDeletionsInProgress.TYPE);
+        if (deletionsInProgress != null) {
+            for (SnapshotDeletionsInProgress.Entry entry : deletionsInProgress.getEntries()) {
+                if (entry.getSnapshot().getRepository().equals(repository)) {
+                    return true;
+                }
+            }
+        }
         return false;
     }
 
@@ -1124,15 +1198,62 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
      *
      * @param snapshot   snapshot
      * @param listener   listener
+     * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began
      */
-    private void deleteSnapshotFromRepository(final Snapshot snapshot, final DeleteSnapshotListener listener) {
+    private void deleteSnapshotFromRepository(final Snapshot snapshot, @Nullable final DeleteSnapshotListener listener,
+                                              long repositoryStateId) {
         threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
             try {
                 Repository repository = repositoriesService.repository(snapshot.getRepository());
-                repository.deleteSnapshot(snapshot.getSnapshotId());
-                listener.onResponse();
-            } catch (Exception t) {
-                listener.onFailure(t);
+                repository.deleteSnapshot(snapshot.getSnapshotId(), repositoryStateId);
+                removeSnapshotDeletionFromClusterState(snapshot, null, listener);
+            } catch (Exception ex) {
+                removeSnapshotDeletionFromClusterState(snapshot, ex, listener);
+            }
+        });
+    }
+
+    /**
+     * Removes the snapshot deletion from {@link SnapshotDeletionsInProgress} in the cluster state.
+     */
+    private void removeSnapshotDeletionFromClusterState(final Snapshot snapshot, @Nullable final Exception failure,
+                                                        @Nullable final DeleteSnapshotListener listener) {
+        clusterService.submitStateUpdateTask("remove snapshot deletion metadata", new ClusterStateUpdateTask() {
+            @Override
+            public ClusterState execute(ClusterState currentState) {
+                SnapshotDeletionsInProgress deletions = currentState.custom(SnapshotDeletionsInProgress.TYPE);
+                if (deletions != null) {
+                    boolean changed = false;
+                    if (deletions.hasDeletionsInProgress()) {
+                        assert deletions.getEntries().size() == 1 : "should have exactly one deletion in progress";
+                        SnapshotDeletionsInProgress.Entry entry = deletions.getEntries().get(0);
+                        deletions = deletions.withRemovedEntry(entry);
+                        changed = true;
+                    }
+                    if (changed) {
+                        return ClusterState.builder(currentState).putCustom(SnapshotDeletionsInProgress.TYPE, deletions).build();
+                    }
+                }
+                return currentState;
+            }
+
+            @Override
+            public void onFailure(String source, Exception e) {
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to remove snapshot deletion metadata", snapshot), e);
+                if (listener != null) {
+                    listener.onFailure(e);
+                }
+            }
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                if (listener != null) {
+                    if (failure != null) {
+                        listener.onFailure(failure);
+                    } else {
+                        listener.onResponse();
+                    }
+                }
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java b/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
index eb084a57e10..b3c1a8929a6 100644
--- a/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
+++ b/core/src/main/java/org/elasticsearch/tasks/CancellableTask.java
@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicReference;
 /**
  * A task that can be canceled
  */
-public class CancellableTask extends Task {
+public abstract class CancellableTask extends Task {
 
     private final AtomicReference<String> reason = new AtomicReference<>();
 
@@ -51,6 +51,11 @@ public class CancellableTask extends Task {
         return true;
     }
 
+    /**
+     * Returns true if this task should can potentially have children that needs to be cancelled when the parent is cancelled.
+     */
+    public abstract boolean shouldCancelChildrenOnCancellation();
+
     public boolean isCancelled() {
         return reason.get() != null;
     }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
index 9792b894b58..59b3ef95306 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.tasks;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -164,12 +163,6 @@ public final class TaskInfo implements Writeable, ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        innerToXContent(builder, params);
-        return builder.endObject();
-    }
-
-    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("node", taskId.getNodeId());
         builder.field("id", taskId.getId());
         builder.field("type", type);
@@ -189,7 +182,7 @@ public final class TaskInfo implements Writeable, ToXContent {
         return builder;
     }
 
-    public static final ConstructingObjectParser<TaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<TaskInfo, Void> PARSER = new ConstructingObjectParser<>(
             "task_info", a -> {
                 int i = 0;
                 TaskId id = new TaskId((String) a[i++], (Long) a[i++]);
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 61c36f9015d..bf62b5bb013 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -125,15 +125,18 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
     /**
      * Cancels a task
      * <p>
-     * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+     * Returns true if cancellation was started successful, null otherwise.
+     *
+     * After starting cancellation on the parent task, the task manager tries to cancel all children tasks
+     * of the current task. Once cancellation of the children tasks is done, the listener is triggered.
      */
-    public Set<String> cancel(CancellableTask task, String reason, Consumer<Set<String>> listener) {
+    public boolean cancel(CancellableTask task, String reason, Runnable listener) {
         CancellableTaskHolder holder = cancellableTasks.get(task.getId());
         if (holder != null) {
             logger.trace("cancelling task with id {}", task.getId());
             return holder.cancel(reason, listener);
         }
-        return null;
+        return false;
     }
 
     /**
@@ -344,17 +347,6 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
         }
     }
 
-    public void registerChildTask(Task task, String node) {
-        if (task == null || task instanceof CancellableTask == false) {
-            // We don't have a cancellable task - not much we can do here
-            return;
-        }
-        CancellableTaskHolder holder = cancellableTasks.get(task.getId());
-        if (holder != null) {
-            holder.registerChildTaskNode(node);
-        }
-    }
-
     /**
      * Blocks the calling thread, waiting for the task to vanish from the TaskManager.
      */
@@ -378,46 +370,44 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
 
         private final CancellableTask task;
 
-        private final Set<String> nodesWithChildTasks = new HashSet<>();
-
         private volatile String cancellationReason = null;
 
-        private volatile Consumer<Set<String>> cancellationListener = null;
+        private volatile Runnable cancellationListener = null;
 
-        public CancellableTaskHolder(CancellableTask task) {
+        CancellableTaskHolder(CancellableTask task) {
             this.task = task;
         }
 
         /**
          * Marks task as cancelled.
          * <p>
-         * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+         * Returns true if cancellation was successful, false otherwise.
          */
-        public Set<String> cancel(String reason, Consumer<Set<String>> listener) {
-            Set<String> nodes;
+        public boolean cancel(String reason, Runnable listener) {
+            final boolean cancelled;
             synchronized (this) {
                 assert reason != null;
                 if (cancellationReason == null) {
                     cancellationReason = reason;
                     cancellationListener = listener;
-                    nodes = Collections.unmodifiableSet(nodesWithChildTasks);
+                    cancelled = true;
                 } else {
                     // Already cancelled by somebody else
-                    nodes = null;
+                    cancelled = false;
                 }
             }
-            if (nodes != null) {
+            if (cancelled) {
                 task.cancel(reason);
             }
-            return nodes;
+            return cancelled;
         }
 
         /**
          * Marks task as cancelled.
          * <p>
-         * Returns a set of nodes with child tasks where this task should be cancelled if cancellation was successful, null otherwise.
+         * Returns true if cancellation was successful, false otherwise.
          */
-        public Set<String> cancel(String reason) {
+        public boolean cancel(String reason) {
             return cancel(reason, null);
         }
 
@@ -425,14 +415,12 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
          * Marks task as finished.
          */
         public void finish() {
-            Consumer<Set<String>> listener = null;
-            Set<String> nodes = null;
+            Runnable listener = null;
             synchronized (this) {
                 if (cancellationReason != null) {
                     // The task was cancelled, we need to notify the listener
                     if (cancellationListener != null) {
                         listener = cancellationListener;
-                        nodes = Collections.unmodifiableSet(nodesWithChildTasks);
                         cancellationListener = null;
                     }
                 } else {
@@ -442,7 +430,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
             // We need to call the listener outside of the synchronised section to avoid potential bottle necks
             // in the listener synchronization
             if (listener != null) {
-                listener.accept(nodes);
+                listener.run();
             }
 
         }
@@ -454,14 +442,6 @@ public class TaskManager extends AbstractComponent implements ClusterStateApplie
         public CancellableTask getTask() {
             return task;
         }
-
-        public synchronized void registerChildTaskNode(String nodeId) {
-            if (cancellationReason == null) {
-                nodesWithChildTasks.add(nodeId);
-            } else {
-                throw new TaskCancelledException("cannot register child task request, the task is already cancelled");
-            }
-        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
index b5918ad9593..ba80879b5df 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
@@ -22,7 +22,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -161,7 +160,9 @@ public final class TaskResult implements Writeable, ToXContent {
 
     public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("completed", completed);
-        builder.field("task", task);
+        builder.startObject("task");
+        task.toXContent(builder, params);
+        builder.endObject();
         if (error != null) {
             XContentHelper.writeRawField("error", error, builder, params);
         }
@@ -171,7 +172,7 @@ public final class TaskResult implements Writeable, ToXContent {
         return builder;
     }
 
-    public static final ConstructingObjectParser<TaskResult, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<TaskResult, Void> PARSER = new ConstructingObjectParser<>(
             "stored_task_result", a -> {
                 int i = 0;
                 boolean completed = (boolean) a[i++];
@@ -231,7 +232,7 @@ public final class TaskResult implements Writeable, ToXContent {
     private static BytesReference toXContent(Exception error) throws IOException {
         try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
             builder.startObject();
-            ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, error);
+            ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, error);
             builder.endObject();
             return builder.bytes();
         }
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
index 0f9b79156e6..54f5ab0af38 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
@@ -82,7 +82,7 @@ public abstract class ExecutorBuilder<U extends ExecutorBuilder.ExecutorSettings
 
         protected final String nodeName;
 
-        public ExecutorSettings(String nodeName) {
+        ExecutorSettings(String nodeName) {
             this.nodeName = nodeName;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
index 0ba7b2d100a..162dcc80d7f 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
@@ -123,7 +123,7 @@ public final class FixedExecutorBuilder extends ExecutorBuilder<FixedExecutorBui
         private final int size;
         private final int queueSize;
 
-        public FixedExecutorSettings(final String nodeName, final int size, final int queueSize) {
+        FixedExecutorSettings(final String nodeName, final int size, final int queueSize) {
             super(nodeName);
             this.size = size;
             this.queueSize = queueSize;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
index 96dceb8c912..06df1fcf8b1 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
@@ -118,7 +118,7 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder<ScalingExecuto
         private final int max;
         private final TimeValue keepAlive;
 
-        public ScalingExecutorSettings(final String nodeName, final int core, final int max, final TimeValue keepAlive) {
+        ScalingExecutorSettings(final String nodeName, final int core, final int max, final TimeValue keepAlive) {
             super(nodeName);
             this.core = core;
             this.max = max;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 15d3995950b..6817ae991a3 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -169,7 +169,7 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         final int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512);
         builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30)));
         builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200));
-        builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 50));
+        builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 200)); // now that we reuse bulk for index/delete ops
         builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, availableProcessors, 1000));
         builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(availableProcessors), 1000));
         builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
diff --git a/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java b/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
index 92421adea6a..2dc605cf3d4 100644
--- a/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
+++ b/core/src/main/java/org/elasticsearch/transport/ConnectionProfile.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.transport;
 
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.util.ArrayList;
@@ -35,16 +36,25 @@ import java.util.concurrent.atomic.AtomicInteger;
 public final class ConnectionProfile {
 
     /**
-     * A pre-built light connection profile that shares a single connection across all
-     * types.
+     * Builds a connection profile that is dedicated to a single channel type. Use this
+     * when opening single use connections
      */
-    public static final ConnectionProfile LIGHT_PROFILE = new ConnectionProfile(
-        Collections.singletonList(new ConnectionTypeHandle(0, 1, EnumSet.of(
-            TransportRequestOptions.Type.BULK,
-            TransportRequestOptions.Type.PING,
-            TransportRequestOptions.Type.RECOVERY,
-            TransportRequestOptions.Type.REG,
-            TransportRequestOptions.Type.STATE))), 1, null, null);
+    public static ConnectionProfile buildSingleChannelProfile(TransportRequestOptions.Type channelType,
+                                                              @Nullable TimeValue connectTimeout,
+                                                               @Nullable TimeValue handshakeTimeout) {
+        Builder builder = new Builder();
+        builder.addConnections(1, channelType);
+        final EnumSet<TransportRequestOptions.Type> otherTypes = EnumSet.allOf(TransportRequestOptions.Type.class);
+        otherTypes.remove(channelType);
+        builder.addConnections(0, otherTypes.stream().toArray(TransportRequestOptions.Type[]::new));
+        if (connectTimeout != null) {
+            builder.setConnectTimeout(connectTimeout);
+        }
+        if (handshakeTimeout != null) {
+            builder.setHandshakeTimeout(handshakeTimeout);
+        }
+        return builder.build();
+    }
 
     private final List<ConnectionTypeHandle> handles;
     private final int numConnections;
@@ -188,7 +198,7 @@ public final class ConnectionProfile {
          */
         <T> T getChannel(T[] channels) {
             if (length == 0) {
-                throw new IllegalStateException("can't select channel size is 0");
+                throw new IllegalStateException("can't select channel size is 0 for types: " + types);
             }
             assert channels.length >= offset + length : "illegal size: " + channels.length + " expected >= " + (offset + length);
             return channels[offset + Math.floorMod(counter.incrementAndGet(), length)];
diff --git a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
index b4a79edbc0b..2e56ff91021 100644
--- a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
+++ b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
@@ -96,7 +96,7 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
 
         private final TaskManager taskManager;
 
-        public TransportChannelWrapper(TaskManager taskManager, Task task, TransportChannel channel) {
+        TransportChannelWrapper(TaskManager taskManager, Task task, TransportChannel channel) {
             super(channel);
             this.task = task;
             this.taskManager = taskManager;
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 2e8cb4f65ce..5cfb5f7a423 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -112,8 +112,6 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new
 public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent implements Transport {
 
     public static final String TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX = "transport_server_worker";
-    public static final String TRANSPORT_SERVER_BOSS_THREAD_NAME_PREFIX = "transport_server_boss";
-    public static final String TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX = "transport_client_worker";
     public static final String TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX = "transport_client_boss";
 
     // the scheduled internal ping interval setting, defaults to disabled (-1)
@@ -137,10 +135,6 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         boolSetting("transport.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Setting.Property.NodeScope);
     public static final Setting<Boolean> TCP_REUSE_ADDRESS =
         boolSetting("transport.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Setting.Property.NodeScope);
-    public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
-        boolSetting("transport.tcp.blocking_client", NetworkService.TcpSettings.TCP_BLOCKING_CLIENT, Setting.Property.NodeScope);
-    public static final Setting<Boolean> TCP_BLOCKING_SERVER =
-        boolSetting("transport.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Setting.Property.NodeScope);
     public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
         Setting.byteSizeSetting("transport.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE,
             Setting.Property.NodeScope);
@@ -150,7 +144,6 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
 
     private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.9);
     private static final int PING_DATA_SIZE = -1;
-    protected final boolean blockingClient;
     private final CircuitBreakerService circuitBreakerService;
     // package visibility for tests
     protected final ScheduledPing scheduledPing;
@@ -194,7 +187,6 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
         this.networkService = networkService;
         this.transportName = transportName;
-        this.blockingClient = TCP_BLOCKING_CLIENT.get(settings);
         defaultConnectionProfile = buildDefaultConnectionProfile(settings);
     }
 
@@ -243,7 +235,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
         final Channel channel;
 
-        public HandshakeResponseHandler(Channel channel) {
+        HandshakeResponseHandler(Channel channel) {
             this.channel = channel;
         }
 
@@ -432,19 +424,13 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     @Override
     public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile) {
         connectionProfile = connectionProfile == null ? defaultConnectionProfile : connectionProfile;
-        if (!lifecycle.started()) {
-            throw new IllegalStateException("can't add nodes to a stopped transport");
-        }
         if (node == null) {
             throw new ConnectTransportException(null, "can't connect to a null node");
         }
-        globalLock.readLock().lock();
+        globalLock.readLock().lock(); // ensure we don't open connections while we are closing
         try {
-
+            ensureOpen();
             try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                if (!lifecycle.started()) {
-                    throw new IllegalStateException("can't add nodes to a stopped transport");
-                }
                 NodeChannels nodeChannels = connectedNodes.get(node);
                 if (nodeChannels != null) {
                     return;
@@ -458,13 +444,6 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                                 "failed to connect to [{}], cleaning dangling connections", node), e);
                         throw e;
                     }
-                    Channel channel = nodeChannels.channel(TransportRequestOptions.Type.PING);
-                    final TimeValue connectTimeout = connectionProfile.getConnectTimeout() == null ?
-                        defaultConnectionProfile.getConnectTimeout() :
-                        connectionProfile.getConnectTimeout();
-                    final TimeValue handshakeTimeout = connectionProfile.getHandshakeTimeout() == null ?
-                        connectTimeout : connectionProfile.getHandshakeTimeout();
-                    Version version = executeHandshake(node, channel, handshakeTimeout);
                     // we acquire a connection lock, so no way there is an existing connection
                     connectedNodes.put(node, nodeChannels);
                     if (logger.isDebugEnabled()) {
@@ -483,17 +462,41 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     }
 
     @Override
-    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile profile) throws IOException {
+    public final NodeChannels openConnection(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException {
+        if (node == null) {
+            throw new ConnectTransportException(null, "can't open connection to a null node");
+        }
+        boolean success = false;
+        NodeChannels nodeChannels = null;
+        globalLock.readLock().lock(); // ensure we don't open connections while we are closing
         try {
-            NodeChannels nodeChannels = connectToChannels(node, profile);
-            transportServiceAdapter.onConnectionOpened(node);
-            return nodeChannels;
-        } catch (ConnectTransportException e) {
-            throw e;
-        } catch (Exception e) {
-            // ConnectTransportExceptions are handled specifically on the caller end - we wrap the actual exception to ensure
-            // only relevant exceptions are logged on the caller end.. this is the same as in connectToNode
-            throw new ConnectTransportException(node, "general node connection failure", e);
+            ensureOpen();
+            try {
+                nodeChannels = connectToChannels(node, connectionProfile);
+                final Channel channel = nodeChannels.getChannels().get(0); // one channel is guaranteed by the connection profile
+                final TimeValue connectTimeout = connectionProfile.getConnectTimeout() == null ?
+                    defaultConnectionProfile.getConnectTimeout() :
+                    connectionProfile.getConnectTimeout();
+                final TimeValue handshakeTimeout = connectionProfile.getHandshakeTimeout() == null ?
+                    connectTimeout : connectionProfile.getHandshakeTimeout();
+                final Version version = executeHandshake(node, channel, handshakeTimeout);
+                transportServiceAdapter.onConnectionOpened(node);
+                nodeChannels = new NodeChannels(nodeChannels, version);// clone the channels - we now have the correct version
+                success = true;
+                return nodeChannels;
+            } catch (ConnectTransportException e) {
+                throw e;
+            } catch (Exception e) {
+                // ConnectTransportExceptions are handled specifically on the caller end - we wrap the actual exception to ensure
+                // only relevant exceptions are logged on the caller end.. this is the same as in connectToNode
+                throw new ConnectTransportException(node, "general node connection failure", e);
+            } finally {
+                if (success == false) {
+                    IOUtils.closeWhileHandlingException(nodeChannels);
+                }
+            }
+        } finally {
+            globalLock.readLock().unlock();
         }
     }
 
@@ -853,7 +856,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                     }
                 }
 
-                for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
+                for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext();) {
                     NodeChannels nodeChannels = it.next();
                     it.remove();
                     IOUtils.closeWhileHandlingException(nodeChannels);
@@ -875,7 +878,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
 
     protected void onException(Channel channel, Exception e) throws IOException {
         if (!lifecycle.started()) {
-            // ignore
+            // just close and ignore - we are already stopped and just need to make sure we release all resources
+            disconnectFromNodeChannel(channel, e);
             return;
         }
         if (isCloseConnectionException(e)) {
@@ -1440,7 +1444,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         private final TransportRequest request;
         private final TransportChannel transportChannel;
 
-        public RequestHandler(RequestHandlerRegistry reg, TransportRequest request, TransportChannel transportChannel) {
+        RequestHandler(RequestHandlerRegistry reg, TransportRequest request, TransportChannel transportChannel) {
             this.reg = reg;
             this.request = request;
             this.transportChannel = transportChannel;
@@ -1569,4 +1573,14 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             }
         }
     }
+
+    /**
+     * Ensures this transport is still started / open
+     * @throws IllegalStateException if the transport is not started / open
+     */
+    protected final void ensureOpen() {
+        if (lifecycle.started() == false) {
+            throw new IllegalStateException("transport has been stopped");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/Transport.java b/core/src/main/java/org/elasticsearch/transport/Transport.java
index e337aaf41b2..44c72e1f548 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transport.java
@@ -63,8 +63,7 @@ public interface Transport extends LifecycleComponent {
     boolean nodeConnected(DiscoveryNode node);
 
     /**
-     * Connects to a node with the given connection profile. Use {@link ConnectionProfile#LIGHT_PROFILE} when just connecting for ping
-     * and then disconnecting. If the node is already connected this method has no effect
+     * Connects to a node with the given connection profile. If the node is already connected this method has no effect
      */
     void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile) throws ConnectTransportException;
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
new file mode 100644
index 00000000000..30b7299f59d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TransportActionProxy.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+/**
+ * TransportActionProxy allows an arbitrary action to be executed on a defined target node while the initial request is sent to a second
+ * node that acts as a request proxy to the target node. This is useful if a node is not directly connected to a target node but is
+ * connected to an intermediate node that establishes a transitive connection.
+ */
+public final class TransportActionProxy {
+
+    private TransportActionProxy() {} // no instance
+
+    private static class ProxyRequestHandler<T extends ProxyRequest> implements TransportRequestHandler<T> {
+
+        private final TransportService service;
+        private final String action;
+        private final Supplier<TransportResponse> responseFactory;
+
+        ProxyRequestHandler(TransportService service, String action, Supplier<TransportResponse> responseFactory) {
+            this.service = service;
+            this.action = action;
+            this.responseFactory = responseFactory;
+        }
+
+        @Override
+        public void messageReceived(T request, TransportChannel channel) throws Exception {
+            DiscoveryNode targetNode = request.targetNode;
+            TransportRequest wrappedRequest = request.wrapped;
+            service.sendRequest(targetNode, action, wrappedRequest, new ProxyResponseHandler<>(channel, responseFactory));
+        }
+    }
+
+    private static class ProxyResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+
+        private final Supplier<T> responseFactory;
+        private final TransportChannel channel;
+
+        ProxyResponseHandler(TransportChannel channel, Supplier<T> responseFactory) {
+            this.responseFactory = responseFactory;
+            this.channel = channel;
+
+        }
+        @Override
+        public T newInstance() {
+            return responseFactory.get();
+        }
+
+        @Override
+        public void handleResponse(T response) {
+            try {
+                channel.sendResponse(response);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public void handleException(TransportException exp) {
+            try {
+                channel.sendResponse(exp);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }
+
+        @Override
+        public String executor() {
+            return ThreadPool.Names.SAME;
+        }
+    }
+
+    static class ProxyRequest<T extends TransportRequest> extends TransportRequest {
+        T wrapped;
+        Supplier<T> supplier;
+        DiscoveryNode targetNode;
+
+        ProxyRequest(Supplier<T> supplier) {
+            this.supplier = supplier;
+        }
+
+        ProxyRequest(T wrapped, DiscoveryNode targetNode) {
+            this.wrapped = wrapped;
+            this.targetNode = targetNode;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            targetNode = new DiscoveryNode(in);
+            wrapped = supplier.get();
+            wrapped.readFrom(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            targetNode.writeTo(out);
+            wrapped.writeTo(out);
+        }
+    }
+
+    /**
+     * Registers a proxy request handler that allows to forward requests for the given action to another node.
+     */
+    public static void registerProxyAction(TransportService service, String action, Supplier<TransportResponse> responseSupplier) {
+        RequestHandlerRegistry requestHandler = service.getRequestHandler(action);
+        service.registerRequestHandler(getProxyAction(action), () -> new ProxyRequest(requestHandler::newRequest), ThreadPool.Names.SAME,
+            true, false, new ProxyRequestHandler<>(service, action, responseSupplier));
+    }
+
+    /**
+     * Returns the corresponding proxy action for the given action
+     */
+    public static String getProxyAction(String action) {
+        return "internal:transport/proxy/" + action;
+    }
+
+    /**
+     * Wraps the actual request in a proxy request object that encodes the target node.
+     */
+    public static TransportRequest wrapRequest(DiscoveryNode node, TransportRequest request) {
+        return new ProxyRequest<>(request, node);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java b/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java
index 8dfcf39bd9d..cc90a6c6d23 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportInterceptor.java
@@ -34,6 +34,7 @@ public interface TransportInterceptor {
      * used instead of the passed in handler. By default the provided handler is returned.
      */
     default <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                     boolean forceExecution,
                                                                                      TransportRequestHandler<T> actualHandler) {
         return actualHandler;
     }
@@ -54,8 +55,8 @@ public interface TransportInterceptor {
      * {@link #sendRequest(Transport.Connection, String, TransportRequest, TransportRequestOptions, TransportResponseHandler)}
      */
     interface AsyncSender {
-        <T extends TransportResponse> void sendRequest(Transport.Connection connection, final String action,
-                                                       final TransportRequest request, final TransportRequestOptions options,
+        <T extends TransportResponse> void sendRequest(Transport.Connection connection, String action,
+                                                       TransportRequest request, TransportRequestOptions options,
                                                        TransportResponseHandler<T> handler);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 8884177ba63..5e26d0a4b37 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -62,9 +62,10 @@ import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.settings.Setting.listSetting;
@@ -80,6 +81,7 @@ public class TransportService extends AbstractLifecycleComponent {
     protected final ClusterName clusterName;
     protected final TaskManager taskManager;
     private final TransportInterceptor.AsyncSender asyncSender;
+    private final Function<BoundTransportAddress, DiscoveryNode> localNodeFactory;
 
     volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
     final Object requestHandlerMutex = new Object();
@@ -143,10 +145,11 @@ public class TransportService extends AbstractLifecycleComponent {
      *        updates for {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
      */
     public TransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor transportInterceptor,
-            @Nullable ClusterSettings clusterSettings) {
+                            Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings) {
         super(settings);
         this.transport = transport;
         this.threadPool = threadPool;
+        this.localNodeFactory = localNodeFactory;
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
         setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
@@ -161,16 +164,7 @@ public class TransportService extends AbstractLifecycleComponent {
         }
     }
 
-    /**
-     * makes the transport service aware of the local node. this allows it to optimize requests sent
-     * from the local node to it self and by pass the network stack/ serialization
-     */
-    public void setLocalNode(DiscoveryNode localNode) {
-        this.localNode = localNode;
-    }
-
-    // for testing
-    DiscoveryNode getLocalNode() {
+    public DiscoveryNode getLocalNode() {
         return localNode;
     }
 
@@ -200,18 +194,20 @@ public class TransportService extends AbstractLifecycleComponent {
         adapter.txMetric.clear();
         transport.transportServiceAdapter(adapter);
         transport.start();
+
         if (transport.boundAddress() != null && logger.isInfoEnabled()) {
             logger.info("{}", transport.boundAddress());
             for (Map.Entry<String, BoundTransportAddress> entry : transport.profileBoundAddresses().entrySet()) {
                 logger.info("profile [{}]: {}", entry.getKey(), entry.getValue());
             }
         }
+        localNode = localNodeFactory.apply(transport.boundAddress());
         registerRequestHandler(
             HANDSHAKE_ACTION_NAME,
             () -> HandshakeRequest.INSTANCE,
             ThreadPool.Names.SAME,
             (request, channel) -> channel.sendResponse(
-                    new HandshakeResponse(localNode, clusterName, localNode != null ? localNode.getVersion() : Version.CURRENT)));
+                    new HandshakeResponse(localNode, clusterName, localNode.getVersion())));
     }
 
     @Override
@@ -294,7 +290,7 @@ public class TransportService extends AbstractLifecycleComponent {
      * Returns <code>true</code> iff the given node is already connected.
      */
     public boolean nodeConnected(DiscoveryNode node) {
-        return node.equals(localNode) || transport.nodeConnected(node);
+        return isLocalNode(node) || transport.nodeConnected(node);
     }
 
     public void connectToNode(DiscoveryNode node) throws ConnectTransportException {
@@ -308,7 +304,7 @@ public class TransportService extends AbstractLifecycleComponent {
      * @param connectionProfile the connection profile to use when connecting to this node
      */
     public void connectToNode(final DiscoveryNode node, ConnectionProfile connectionProfile) {
-        if (node.equals(localNode)) {
+        if (isLocalNode(node)) {
             return;
         }
         transport.connectToNode(node, connectionProfile);
@@ -321,39 +317,13 @@ public class TransportService extends AbstractLifecycleComponent {
      * @param profile the connection profile to use
      */
     public Transport.Connection openConnection(final DiscoveryNode node, ConnectionProfile profile) throws IOException {
-        if (node.equals(localNode)) {
+        if (isLocalNode(node)) {
             return localNodeConnection;
         } else {
             return transport.openConnection(node, profile);
         }
     }
 
-    /**
-     * Lightly connect to the specified node, returning updated node
-     * information. The handshake will fail if the cluster name on the
-     * target node mismatches the local cluster name and
-     * {@code checkClusterName} is {@code true}.
-     *
-     * @param node             the node to connect to
-     * @param handshakeTimeout handshake timeout
-     * @return the connected node
-     * @throws ConnectTransportException if the connection failed
-     * @throws IllegalStateException if the handshake failed
-     */
-    public DiscoveryNode connectToNodeAndHandshake(
-        final DiscoveryNode node,
-        final long handshakeTimeout) throws IOException {
-        if (node.equals(localNode)) {
-            return localNode;
-        }
-        DiscoveryNode handshakeNode;
-        try (Transport.Connection connection = transport.openConnection(node, ConnectionProfile.LIGHT_PROFILE)) {
-            handshakeNode = handshake(connection, handshakeTimeout);
-        }
-        connectToNode(node, ConnectionProfile.LIGHT_PROFILE);
-        return handshakeNode;
-    }
-
     /**
      * Executes a high-level handshake using the given connection
      * and returns the discovery node of the node the connection
@@ -369,6 +339,25 @@ public class TransportService extends AbstractLifecycleComponent {
     public DiscoveryNode handshake(
             final Transport.Connection connection,
             final long handshakeTimeout) throws ConnectTransportException {
+        return handshake(connection, handshakeTimeout, clusterName::equals);
+    }
+
+    /**
+     * Executes a high-level handshake using the given connection
+     * and returns the discovery node of the node the connection
+     * was established with. The handshake will fail if the cluster
+     * name on the target node doesn't match the local cluster name.
+     *
+     * @param connection       the connection to a specific node
+     * @param handshakeTimeout handshake timeout
+     * @param clusterNamePredicate cluster name validation predicate
+     * @return the connected node
+     * @throws ConnectTransportException if the connection failed
+     * @throws IllegalStateException if the handshake failed
+     */
+    public DiscoveryNode handshake(
+        final Transport.Connection connection,
+        final long handshakeTimeout, Predicate<ClusterName> clusterNamePredicate) throws ConnectTransportException {
         final HandshakeResponse response;
         final DiscoveryNode node = connection.getNode();
         try {
@@ -386,9 +375,9 @@ public class TransportService extends AbstractLifecycleComponent {
             throw new IllegalStateException("handshake failed with " + node, e);
         }
 
-        if (!Objects.equals(clusterName, response.clusterName)) {
+        if (!clusterNamePredicate.test(response.clusterName)) {
             throw new IllegalStateException("handshake failed, mismatched cluster name [" + response.clusterName + "] - " + node);
-        } else if (response.version.isCompatible((localNode != null ? localNode.getVersion() : Version.CURRENT)) == false) {
+        } else if (response.version.isCompatible(localNode.getVersion()) == false) {
             throw new IllegalStateException("handshake failed, incompatible version [" + response.version + "] - " + node);
         }
 
@@ -409,10 +398,10 @@ public class TransportService extends AbstractLifecycleComponent {
         private ClusterName clusterName;
         private Version version;
 
-        public HandshakeResponse() {
+        HandshakeResponse() {
         }
 
-        public HandshakeResponse(DiscoveryNode discoveryNode, ClusterName clusterName, Version version) {
+        HandshakeResponse(DiscoveryNode discoveryNode, ClusterName clusterName, Version version) {
             this.discoveryNode = discoveryNode;
             this.version = version;
             this.clusterName = clusterName;
@@ -436,7 +425,7 @@ public class TransportService extends AbstractLifecycleComponent {
     }
 
     public void disconnectFromNode(DiscoveryNode node) {
-        if (node.equals(localNode)) {
+        if (isLocalNode(node)) {
             return;
         }
         transport.disconnectFromNode(node);
@@ -507,27 +496,25 @@ public class TransportService extends AbstractLifecycleComponent {
      * @throws NodeNotConnectedException if the given node is not connected
      */
     public Transport.Connection getConnection(DiscoveryNode node) {
-        if (Objects.requireNonNull(node, "node must be non-null").equals(localNode)) {
+        if (isLocalNode(node)) {
             return localNodeConnection;
         } else {
             return transport.getConnection(node);
         }
     }
 
-    public <T extends TransportResponse> void sendChildRequest(final DiscoveryNode node, final String action,
+    public <T extends TransportResponse> void sendChildRequest(final Transport.Connection connection, final String action,
                                                                final TransportRequest request, final Task parentTask,
                                                                final TransportResponseHandler<T> handler) {
-        sendChildRequest(node, action, request, parentTask, TransportRequestOptions.EMPTY, handler);
+        sendChildRequest(connection, action, request, parentTask, TransportRequestOptions.EMPTY, handler);
     }
 
-    public <T extends TransportResponse> void sendChildRequest(final DiscoveryNode node, final String action,
+    public <T extends TransportResponse> void sendChildRequest(final Transport.Connection connection, final String action,
                                                                final TransportRequest request, final Task parentTask,
                                                                final TransportRequestOptions options,
                                                                final TransportResponseHandler<T> handler) {
         request.setParentTask(localNode.getId(), parentTask.getId());
         try {
-            taskManager.registerChildTask(parentTask, node.getId());
-            final Transport.Connection connection = getConnection(node);
             sendRequest(connection, action, request, options, handler);
         } catch (TaskCancelledException ex) {
             // The parent task is already cancelled - just fail the request
@@ -556,8 +543,8 @@ public class TransportService extends AbstractLifecycleComponent {
             } else {
                 timeoutHandler = new TimeoutHandler(requestId);
             }
-            TransportResponseHandler<T> responseHandler =
-                new ContextRestoreResponseHandler<>(threadPool.getThreadContext().newStoredContext(), handler);
+            Supplier<ThreadContext.StoredContext> storedContextSupplier = threadPool.getThreadContext().newRestorableContext(true);
+            TransportResponseHandler<T> responseHandler = new ContextRestoreResponseHandler<>(storedContextSupplier, handler);
             clientHandlers.put(requestId, new RequestHolder<>(responseHandler, connection.getNode(), action, timeoutHandler));
             if (lifecycle.stoppedOrClosed()) {
                 // if we are not started the exception handling will remove the RequestHolder again and calls the handler to notify
@@ -684,7 +671,7 @@ public class TransportService extends AbstractLifecycleComponent {
      */
     public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory,
                                                     String executor, TransportRequestHandler<Request> handler) {
-        handler = interceptor.interceptHandler(action, executor, handler);
+        handler = interceptor.interceptHandler(action, executor, false, handler);
         RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
             action, requestFactory, taskManager, handler, executor, false, true);
         registerRequestHandler(reg);
@@ -704,7 +691,7 @@ public class TransportService extends AbstractLifecycleComponent {
                                                                           String executor, boolean forceExecution,
                                                                           boolean canTripCircuitBreaker,
                                                                           TransportRequestHandler<Request> handler) {
-        handler = interceptor.interceptHandler(action, executor, handler);
+        handler = interceptor.interceptHandler(action, executor, forceExecution, handler);
         RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
             action, request, taskManager, handler, executor, forceExecution, canTripCircuitBreaker);
         registerRequestHandler(reg);
@@ -834,20 +821,20 @@ public class TransportService extends AbstractLifecycleComponent {
 
         @Override
         public void onNodeConnected(final DiscoveryNode node) {
-            threadPool.generic().execute(() -> {
-                for (TransportConnectionListener connectionListener : connectionListeners) {
-                    connectionListener.onNodeConnected(node);
-                }
-            });
+            // capture listeners before spawning the background callback so the following pattern won't trigger a call
+            // connectToNode(); connection is completed successfully
+            // addConnectionListener(); this listener shouldn't be called
+            final Stream<TransportConnectionListener> listenersToNotify = TransportService.this.connectionListeners.stream();
+            threadPool.generic().execute(() -> listenersToNotify.forEach(listener -> listener.onNodeConnected(node)));
         }
 
         @Override
         public void onConnectionOpened(DiscoveryNode node) {
-            threadPool.generic().execute(() -> {
-                for (TransportConnectionListener connectionListener : connectionListeners) {
-                    connectionListener.onConnectionOpened(node);
-                }
-            });
+            // capture listeners before spawning the background callback so the following pattern won't trigger a call
+            // connectToNode(); connection is completed successfully
+            // addConnectionListener(); this listener shouldn't be called
+            final Stream<TransportConnectionListener> listenersToNotify = TransportService.this.connectionListeners.stream();
+            threadPool.generic().execute(() -> listenersToNotify.forEach(listener -> listener.onConnectionOpened(node)));
         }
 
         @Override
@@ -1013,14 +1000,14 @@ public class TransportService extends AbstractLifecycleComponent {
      * This handler wrapper ensures that the response thread executes with the correct thread context. Before any of the4 handle methods
      * are invoked we restore the context.
      */
-    private static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+    public static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
 
         private final TransportResponseHandler<T> delegate;
-        private final ThreadContext.StoredContext threadContext;
+        private final Supplier<ThreadContext.StoredContext> contextSupplier;
 
-        private ContextRestoreResponseHandler(ThreadContext.StoredContext threadContext, TransportResponseHandler<T> delegate) {
+        public ContextRestoreResponseHandler(Supplier<ThreadContext.StoredContext> contextSupplier, TransportResponseHandler<T> delegate) {
             this.delegate = delegate;
-            this.threadContext = threadContext;
+            this.contextSupplier = contextSupplier;
         }
 
         @Override
@@ -1030,14 +1017,16 @@ public class TransportService extends AbstractLifecycleComponent {
 
         @Override
         public void handleResponse(T response) {
-            threadContext.restore();
-            delegate.handleResponse(response);
+            try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+                delegate.handleResponse(response);
+            }
         }
 
         @Override
         public void handleException(TransportException exp) {
-            threadContext.restore();
-            delegate.handleException(exp);
+            try (ThreadContext.StoredContext ignore = contextSupplier.get()) {
+                delegate.handleException(exp);
+            }
         }
 
         @Override
@@ -1060,7 +1049,7 @@ public class TransportService extends AbstractLifecycleComponent {
         final TransportServiceAdapter adapter;
         final ThreadPool threadPool;
 
-        public DirectResponseChannel(Logger logger, DiscoveryNode localNode, String action, long requestId,
+        DirectResponseChannel(Logger logger, DiscoveryNode localNode, String action, long requestId,
                                      TransportServiceAdapter adapter, ThreadPool threadPool) {
             this.logger = logger;
             this.localNode = localNode;
@@ -1094,13 +1083,7 @@ public class TransportService extends AbstractLifecycleComponent {
                 if (ThreadPool.Names.SAME.equals(executor)) {
                     processResponse(handler, response);
                 } else {
-                    threadPool.executor(executor).execute(new Runnable() {
-                        @SuppressWarnings({"unchecked"})
-                        @Override
-                        public void run() {
-                            processResponse(handler, response);
-                        }
-                    });
+                    threadPool.executor(executor).execute(() -> processResponse(handler, response));
                 }
             }
         }
@@ -1162,4 +1145,15 @@ public class TransportService extends AbstractLifecycleComponent {
             return "direct";
         }
     }
+
+    /**
+     * Returns the internal thread pool
+     */
+    public ThreadPool getThreadPool() {
+        return threadPool;
+    }
+
+    private boolean isLocalNode(DiscoveryNode discoveryNode) {
+        return Objects.requireNonNull(discoveryNode, "discovery node must not be null").equals(localNode);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/Transports.java b/core/src/main/java/org/elasticsearch/transport/Transports.java
index c187e3baf23..c7a0fe4d4f5 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transports.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transports.java
@@ -37,11 +37,8 @@ public enum Transports {
     public static final boolean isTransportThread(Thread t) {
         final String threadName = t.getName();
         for (String s : Arrays.asList(
-                HttpServerTransport.HTTP_SERVER_BOSS_THREAD_NAME_PREFIX,
                 HttpServerTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX,
-                TcpTransport.TRANSPORT_SERVER_BOSS_THREAD_NAME_PREFIX,
                 TcpTransport.TRANSPORT_SERVER_WORKER_THREAD_NAME_PREFIX,
-                TcpTransport.TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX,
                 TcpTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX,
                 TEST_MOCK_TRANSPORT_THREAD_PREFIX)) {
             if (threadName.contains(s)) {
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index d976f9229b8..fb9d7babf20 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -44,6 +44,10 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.hash.MurmurHash3;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
@@ -61,6 +65,7 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TransportSettings;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -207,10 +212,13 @@ public class TribeService extends AbstractLifecycleComponent {
 
     private final List<Node> nodes = new CopyOnWriteArrayList<>();
 
+    private final NamedWriteableRegistry namedWriteableRegistry;
+
     public TribeService(Settings settings, ClusterService clusterService, final String tribeNodeId,
-                        Function<Settings, Node> clientNodeBuilder) {
+                        NamedWriteableRegistry namedWriteableRegistry, Function<Settings, Node> clientNodeBuilder) {
         super(settings);
         this.clusterService = clusterService;
+        this.namedWriteableRegistry = namedWriteableRegistry;
         Map<String, Settings> nodesSettings = new HashMap<>(settings.getGroups("tribe", true));
         nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client
         nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client
@@ -359,19 +367,19 @@ public class TribeService extends AbstractLifecycleComponent {
             this.tribeName = tribeName;
         }
 
-        @Override
-        public boolean runOnlyOnMaster() {
-            return false;
-        }
-
         @Override
         public String describeTasks(List<ClusterChangedEvent> tasks) {
             return tasks.stream().map(ClusterChangedEvent::source).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
         }
 
         @Override
-        public BatchResult<ClusterChangedEvent> execute(ClusterState currentState, List<ClusterChangedEvent> tasks) throws Exception {
-            BatchResult.Builder<ClusterChangedEvent> builder = BatchResult.builder();
+        public boolean runOnlyOnMaster() {
+            return false;
+        }
+
+        @Override
+        public ClusterTasksResult<ClusterChangedEvent> execute(ClusterState currentState, List<ClusterChangedEvent> tasks) throws Exception {
+            ClusterTasksResult.Builder<ClusterChangedEvent> builder = ClusterTasksResult.builder();
             ClusterState.Builder newState = ClusterState.builder(currentState).incrementVersion();
             boolean clusterStateChanged = updateNodes(currentState, tasks, newState);
             clusterStateChanged |= updateIndicesAndMetaData(currentState, tasks, newState);
@@ -512,7 +520,7 @@ public class TribeService extends AbstractLifecycleComponent {
                             .map(ClusterState::metaData)
                             .map(clusterMetaData -> ((MetaData.Custom) clusterMetaData.custom(customMetaDataType)))
                             .filter(custom1 -> custom1 != null && custom1 instanceof MergableCustomMetaData)
-                            .map(custom2 -> (MergableCustomMetaData) custom2)
+                            .map(custom2 -> (MergableCustomMetaData) marshal(custom2))
                             .collect(Collectors.toList())
             );
             for (String changedCustomMetaDataType : changedCustomMetaDataTypeSet) {
@@ -577,4 +585,21 @@ public class TribeService extends AbstractLifecycleComponent {
         }
         return changedCustomMetaDataMap;
     }
+
+    /**
+     * Since custom metadata can be loaded by a plugin class loader that resides in a sub-node, we need to
+     * marshal this object into something the tribe node can work with
+     */
+    private MetaData.Custom marshal(MetaData.Custom custom)  {
+        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()){
+            bytesStreamOutput.writeNamedWriteable(custom);
+            try(StreamInput input = bytesStreamOutput.bytes().streamInput()) {
+                StreamInput namedInput = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
+                MetaData.Custom marshaled = namedInput.readNamedWriteable(MetaData.Custom.class);
+                return marshaled;
+            }
+        } catch (IOException ex) {
+            throw new IllegalStateException("cannot marshal object with type " + custom.getWriteableName() + " to tribe node");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
index 8d5c04e7708..0b0504f4609 100644
--- a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
+++ b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
@@ -80,7 +80,7 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
         private boolean isDirectory;
         private FileObserver[] children;
 
-        public FileObserver(Path file) {
+        FileObserver(Path file) {
             this.file = file;
         }
 
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 623f883f492..e93b1af8cc7 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.1.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.4.0-snapshot-ec38570.jar}" {
+grant codeBase "${codebase.lucene-core-6.4.0.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.4.0-snapshot-ec38570.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-6.4.0-snapshot-ec38570.jar}" {
+grant codeBase "${codebase.lucene-misc-6.4.0.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };
@@ -55,8 +55,8 @@ grant {
   // third party code, to safeguard these against unprivileged code like scripts.
   permission org.elasticsearch.SpecialPermission;
 
-  // Allow connecting to the internet anywhere
-  permission java.net.SocketPermission "*", "accept,connect,resolve";
+  // Allow host/ip name service lookups
+  permission java.net.SocketPermission "*", "resolve";
 
   // Allow read access to all system properties
   permission java.util.PropertyPermission "*", "read";
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 6f44c37d233..ad43609d1c8 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.4.0-snapshot-ec38570.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.4.0.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS
@@ -62,3 +62,24 @@ grant codeBase "${codebase.junit-4.11.jar}" {
   // needed for TestClass creation
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
+
+grant codeBase "${codebase.mocksocket-1.1.jar}" {
+  // mocksocket makes and accepts socket connections
+  permission java.net.SocketPermission "*", "accept,connect";
+};
+
+
+grant codeBase "${codebase.rest-6.0.0-alpha1-SNAPSHOT.jar}" {
+  // rest makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+};
+
+grant codeBase "${codebase.httpcore-nio-4.4.5.jar}" {
+  // httpcore makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+};
+
+grant codeBase "${codebase.httpasyncclient-4.1.2.jar}" {
+  // httpasyncclient makes socket connections for rest tests
+  permission java.net.SocketPermission "*", "connect";
+};
diff --git a/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
new file mode 100644
index 00000000000..1aff2af5235
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/grouping/CollapsingTopDocsCollectorTests.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lucene.grouping;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.CompositeReaderContext;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
+import org.apache.lucene.search.TopFieldCollector;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.grouping.CollapseTopFieldDocs;
+import org.apache.lucene.search.grouping.CollapsingTopDocsCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class CollapsingTopDocsCollectorTests extends ESTestCase {
+    private static class SegmentSearcher extends IndexSearcher {
+        private final List<LeafReaderContext> ctx;
+
+        SegmentSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
+            super(parent);
+            this.ctx = Collections.singletonList(ctx);
+        }
+
+        public void search(Weight weight, Collector collector) throws IOException {
+            search(ctx, weight, collector);
+        }
+
+        @Override
+        public String toString() {
+            return "ShardSearcher(" + ctx.get(0) + ")";
+        }
+    }
+
+    interface CollapsingDocValuesProducer<T extends Comparable> {
+        T randomGroup(int maxGroup);
+
+        void add(Document doc, T value, boolean multivalued);
+
+        SortField sortField(boolean multivalued);
+    }
+
+    <T extends Comparable> void assertSearchCollapse(CollapsingDocValuesProducer<T> dvProducers, boolean numeric) throws IOException {
+        assertSearchCollapse(dvProducers, numeric, true);
+        assertSearchCollapse(dvProducers, numeric, false);
+    }
+
+    private <T extends Comparable> void assertSearchCollapse(CollapsingDocValuesProducer<T> dvProducers,
+                                                             boolean numeric, boolean multivalued) throws IOException {
+        final int numDocs = randomIntBetween(1000, 2000);
+        int maxGroup = randomIntBetween(2, 500);
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Set<T> values = new HashSet<>();
+        int totalHits = 0;
+        for (int i = 0; i < numDocs; i++) {
+            final T value = dvProducers.randomGroup(maxGroup);
+            values.add(value);
+            Document doc = new Document();
+            dvProducers.add(doc, value, multivalued);
+            doc.add(new NumericDocValuesField("sort1", randomIntBetween(0, 10)));
+            doc.add(new NumericDocValuesField("sort2", randomLong()));
+            w.addDocument(doc);
+            totalHits++;
+        }
+        List<T> valueList = new ArrayList<>(values);
+        Collections.sort(valueList);
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        final SortField collapseField = dvProducers.sortField(multivalued);
+        final SortField sort1 = new SortField("sort1", SortField.Type.INT);
+        final SortField sort2 = new SortField("sort2", SortField.Type.LONG);
+        Sort sort = new Sort(sort1, sort2, collapseField);
+
+        int expectedNumGroups = values.size();
+
+        final CollapsingTopDocsCollector collapsingCollector;
+        if (numeric) {
+            collapsingCollector =
+                CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false);
+        } else {
+            collapsingCollector =
+                CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false);
+        }
+
+        TopFieldCollector topFieldCollector =
+            TopFieldCollector.create(sort, totalHits, true, false, false);
+
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        searcher.search(new MatchAllDocsQuery(), topFieldCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        TopFieldDocs topDocs = topFieldCollector.topDocs();
+        assertEquals(collapseField.getField(), collapseTopFieldDocs.field);
+        assertEquals(expectedNumGroups, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(totalHits, collapseTopFieldDocs.totalHits);
+        assertEquals(totalHits, topDocs.scoreDocs.length);
+        assertEquals(totalHits, topDocs.totalHits);
+
+        Set<Object> seen = new HashSet<>();
+        // collapse field is the last sort
+        int collapseIndex = sort.getSort().length - 1;
+        int topDocsIndex = 0;
+        for (int i = 0; i < expectedNumGroups; i++) {
+            FieldDoc fieldDoc = null;
+            for (; topDocsIndex < totalHits; topDocsIndex++) {
+                fieldDoc = (FieldDoc) topDocs.scoreDocs[topDocsIndex];
+                if (seen.contains(fieldDoc.fields[collapseIndex]) == false) {
+                    break;
+                }
+            }
+            FieldDoc collapseFieldDoc = (FieldDoc) collapseTopFieldDocs.scoreDocs[i];
+            assertNotNull(fieldDoc);
+            assertEquals(collapseFieldDoc.doc, fieldDoc.doc);
+            assertArrayEquals(collapseFieldDoc.fields, fieldDoc.fields);
+            seen.add(fieldDoc.fields[fieldDoc.fields.length - 1]);
+        }
+        for (; topDocsIndex < totalHits; topDocsIndex++) {
+            FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[topDocsIndex];
+            assertTrue(seen.contains(fieldDoc.fields[collapseIndex]));
+        }
+
+
+        // check merge
+        final IndexReaderContext ctx = searcher.getTopReaderContext();
+        final SegmentSearcher[] subSearchers;
+        final int[] docStarts;
+
+        if (ctx instanceof LeafReaderContext) {
+            subSearchers = new SegmentSearcher[1];
+            docStarts = new int[1];
+            subSearchers[0] = new SegmentSearcher((LeafReaderContext) ctx, ctx);
+            docStarts[0] = 0;
+        } else {
+            final CompositeReaderContext compCTX = (CompositeReaderContext) ctx;
+            final int size = compCTX.leaves().size();
+            subSearchers = new SegmentSearcher[size];
+            docStarts = new int[size];
+            int docBase = 0;
+            for (int searcherIDX = 0; searcherIDX < subSearchers.length; searcherIDX++) {
+                final LeafReaderContext leave = compCTX.leaves().get(searcherIDX);
+                subSearchers[searcherIDX] = new SegmentSearcher(leave, compCTX);
+                docStarts[searcherIDX] = docBase;
+                docBase += leave.reader().maxDoc();
+            }
+        }
+
+        final CollapseTopFieldDocs[] shardHits = new CollapseTopFieldDocs[subSearchers.length];
+        final Weight weight = searcher.createNormalizedWeight(new MatchAllDocsQuery(), false);
+        for (int shardIDX = 0; shardIDX < subSearchers.length; shardIDX++) {
+            final SegmentSearcher subSearcher = subSearchers[shardIDX];
+            final CollapsingTopDocsCollector c;
+            if (numeric) {
+                c = CollapsingTopDocsCollector.createNumeric(collapseField.getField(), sort, expectedNumGroups, false);
+            } else {
+                c = CollapsingTopDocsCollector.createKeyword(collapseField.getField(), sort, expectedNumGroups, false);
+            }
+            subSearcher.search(weight, c);
+            shardHits[shardIDX] = c.getTopDocs();
+        }
+        CollapseTopFieldDocs mergedFieldDocs = CollapseTopFieldDocs.merge(sort, 0, expectedNumGroups, shardHits);
+        assertTopDocsEquals(mergedFieldDocs, collapseTopFieldDocs);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+
+    private static void assertTopDocsEquals(CollapseTopFieldDocs topDocs1, CollapseTopFieldDocs topDocs2) {
+        TestUtil.assertEquals(topDocs1, topDocs2);
+        assertArrayEquals(topDocs1.collapseValues, topDocs2.collapseValues);
+    }
+
+    public void testCollapseLong() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Long>() {
+            @Override
+            public Long randomGroup(int maxGroup) {
+                return randomNonNegativeLong() % maxGroup;
+            }
+
+            @Override
+            public void add(Document doc, Long value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", value));
+                } else {
+                    doc.add(new NumericDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.LONG);
+                } else {
+                    return new SortField("field", SortField.Type.LONG);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseInt() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Integer>() {
+            @Override
+            public Integer randomGroup(int maxGroup) {
+                return randomIntBetween(0, maxGroup - 1);
+            }
+
+            @Override
+            public void add(Document doc, Integer value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", value));
+                } else {
+                    doc.add(new NumericDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.INT);
+                } else {
+                    return new SortField("field", SortField.Type.INT);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseFloat() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Float>() {
+            @Override
+            public Float randomGroup(int maxGroup) {
+                return new Float(randomIntBetween(0, maxGroup - 1));
+            }
+
+            @Override
+            public void add(Document doc, Float value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", NumericUtils.floatToSortableInt(value)));
+                } else {
+                    doc.add(new NumericDocValuesField("field", Float.floatToIntBits(value)));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.FLOAT);
+                } else {
+                    return new SortField("field", SortField.Type.FLOAT);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseDouble() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<Double>() {
+            @Override
+            public Double randomGroup(int maxGroup) {
+                return new Double(randomIntBetween(0, maxGroup - 1));
+            }
+
+            @Override
+            public void add(Document doc, Double value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value)));
+                } else {
+                    doc.add(new NumericDocValuesField("field", Double.doubleToLongBits(value)));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedNumericSortField("field", SortField.Type.DOUBLE);
+                } else {
+                    return new SortField("field", SortField.Type.DOUBLE);
+                }
+            }
+        };
+        assertSearchCollapse(producer, true);
+    }
+
+    public void testCollapseString() throws Exception {
+        CollapsingDocValuesProducer producer = new CollapsingDocValuesProducer<BytesRef>() {
+            @Override
+            public BytesRef randomGroup(int maxGroup) {
+                return new BytesRef(Integer.toString(randomIntBetween(0, maxGroup - 1)));
+            }
+
+            @Override
+            public void add(Document doc, BytesRef value, boolean multivalued) {
+                if (multivalued) {
+                    doc.add(new SortedSetDocValuesField("field", value));
+                } else {
+                    doc.add(new SortedDocValuesField("field", value));
+                }
+            }
+
+            @Override
+            public SortField sortField(boolean multivalued) {
+                if (multivalued) {
+                    return new SortedSetSortField("field", false);
+                } else {
+                    return new SortField("field", SortField.Type.STRING_VAL);
+                }
+            }
+        };
+        assertSearchCollapse(producer, false);
+    }
+
+    public void testEmptyNumericSegment() throws Exception {
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Document doc = new Document();
+        doc.add(new NumericDocValuesField("group", 0));
+        w.addDocument(doc);
+        doc.clear();
+        doc.add(new NumericDocValuesField("group", 1));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("group", 10));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("category", 0));
+        w.addDocument(doc);
+        w.commit();
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        SortField sortField = new SortField("group", SortField.Type.LONG);
+        sortField.setMissingValue(Long.MAX_VALUE);
+        Sort sort = new Sort(sortField);
+        final CollapsingTopDocsCollector collapsingCollector =
+                CollapsingTopDocsCollector.createNumeric("group", sort, 10, false);
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        assertEquals(4, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(4, collapseTopFieldDocs.collapseValues.length);
+        assertEquals(0L, collapseTopFieldDocs.collapseValues[0]);
+        assertEquals(1L, collapseTopFieldDocs.collapseValues[1]);
+        assertEquals(10L, collapseTopFieldDocs.collapseValues[2]);
+        assertNull(collapseTopFieldDocs.collapseValues[3]);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+
+    public void testEmptySortedSegment() throws Exception {
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        Document doc = new Document();
+        doc.add(new SortedDocValuesField("group", new BytesRef("0")));
+        w.addDocument(doc);
+        doc.clear();
+        doc.add(new SortedDocValuesField("group", new BytesRef("1")));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new SortedDocValuesField("group", new BytesRef("10")));
+        w.addDocument(doc);
+        w.commit();
+        doc.clear();
+        doc.add(new NumericDocValuesField("category", 0));
+        w.addDocument(doc);
+        w.commit();
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+        Sort sort = new Sort(new SortField("group", SortField.Type.STRING_VAL));
+        final CollapsingTopDocsCollector collapsingCollector =
+            CollapsingTopDocsCollector.createKeyword("group", sort, 10, false);
+        searcher.search(new MatchAllDocsQuery(), collapsingCollector);
+        CollapseTopFieldDocs collapseTopFieldDocs = collapsingCollector.getTopDocs();
+        assertEquals(4, collapseTopFieldDocs.scoreDocs.length);
+        assertEquals(4, collapseTopFieldDocs.collapseValues.length);
+        assertNull(collapseTopFieldDocs.collapseValues[0]);
+        assertEquals(new BytesRef("0"), collapseTopFieldDocs.collapseValues[1]);
+        assertEquals(new BytesRef("1"), collapseTopFieldDocs.collapseValues[2]);
+        assertEquals(new BytesRef("10"), collapseTopFieldDocs.collapseValues[3]);
+        w.close();
+        reader.close();
+        dir.close();
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
index fcddc58f77a..2d43a1ca64e 100644
--- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
+++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPassageFormatterTests.java
@@ -19,6 +19,7 @@
 
 package org.apache.lucene.search.postingshighlight;
 
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.highlight.DefaultEncoder;
 import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
 import org.apache.lucene.util.BytesRef;
diff --git a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
index c95819a0e53..315e38d12fe 100644
--- a/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
+++ b/core/src/test/java/org/apache/lucene/search/postingshighlight/CustomPostingsHighlighterTests.java
@@ -31,6 +31,7 @@ import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.highlight.Snippet;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
diff --git a/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java
new file mode 100644
index 00000000000..4e664c3e241
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomPassageFormatterTests.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.search.highlight.DefaultEncoder;
+import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+
+
+public class CustomPassageFormatterTests extends ESTestCase {
+    public void testSimpleFormat() {
+        String content = "This is a really cool highlighter. Unified highlighter gives nice snippets back. No matches here.";
+
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new DefaultEncoder());
+
+        Passage[] passages = new Passage[3];
+        String match = "highlighter";
+        BytesRef matchBytesRef = new BytesRef(match);
+
+        Passage passage1 = new Passage();
+        int start = content.indexOf(match);
+        int end = start + match.length();
+        passage1.setStartOffset(0);
+        passage1.setEndOffset(end + 2); //lets include the whitespace at the end to make sure we trim it
+        passage1.addMatch(start, end, matchBytesRef);
+        passages[0] = passage1;
+
+        Passage passage2 = new Passage();
+        start = content.lastIndexOf(match);
+        end = start + match.length();
+        passage2.setStartOffset(passage1.getEndOffset());
+        passage2.setEndOffset(end + 26);
+        passage2.addMatch(start, end, matchBytesRef);
+        passages[1] = passage2;
+
+        Passage passage3 = new Passage();
+        passage3.setStartOffset(passage2.getEndOffset());
+        passage3.setEndOffset(content.length());
+        passages[2] = passage3;
+
+        Snippet[] fragments = passageFormatter.format(passages, content);
+        assertThat(fragments, notNullValue());
+        assertThat(fragments.length, equalTo(3));
+        assertThat(fragments[0].getText(), equalTo("This is a really cool <em>highlighter</em>."));
+        assertThat(fragments[0].isHighlighted(), equalTo(true));
+        assertThat(fragments[1].getText(), equalTo("Unified <em>highlighter</em> gives nice snippets back."));
+        assertThat(fragments[1].isHighlighted(), equalTo(true));
+        assertThat(fragments[2].getText(), equalTo("No matches here."));
+        assertThat(fragments[2].isHighlighted(), equalTo(false));
+    }
+
+    public void testHtmlEncodeFormat() {
+        String content = "<b>This is a really cool highlighter.</b> Unified highlighter gives nice snippets back.";
+
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<em>", "</em>", new SimpleHTMLEncoder());
+
+        Passage[] passages = new Passage[2];
+        String match = "highlighter";
+        BytesRef matchBytesRef = new BytesRef(match);
+
+        Passage passage1 = new Passage();
+        int start = content.indexOf(match);
+        int end = start + match.length();
+        passage1.setStartOffset(0);
+        passage1.setEndOffset(end + 6); //lets include the whitespace at the end to make sure we trim it
+        passage1.addMatch(start, end, matchBytesRef);
+        passages[0] = passage1;
+
+        Passage passage2 = new Passage();
+        start = content.lastIndexOf(match);
+        end = start + match.length();
+        passage2.setStartOffset(passage1.getEndOffset());
+        passage2.setEndOffset(content.length());
+        passage2.addMatch(start, end, matchBytesRef);
+        passages[1] = passage2;
+
+        Snippet[] fragments = passageFormatter.format(passages, content);
+        assertThat(fragments, notNullValue());
+        assertThat(fragments.length, equalTo(2));
+        assertThat(fragments[0].getText(), equalTo("&lt;b&gt;This is a really cool <em>highlighter</em>.&lt;&#x2F;b&gt;"));
+        assertThat(fragments[1].getText(), equalTo("Unified <em>highlighter</em> gives nice snippets back."));
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
new file mode 100644
index 00000000000..83b42750f92
--- /dev/null
+++ b/core/src/test/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.CommonTermsQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.highlight.DefaultEncoder;
+import org.apache.lucene.search.highlight.Snippet;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.lucene.all.AllTermQuery;
+import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class CustomUnifiedHighlighterTests extends ESTestCase {
+    public void testCustomUnifiedHighlighter() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
+        iwc.setMergePolicy(newLogMergePolicy());
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+        offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+        offsetsType.setStoreTermVectorOffsets(true);
+        offsetsType.setStoreTermVectorPositions(true);
+        offsetsType.setStoreTermVectors(true);
+
+        //good position but only one match
+        final String firstValue = "This is a test. Just a test1 highlighting from unified highlighter.";
+        Field body = new Field("body", "", offsetsType);
+        Document doc = new Document();
+        doc.add(body);
+        body.setStringValue(firstValue);
+
+        //two matches, not the best snippet due to its length though
+        final String secondValue = "This is the second highlighting value to perform highlighting on a longer text " +
+            "that gets scored lower.";
+        Field body2 = new Field("body", "", offsetsType);
+        doc.add(body2);
+        body2.setStringValue(secondValue);
+
+        //two matches and short, will be scored highest
+        final String thirdValue = "This is highlighting the third short highlighting value.";
+        Field body3 = new Field("body", "", offsetsType);
+        doc.add(body3);
+        body3.setStringValue(thirdValue);
+
+        //one match, same as first but at the end, will be scored lower due to its position
+        final String fourthValue = "Just a test4 highlighting from unified highlighter.";
+        Field body4 = new Field("body", "", offsetsType);
+        doc.add(body4);
+        body4.setStringValue(fourthValue);
+
+        iw.addDocument(doc);
+
+        IndexReader ir = iw.getReader();
+        iw.close();
+
+        String firstHlValue = "Just a test1 <b>highlighting</b> from unified highlighter.";
+        String secondHlValue = "This is the second <b>highlighting</b> value to perform <b>highlighting</b> on a" +
+            " longer text that gets scored lower.";
+        String thirdHlValue = "This is <b>highlighting</b> the third short <b>highlighting</b> value.";
+        String fourthHlValue = "Just a test4 <b>highlighting</b> from unified highlighter.";
+
+        IndexSearcher searcher = newSearcher(ir);
+        Query query = new TermQuery(new Term("body", "highlighting"));
+
+        TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+
+        int docId = topDocs.scoreDocs[0].doc;
+
+        String fieldValue = firstValue + HighlightUtils.PARAGRAPH_SEPARATOR + secondValue +
+            HighlightUtils.PARAGRAPH_SEPARATOR + thirdValue + HighlightUtils.PARAGRAPH_SEPARATOR + fourthValue;
+
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, iwc.getAnalyzer(),
+            new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder()), null, fieldValue, true);
+        Snippet[] snippets = highlighter.highlightField("body", query, docId, 5);
+
+        assertThat(snippets.length, equalTo(4));
+
+        assertThat(snippets[0].getText(), equalTo(firstHlValue));
+        assertThat(snippets[1].getText(), equalTo(secondHlValue));
+        assertThat(snippets[2].getText(), equalTo(thirdHlValue));
+        assertThat(snippets[3].getText(), equalTo(fourthHlValue));
+        ir.close();
+        dir.close();
+    }
+
+    public void testNoMatchSize() throws Exception {
+        Directory dir = newDirectory();
+        Analyzer analyzer = new StandardAnalyzer();
+        IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
+        iwc.setMergePolicy(newLogMergePolicy());
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        FieldType offsetsType = new FieldType(TextField.TYPE_STORED);
+        offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+        offsetsType.setStoreTermVectorOffsets(true);
+        offsetsType.setStoreTermVectorPositions(true);
+        offsetsType.setStoreTermVectors(true);
+        Field body = new Field("body", "", offsetsType);
+        Field none = new Field("none", "", offsetsType);
+        Document doc = new Document();
+        doc.add(body);
+        doc.add(none);
+
+        String firstValue = "This is a test. Just a test highlighting from unified. Feel free to ignore.";
+        body.setStringValue(firstValue);
+        none.setStringValue(firstValue);
+        iw.addDocument(doc);
+
+        IndexReader ir = iw.getReader();
+        iw.close();
+
+        Query query = new TermQuery(new Term("none", "highlighting"));
+
+        IndexSearcher searcher = newSearcher(ir);
+        TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+        int docId = topDocs.scoreDocs[0].doc;
+
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer, passageFormatter,
+            null, firstValue, false);
+        Snippet[] snippets = highlighter.highlightField("body", query, docId, 5);
+        assertThat(snippets.length, equalTo(0));
+
+        highlighter = new CustomUnifiedHighlighter(searcher, analyzer, passageFormatter, null, firstValue, true);
+        snippets = highlighter.highlightField("body", query, docId, 5);
+        assertThat(snippets.length, equalTo(1));
+        assertThat(snippets[0].getText(), equalTo("This is a test."));
+        ir.close();
+        dir.close();
+    }
+
+
+    private IndexReader indexOneDoc(Directory dir, String field, String value, Analyzer analyzer) throws IOException {
+        IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
+        iwc.setMergePolicy(newLogMergePolicy());
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+
+        FieldType ft = new FieldType(TextField.TYPE_STORED);
+        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+        Field textField = new Field(field, "", ft);
+        Document doc = new Document();
+        doc.add(textField);
+
+        textField.setStringValue(value);
+        iw.addDocument(doc);
+        IndexReader ir = iw.getReader();
+        iw.close();
+        return ir;
+    }
+
+    public void testMultiPhrasePrefixQuery() throws Exception {
+        Analyzer analyzer = new StandardAnalyzer();
+        Directory dir = newDirectory();
+        String value = "The quick brown fox.";
+        IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
+        MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery();
+        query.add(new Term("text", "quick"));
+        query.add(new Term("text", "brown"));
+        query.add(new Term("text", "fo"));
+        IndexSearcher searcher = newSearcher(ir);
+        TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+        int docId = topDocs.scoreDocs[0].doc;
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
+            passageFormatter, null, value, false);
+        Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
+        assertThat(snippets.length, equalTo(1));
+        assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
+        ir.close();
+        dir.close();
+    }
+
+    public void testAllTermQuery() throws IOException {
+        Directory dir = newDirectory();
+        String value = "The quick brown fox.";
+        Analyzer analyzer = new StandardAnalyzer();
+        IndexReader ir = indexOneDoc(dir, "all", value, analyzer);
+        AllTermQuery query = new AllTermQuery(new Term("all", "fox"));
+        IndexSearcher searcher = newSearcher(ir);
+        TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+        int docId = topDocs.scoreDocs[0].doc;
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
+            passageFormatter, null, value, false);
+        Snippet[] snippets = highlighter.highlightField("all", query, docId, 5);
+        assertThat(snippets.length, equalTo(1));
+        assertThat(snippets[0].getText(), equalTo("The quick brown <b>fox</b>."));
+        ir.close();
+        dir.close();
+    }
+
+    public void testCommonTermsQuery() throws IOException {
+        Directory dir = newDirectory();
+        String value = "The quick brown fox.";
+        Analyzer analyzer = new StandardAnalyzer();
+        IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
+        CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
+        query.add(new Term("text", "quick"));
+        query.add(new Term("text", "brown"));
+        query.add(new Term("text", "fox"));
+        IndexSearcher searcher = newSearcher(ir);
+        TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
+        assertThat(topDocs.totalHits, equalTo(1));
+        int docId = topDocs.scoreDocs[0].doc;
+        CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
+        CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
+            passageFormatter, null, value, false);
+        Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
+        assertThat(snippets.length, equalTo(1));
+        assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
+        ir.close();
+        dir.close();
+    }
+}
diff --git a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
deleted file mode 100644
index 502d11e3b44..00000000000
--- a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lucene.util;
-
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.number.IsCloseTo.closeTo;
-
-public class SloppyMathTests extends ESTestCase {
-    public void testAccuracy() {
-        for (double lat1 = -89; lat1 <= 89; lat1+=1) {
-            final double lon1 = randomLongitude();
-
-            for (double i = -180; i <= 180; i+=1) {
-                final double lon2 = i;
-                final double lat2 = randomLatitude();
-
-                assertAccurate(lat1, lon1, lat2, lon2);
-            }
-        }
-    }
-
-    public void testSloppyMath() {
-        testSloppyMath(DistanceUnit.METERS, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.KILOMETERS, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.INCH, 0.01, 5, 45, 90);
-        testSloppyMath(DistanceUnit.MILES, 0.01, 5, 45, 90);
-    }
-
-    private static double maxError(double distance) {
-        return distance / 1000.0;
-    }
-
-    private void testSloppyMath(DistanceUnit unit, double...deltaDeg) {
-        final double lat1 = randomLatitude();
-        final double lon1 = randomLongitude();
-        logger.info("testing SloppyMath with {} at \"{}, {}\"", unit, lat1, lon1);
-
-        for (int test = 0; test < deltaDeg.length; test++) {
-            for (int i = 0; i < 100; i++) {
-                // crop pole areas, sine we now there the function
-                // is not accurate around lat(89°, 90°) and lat(-90°, -89°)
-                final double lat2 = Math.max(-89.0, Math.min(+89.0, lat1 + (random().nextDouble() - 0.5) * 2 * deltaDeg[test]));
-                final double lon2 = lon1 + (random().nextDouble() - 0.5) * 2 * deltaDeg[test];
-
-                final double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, unit);
-                final double dist = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, unit);
-
-                assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", dist, closeTo(accurate, maxError(accurate)));
-            }
-        }
-    }
-
-    private static void assertAccurate(double lat1, double lon1, double lat2, double lon2) {
-        double accurate = GeoDistance.ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS);
-        double sloppy = GeoDistance.SLOPPY_ARC.calculate(lat1, lon1, lat2, lon2, DistanceUnit.METERS);
-        assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", sloppy, closeTo(accurate, maxError(accurate)));
-    }
-
-    private static double randomLatitude() {
-        // crop pole areas, sine we now there the function
-        // is not accurate around lat(89°, 90°) and lat(-90°, -89°)
-        return (random().nextDouble() - 0.5) * 178.0;
-    }
-
-    private static double randomLongitude() {
-        return (random().nextDouble() - 0.5) * 360.0;
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
deleted file mode 100644
index b59c5625939..00000000000
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.TestSearchContext;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.hamcrest.Matchers;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.file.NoSuchFileException;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class ESExceptionTests extends ESTestCase {
-    private static final ToXContent.Params PARAMS = ToXContent.EMPTY_PARAMS;
-
-    private class UnknownException extends Exception {
-
-        UnknownException(final String message, final Exception cause) {
-            super(message, cause);
-        }
-
-    }
-
-    public void testStatus() {
-        ElasticsearchException exception = new ElasticsearchException("test");
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new ElasticsearchException("test", new RuntimeException());
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new ElasticsearchException("test", new ResourceNotFoundException("test"));
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-
-        exception = new RemoteTransportException("test", new ResourceNotFoundException("test"));
-        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
-
-        exception = new RemoteTransportException("test", new ResourceAlreadyExistsException("test"));
-        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
-
-        exception = new RemoteTransportException("test", new IllegalArgumentException("foobar"));
-        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
-
-        exception = new RemoteTransportException("test", new IllegalStateException("foobar"));
-        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
-    }
-
-    public void testGuessRootCause() {
-        {
-            ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IndexNotFoundException("foo", new RuntimeException("foobar"))));
-            ElasticsearchException[] rootCauses = exception.guessRootCauses();
-            assertEquals(rootCauses.length, 1);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception");
-            assertEquals(rootCauses[0].getMessage(), "no such index");
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1});
-            if (randomBoolean()) {
-                rootCauses = (randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex).guessRootCauses();
-            } else {
-                rootCauses = ElasticsearchException.guessRootCauses(randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex);
-            }
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
-            assertEquals(rootCauses[0].getMessage(), "foobar");
-
-            ElasticsearchException oneLevel = new ElasticsearchException("foo", new RuntimeException("foobar"));
-            rootCauses = oneLevel.guessRootCauses();
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "exception");
-            assertEquals(rootCauses[0].getMessage(), "foo");
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(
-                    new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
-            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2});
-            final ElasticsearchException[] rootCauses = ex.guessRootCauses();
-            assertEquals(rootCauses.length, 2);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
-            assertEquals(rootCauses[0].getMessage(), "foobar");
-            assertEquals(((ParsingException) rootCauses[0]).getLineNumber(), 1);
-            assertEquals(((ParsingException) rootCauses[0]).getColumnNumber(), 2);
-            assertEquals(ElasticsearchException.getExceptionName(rootCauses[1]), "query_shard_exception");
-            assertEquals((rootCauses[1]).getIndex().getName(), "foo1");
-            assertEquals(rootCauses[1].getMessage(), "foobar");
-        }
-
-        {
-            final ElasticsearchException[] foobars = ElasticsearchException.guessRootCauses(new IllegalArgumentException("foobar"));
-            assertEquals(foobars.length, 1);
-            assertTrue(foobars[0] instanceof ElasticsearchException);
-            assertEquals(foobars[0].getMessage(), "foobar");
-            assertEquals(foobars[0].getCause().getClass(), IllegalArgumentException.class);
-            assertEquals(foobars[0].getExceptionName(), "illegal_argument_exception");
-        }
-
-    }
-
-    public void testDeduplicate() throws IOException {
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1});
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
-            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[]{failure, failure1, failure2});
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1,\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\",\"index_uuid\":\"_na_\",\"index\":\"foo1\"}}]}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
-            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
-                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
-            NullPointerException nullPointerException = new NullPointerException();
-            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException, new ShardSearchFailure[]{failure, failure1});
-            assertEquals(nullPointerException, ex.getCause());
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}],\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}";
-            assertEquals(expected, builder.string());
-        }
-    }
-
-    /**
-     * Check whether this exception contains an exception of the given type:
-     * either it is of the given class itself or it contains a nested cause
-     * of the given type.
-     *
-     * @param exType the exception type to look for
-     * @return whether there is a nested exception of the specified type
-     */
-    private boolean contains(Throwable t, Class<? extends Throwable> exType) {
-        if (exType == null) {
-            return false;
-        }
-        for (Throwable cause = t; t != null; t = t.getCause()) {
-            if (exType.isInstance(cause)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public void testGetRootCause() {
-        Exception root = new RuntimeException("foobar");
-        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", root)));
-        assertEquals(root, exception.getRootCause());
-        assertTrue(contains(exception, RuntimeException.class));
-        assertFalse(contains(exception, EOFException.class));
-    }
-
-    public void testToString() {
-        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
-        assertEquals("ElasticsearchException[foo]; nested: ElasticsearchException[bar]; nested: IllegalArgumentException[index is closed]; nested: RuntimeException[foobar];", exception.toString());
-    }
-
-    public void testToXContent() throws IOException {
-        {
-            ElasticsearchException ex = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-
-            String expected = "{\"type\":\"search_parse_exception\",\"reason\":\"foo\",\"line\":1,\"col\":0}";
-            assertEquals(expected, builder.string());
-        }
-        {
-            ElasticsearchException ex = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ex.toXContent(builder, PARAMS);
-            builder.endObject();
-
-            String expected = "{\"type\":\"exception\",\"reason\":\"foo\",\"caused_by\":{\"type\":\"exception\",\"reason\":\"bar\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"index is closed\",\"caused_by\":{\"type\":\"runtime_exception\",\"reason\":\"foobar\"}}}}";
-            assertEquals(expected, builder.string());
-        }
-
-        {
-            Exception ex = new FileNotFoundException("foo not found");
-            if (randomBoolean()) {
-                // just a wrapper which is omitted
-                ex = new RemoteTransportException("foobar", ex);
-            }
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-
-            String expected = "{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}";
-            assertEquals(expected, builder.string());
-        }
-
-        {
-            ParsingException ex = new ParsingException(1, 2, "foobar", null);
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-            String expected = "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}";
-            assertEquals(expected, builder.string());
-        }
-
-        { // test equivalence
-            ElasticsearchException ex =  new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-
-            XContentBuilder otherBuilder = XContentFactory.jsonBuilder();
-
-            otherBuilder.startObject();
-            ex.toXContent(otherBuilder, PARAMS);
-            otherBuilder.endObject();
-            assertEquals(otherBuilder.string(), builder.string());
-            assertEquals("{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}", builder.string());
-        }
-
-        { // render header
-            ParsingException ex = new ParsingException(1, 2, "foobar", null);
-            ex.addHeader("test", "some value");
-            ex.addHeader("test_multi", "some value", "another value");
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            ElasticsearchException.toXContent(builder, PARAMS, ex);
-            builder.endObject();
-            assertThat(builder.string(), Matchers.anyOf( // iteration order depends on platform
-                            equalTo("{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2,\"header\":{\"test_multi\":[\"some value\",\"another value\"],\"test\":\"some value\"}}"),
-                            equalTo("{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2,\"header\":{\"test\":\"some value\",\"test_multi\":[\"some value\",\"another value\"]}}")
-            ));
-        }
-    }
-
-    public void testSerializeElasticsearchException() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        ParsingException ex = new ParsingException(1, 2, "foobar", null);
-        out.writeException(ex);
-
-        StreamInput in = out.bytes().streamInput();
-        ParsingException e = in.readException();
-        assertEquals(ex.getIndex(), e.getIndex());
-        assertEquals(ex.getMessage(), e.getMessage());
-        assertEquals(ex.getLineNumber(), e.getLineNumber());
-        assertEquals(ex.getColumnNumber(), e.getColumnNumber());
-    }
-
-    public void testSerializeUnknownException() throws IOException {
-        BytesStreamOutput out = new BytesStreamOutput();
-        ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
-        final Exception ex = new UnknownException("eggplant", parsingException);
-        out.writeException(ex);
-
-        StreamInput in = out.bytes().streamInput();
-        Throwable throwable = in.readException();
-        assertEquals("unknown_exception: eggplant", throwable.getMessage());
-        assertTrue(throwable instanceof ElasticsearchException);
-        ParsingException e = (ParsingException)throwable.getCause();
-                assertEquals(parsingException.getIndex(), e.getIndex());
-        assertEquals(parsingException.getMessage(), e.getMessage());
-        assertEquals(parsingException.getLineNumber(), e.getLineNumber());
-        assertEquals(parsingException.getColumnNumber(), e.getColumnNumber());
-    }
-
-    public void testWriteThrowable() throws IOException {
-
-        final QueryShardException queryShardException = new QueryShardException(new Index("foo", "_na_"), "foobar", null);
-        final UnknownException unknownException = new UnknownException("this exception is unknown", queryShardException);
-
-        final Exception[] causes = new Exception[]{
-                new IllegalStateException("foobar"),
-                new IllegalArgumentException("alalaal"),
-                new NullPointerException("boom"),
-                new EOFException("dadada"),
-                new ElasticsearchSecurityException("nono!"),
-                new NumberFormatException("not a number"),
-                new CorruptIndexException("baaaam booom", "this is my resource"),
-                new IndexFormatTooNewException("tooo new", 1, 2, 3),
-                new IndexFormatTooOldException("tooo new", 1, 2, 3),
-                new IndexFormatTooOldException("tooo new", "very old version"),
-                new ArrayIndexOutOfBoundsException("booom"),
-                new StringIndexOutOfBoundsException("booom"),
-                new FileNotFoundException("booom"),
-                new NoSuchFileException("booom"),
-                new AlreadyClosedException("closed!!", new NullPointerException()),
-                new LockObtainFailedException("can't lock directory", new NullPointerException()),
-                unknownException};
-        for (final Exception cause : causes) {
-            BytesStreamOutput out = new BytesStreamOutput();
-            ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
-            out.writeException(ex);
-            StreamInput in = out.bytes().streamInput();
-            ElasticsearchException e = in.readException();
-            assertEquals(e.getMessage(), ex.getMessage());
-            assertTrue("Expected: " + e.getCause().getMessage() + " to contain: " +
-                            ex.getCause().getClass().getName() + " but it didn't",
-                    e.getCause().getMessage().contains(ex.getCause().getMessage()));
-            if (ex.getCause().getClass() != UnknownException.class) { // unknown exception is not directly mapped
-                assertEquals(e.getCause().getClass(), ex.getCause().getClass());
-            } else {
-                assertEquals(e.getCause().getClass(), NotSerializableExceptionWrapper.class);
-            }
-            assertArrayEquals(e.getStackTrace(), ex.getStackTrace());
-            assertTrue(e.getStackTrace().length > 1);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), cause);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), ex);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), e);
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
index f172945d84f..bca9e7d2434 100644
--- a/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ElasticsearchExceptionTests.java
@@ -19,72 +19,357 @@
 
 package org.elasticsearch;
 
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.shard.IndexShardRecoveringException;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.node.NodeClosedException;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.search.SearchContextMissingException;
+import org.elasticsearch.search.SearchParseException;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.TestSearchContext;
+import org.elasticsearch.transport.RemoteTransportException;
 import org.hamcrest.Matcher;
 
+import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.FileAlreadyExistsException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
-import static org.hamcrest.CoreMatchers.equalTo;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.CoreMatchers.hasItems;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.startsWith;
 
 public class ElasticsearchExceptionTests extends ESTestCase {
 
-    public void testToXContent() throws IOException {
-        ElasticsearchException e = new ElasticsearchException("test");
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"test\"}"));
+    public void testStatus() {
+        ElasticsearchException exception = new ElasticsearchException("test");
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
 
-        e = new IndexShardRecoveringException(new ShardId("_test", "_0", 5));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"index_shard_recovering_exception\"," +
-                "\"reason\":\"CurrentState[RECOVERING] Already recovering\",\"index_uuid\":\"_0\",\"shard\":\"5\",\"index\":\"_test\"}"));
+        exception = new ElasticsearchException("test", new RuntimeException());
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
 
-        e = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 12), "foo", new IllegalStateException("bar"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}"));
+        exception = new ElasticsearchException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
 
-        e = new ElasticsearchException(new IllegalArgumentException("foo"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"java.lang.IllegalArgumentException: foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"foo\"}}"));
+        exception = new RemoteTransportException("test", new ResourceNotFoundException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND));
 
-        e = new ElasticsearchException("foo", new IllegalStateException("bar"));
-        assertExceptionAsJson(e, false, equalTo("{\"type\":\"exception\",\"reason\":\"foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}}"));
+        exception = new RemoteTransportException("test", new ResourceAlreadyExistsException("test"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
 
-        // Test the same exception but with the "rest.exception.stacktrace.skip" parameter disabled: the stack_trace must be present
-        // in the JSON. Since the stack can be large, it only checks the beginning of the JSON.
-        assertExceptionAsJson(e, true, startsWith("{\"type\":\"exception\",\"reason\":\"foo\"," +
-                "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"," +
-                "\"stack_trace\":\"java.lang.IllegalStateException: bar"));
+        exception = new RemoteTransportException("test", new IllegalArgumentException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST));
+
+        exception = new RemoteTransportException("test", new IllegalStateException("foobar"));
+        assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
     }
 
-    public void testToXContentWithHeaders() throws IOException {
+    public void testGuessRootCause() {
+        {
+            ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                    new IndexNotFoundException("foo", new RuntimeException("foobar"))));
+            ElasticsearchException[] rootCauses = exception.guessRootCauses();
+            assertEquals(rootCauses.length, 1);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "index_not_found_exception");
+            assertEquals(rootCauses[0].getMessage(), "no such index");
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1});
+            if (randomBoolean()) {
+                rootCauses = (randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex).guessRootCauses();
+            } else {
+                rootCauses = ElasticsearchException.guessRootCauses(randomBoolean() ? new RemoteTransportException("remoteboom", ex) : ex);
+            }
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+
+            ElasticsearchException oneLevel = new ElasticsearchException("foo", new RuntimeException("foobar"));
+            rootCauses = oneLevel.guessRootCauses();
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "exception");
+            assertEquals(rootCauses[0].getMessage(), "foo");
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(
+                    new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            final ElasticsearchException[] rootCauses = ex.guessRootCauses();
+            assertEquals(rootCauses.length, 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[0]), "parsing_exception");
+            assertEquals(rootCauses[0].getMessage(), "foobar");
+            assertEquals(((ParsingException) rootCauses[0]).getLineNumber(), 1);
+            assertEquals(((ParsingException) rootCauses[0]).getColumnNumber(), 2);
+            assertEquals(ElasticsearchException.getExceptionName(rootCauses[1]), "query_shard_exception");
+            assertEquals((rootCauses[1]).getIndex().getName(), "foo1");
+            assertEquals(rootCauses[1].getMessage(), "foobar");
+        }
+
+        {
+            final ElasticsearchException[] foobars = ElasticsearchException.guessRootCauses(new IllegalArgumentException("foobar"));
+            assertEquals(foobars.length, 1);
+            assertTrue(foobars[0] instanceof ElasticsearchException);
+            assertEquals(foobars[0].getMessage(), "foobar");
+            assertEquals(foobars[0].getCause().getClass(), IllegalArgumentException.class);
+            assertEquals(foobars[0].getExceptionName(), "illegal_argument_exception");
+        }
+    }
+
+    public void testDeduplicate() throws IOException {
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    randomBoolean() ? failure1.getCause() : failure.getCause(), new ShardSearchFailure[]{failure, failure1});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\"," +
+                    "\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":" +
+                    "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 1));
+            ShardSearchFailure failure2 = new ShardSearchFailure(new QueryShardException(new Index("foo1", "_na_"), "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo1", "_na_"), 2));
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",
+                    new ShardSearchFailure[]{failure, failure1, failure2});
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}},{\"shard\":1," +
+                    "\"index\":\"foo1\",\"node\":\"node_1\",\"reason\":{\"type\":\"query_shard_exception\",\"reason\":\"foobar\"," +
+                    "\"index_uuid\":\"_na_\",\"index\":\"foo1\"}}]}";
+            assertEquals(expected, builder.string());
+        }
+        {
+            ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 1));
+            ShardSearchFailure failure1 = new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                    new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
+            NullPointerException nullPointerException = new NullPointerException();
+            SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", nullPointerException,
+                    new ShardSearchFailure[]{failure, failure1});
+            assertEquals(nullPointerException, ex.getCause());
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            ex.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            String expected = "{\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\"," +
+                    "\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]," +
+                    "\"caused_by\":{\"type\":\"null_pointer_exception\",\"reason\":null}}";
+            assertEquals(expected, builder.string());
+        }
+    }
+
+    /**
+     * Check whether this exception contains an exception of the given type:
+     * either it is of the given class itself or it contains a nested cause
+     * of the given type.
+     *
+     * @param exType the exception type to look for
+     * @return whether there is a nested exception of the specified type
+     */
+    private static boolean contains(Throwable t, Class<? extends Throwable> exType) {
+        if (exType == null) {
+            return false;
+        }
+        for (Throwable cause = t; t != null; t = t.getCause()) {
+            if (exType.isInstance(cause)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public void testGetRootCause() {
+        Exception root = new RuntimeException("foobar");
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", root)));
+        assertEquals(root, exception.getRootCause());
+        assertTrue(contains(exception, RuntimeException.class));
+        assertFalse(contains(exception, EOFException.class));
+    }
+
+    public void testToString() {
+        ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar",
+                new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+        assertEquals("ElasticsearchException[foo]; nested: ElasticsearchException[bar]; nested: IllegalArgumentException" +
+                "[index is closed]; nested: RuntimeException[foobar];", exception.toString());
+    }
+
+    public void testToXContent() throws IOException {
+        {
+            ElasticsearchException e = new ElasticsearchException("test");
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"test\"}");
+        }
+        {
+            ElasticsearchException e = new IndexShardRecoveringException(new ShardId("_test", "_0", 5));
+            assertExceptionAsJson(e, "{\"type\":\"index_shard_recovering_exception\"," +
+                    "\"reason\":\"CurrentState[RECOVERING] Already recovering\",\"index_uuid\":\"_0\"," +
+                    "\"shard\":\"5\",\"index\":\"_test\"}");
+        }
+        {
+            ElasticsearchException e = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 12), "foo",
+                    new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException(new IllegalArgumentException("foo"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"java.lang.IllegalArgumentException: foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"foo\"}}");
+        }
+        {
+            ElasticsearchException e = new SearchParseException(new TestSearchContext(null), "foo", new XContentLocation(1,0));
+            assertExceptionAsJson(e, "{\"type\":\"search_parse_exception\",\"reason\":\"foo\",\"line\":1,\"col\":0}");
+        }
+        {
+            ElasticsearchException ex = new ElasticsearchException("foo",
+                    new ElasticsearchException("bar", new IllegalArgumentException("index is closed", new RuntimeException("foobar"))));
+            assertExceptionAsJson(ex, "{\"type\":\"exception\",\"reason\":\"foo\",\"caused_by\":{\"type\":\"exception\"," +
+                    "\"reason\":\"bar\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"index is closed\"," +
+                    "\"caused_by\":{\"type\":\"runtime_exception\",\"reason\":\"foobar\"}}}}");
+        }
+        {
+            ElasticsearchException e = new ElasticsearchException("foo", new IllegalStateException("bar"));
+            assertExceptionAsJson(e, "{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"}}");
+
+            // Test the same exception but with the "rest.exception.stacktrace.skip" parameter disabled: the stack_trace must be present
+            // in the JSON. Since the stack can be large, it only checks the beginning of the JSON.
+            ToXContent.Params params = new ToXContent.MapParams(
+                    Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"));
+            String actual;
+            try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                builder.startObject();
+                e.toXContent(builder, params);
+                builder.endObject();
+                actual = builder.string();
+            }
+            assertThat(actual, startsWith("{\"type\":\"exception\",\"reason\":\"foo\"," +
+                    "\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"bar\"," +
+                    "\"stack_trace\":\"java.lang.IllegalStateException: bar" +
+                    (Constants.WINDOWS ? "\\r\\n" : "\\n") +
+                    "\\tat org.elasticsearch."));
+        }
+    }
+
+    public void testGenerateThrowableToXContent() throws IOException {
+        {
+            Exception ex;
+            if (randomBoolean()) {
+                // just a wrapper which is omitted
+                ex = new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            } else {
+                ex = new FileNotFoundException("foo not found");
+            }
+            assertExceptionAsJson(ex, "{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}");
+        }
+        {
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            assertExceptionAsJson(ex, "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}");
+        }
+
+        { // test equivalence
+            ElasticsearchException ex =  new RemoteTransportException("foobar", new FileNotFoundException("foo not found"));
+            String toXContentString = Strings.toString(ex);
+            String throwableString = Strings.toString((builder, params) -> {
+                ElasticsearchException.generateThrowableXContent(builder, params, ex);
+                return builder;
+            });
+
+            assertEquals(throwableString, toXContentString);
+            assertEquals("{\"type\":\"file_not_found_exception\",\"reason\":\"foo not found\"}", toXContentString);
+        }
+
+        { // render header and metadata
+            ParsingException ex = new ParsingException(1, 2, "foobar", null);
+            ex.addMetadata("es.test1", "value1");
+            ex.addMetadata("es.test2", "value2");
+            ex.addHeader("test", "some value");
+            ex.addHeader("test_multi", "some value", "another value");
+            String expected = "{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2," +
+                    "\"test1\":\"value1\",\"test2\":\"value2\"," +
+                    "\"header\":{\"test_multi\":" +
+                    "[\"some value\",\"another value\"],\"test\":\"some value\"}}";
+            assertExceptionAsJson(ex, expected);
+        }
+    }
+
+    public void testToXContentWithHeadersAndMetadata() throws IOException {
         ElasticsearchException e = new ElasticsearchException("foo",
                                         new ElasticsearchException("bar",
                                                 new ElasticsearchException("baz",
                                                         new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES)))));
         e.addHeader("foo_0", "0");
         e.addHeader("foo_1", "1");
-        e.addHeader("es.header_foo_0", "foo_0");
-        e.addHeader("es.header_foo_1", "foo_1");
+        e.addMetadata("es.metadata_foo_0", "foo_0");
+        e.addMetadata("es.metadata_foo_1", "foo_1");
 
         final String expectedJson = "{"
             + "\"type\":\"exception\","
             + "\"reason\":\"foo\","
-            + "\"header_foo_0\":\"foo_0\","
-            + "\"header_foo_1\":\"foo_1\","
+            + "\"metadata_foo_0\":\"foo_0\","
+            + "\"metadata_foo_1\":\"foo_1\","
             + "\"caused_by\":{"
                 + "\"type\":\"exception\","
                 + "\"reason\":\"bar\","
@@ -103,10 +388,10 @@ public class ElasticsearchExceptionTests extends ESTestCase {
                 + "}"
         + "}";
 
-        assertExceptionAsJson(e, false, equalTo(expectedJson));
+        assertExceptionAsJson(e, expectedJson);
 
         ElasticsearchException parsed;
-        try (XContentParser parser = XContentType.JSON.xContent().createParser(expectedJson)) {
+        try (XContentParser parser = createParser(XContentType.JSON.xContent(), expectedJson)) {
             assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
             parsed = ElasticsearchException.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
@@ -115,11 +400,12 @@ public class ElasticsearchExceptionTests extends ESTestCase {
 
         assertNotNull(parsed);
         assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
-        assertThat(parsed.getHeaderKeys(), hasSize(4));
-        assertEquals(parsed.getHeader("header_foo_0").get(0), "foo_0");
-        assertEquals(parsed.getHeader("header_foo_1").get(0), "foo_1");
+        assertThat(parsed.getHeaderKeys(), hasSize(2));
         assertEquals(parsed.getHeader("foo_0").get(0), "0");
         assertEquals(parsed.getHeader("foo_1").get(0), "1");
+        assertThat(parsed.getMetadataKeys(), hasSize(2));
+        assertEquals(parsed.getMetadata("es.metadata_foo_0").get(0), "foo_0");
+        assertEquals(parsed.getMetadata("es.metadata_foo_1").get(0), "foo_1");
 
         ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
@@ -142,7 +428,7 @@ public class ElasticsearchExceptionTests extends ESTestCase {
                                                     .endObject();
 
         ElasticsearchException parsed;
-        try (XContentParser parser = xContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(xContent, builder.bytes())) {
             assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
             parsed = ElasticsearchException.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
@@ -163,7 +449,7 @@ public class ElasticsearchExceptionTests extends ESTestCase {
         XContentBuilder builder = XContentBuilder.builder(xContent).startObject().value(e).endObject();
 
         ElasticsearchException parsed;
-        try (XContentParser parser = xContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(builder)) {
             assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
             parsed = ElasticsearchException.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
@@ -182,31 +468,32 @@ public class ElasticsearchExceptionTests extends ESTestCase {
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(),
                 "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
-        assertThat(cause.getHeaderKeys(), hasSize(2));
-        assertThat(cause.getHeader("index"), hasItem("_test"));
-        assertThat(cause.getHeader("index_uuid"), hasItem("_na_"));
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
     }
 
-    public void testFromXContentWithHeaders() throws IOException {
+    public void testFromXContentWithHeadersAndMetadata() throws IOException {
         RoutingMissingException routing = new RoutingMissingException("_test", "_type", "_id");
         ElasticsearchException baz = new ElasticsearchException("baz", routing);
         baz.addHeader("baz_0", "baz0");
-        baz.addHeader("es.baz_1", "baz1");
+        baz.addMetadata("es.baz_1", "baz1");
         baz.addHeader("baz_2", "baz2");
-        baz.addHeader("es.baz_3", "baz3");
+        baz.addMetadata("es.baz_3", "baz3");
         ElasticsearchException bar = new ElasticsearchException("bar", baz);
-        bar.addHeader("es.bar_0", "bar0");
+        bar.addMetadata("es.bar_0", "bar0");
         bar.addHeader("bar_1", "bar1");
-        bar.addHeader("es.bar_2", "bar2");
+        bar.addMetadata("es.bar_2", "bar2");
         ElasticsearchException foo = new ElasticsearchException("foo", bar);
-        foo.addHeader("es.foo_0", "foo0");
+        foo.addMetadata("es.foo_0", "foo0");
         foo.addHeader("foo_1", "foo1");
 
         final XContent xContent = randomFrom(XContentType.values()).xContent();
         XContentBuilder builder = XContentBuilder.builder(xContent).startObject().value(foo).endObject();
 
         ElasticsearchException parsed;
-        try (XContentParser parser = xContent.createParser(builder.bytes())) {
+        try (XContentParser parser = createParser(builder)) {
             assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
             parsed = ElasticsearchException.fromXContent(parser);
             assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
@@ -215,31 +502,308 @@ public class ElasticsearchExceptionTests extends ESTestCase {
 
         assertNotNull(parsed);
         assertEquals(parsed.getMessage(), "Elasticsearch exception [type=exception, reason=foo]");
-        assertThat(parsed.getHeaderKeys(), hasSize(2));
-        assertThat(parsed.getHeader("foo_0"), hasItem("foo0"));
+        assertThat(parsed.getHeaderKeys(), hasSize(1));
         assertThat(parsed.getHeader("foo_1"), hasItem("foo1"));
+        assertThat(parsed.getMetadataKeys(), hasSize(1));
+        assertThat(parsed.getMetadata("es.foo_0"), hasItem("foo0"));
 
         ElasticsearchException cause = (ElasticsearchException) parsed.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=bar]");
-        assertThat(cause.getHeaderKeys(), hasSize(3));
-        assertThat(cause.getHeader("bar_0"), hasItem("bar0"));
+        assertThat(cause.getHeaderKeys(), hasSize(1));
         assertThat(cause.getHeader("bar_1"), hasItem("bar1"));
-        assertThat(cause.getHeader("bar_2"), hasItem("bar2"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.bar_0"), hasItem("bar0"));
+        assertThat(cause.getMetadata("es.bar_2"), hasItem("bar2"));
 
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(), "Elasticsearch exception [type=exception, reason=baz]");
-        assertThat(cause.getHeaderKeys(), hasSize(4));
+        assertThat(cause.getHeaderKeys(), hasSize(2));
         assertThat(cause.getHeader("baz_0"), hasItem("baz0"));
-        assertThat(cause.getHeader("baz_1"), hasItem("baz1"));
         assertThat(cause.getHeader("baz_2"), hasItem("baz2"));
-        assertThat(cause.getHeader("baz_3"), hasItem("baz3"));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.baz_1"), hasItem("baz1"));
+        assertThat(cause.getMetadata("es.baz_3"), hasItem("baz3"));
 
         cause = (ElasticsearchException) cause.getCause();
         assertEquals(cause.getMessage(),
                 "Elasticsearch exception [type=routing_missing_exception, reason=routing is required for [_test]/[_type]/[_id]]");
-        assertThat(cause.getHeaderKeys(), hasSize(2));
-        assertThat(cause.getHeader("index"), hasItem("_test"));
-        assertThat(cause.getHeader("index_uuid"), hasItem("_na_"));
+        assertThat(cause.getHeaderKeys(), hasSize(0));
+        assertThat(cause.getMetadataKeys(), hasSize(2));
+        assertThat(cause.getMetadata("es.index"), hasItem("_test"));
+        assertThat(cause.getMetadata("es.index_uuid"), hasItem("_na_"));
+    }
+
+    /**
+     * Test that some values like arrays of numbers are ignored when parsing back
+     * an exception.
+     */
+    public void testFromXContentWithIgnoredMetadataAndHeaders() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        // The exception content to parse is built using a XContentBuilder
+        // because the current Java API does not allow to add metadata/headers
+        // of other types than list of strings.
+        BytesReference originalBytes;
+        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
+            builder.startObject()
+                    .field("metadata_int", 1)
+                    .array("metadata_array_of_ints", new int[]{8, 13, 21})
+                    .field("reason", "Custom reason")
+                    .array("metadata_array_of_boolean", new boolean[]{false, false})
+                    .startArray("metadata_array_of_objects")
+                        .startObject()
+                            .field("object_array_one", "value_one")
+                        .endObject()
+                        .startObject()
+                            .field("object_array_two", "value_two")
+                        .endObject()
+                    .endArray()
+                    .field("type", "custom_exception")
+                    .field("metadata_long", 1L)
+                    .array("metadata_array_of_longs", new long[]{2L, 3L, 5L})
+                    .field("metadata_other", "some metadata")
+                    .startObject("header")
+                        .field("header_string", "some header")
+                        .array("header_array_of_strings", new String[]{"foo", "bar", "baz"})
+                    .endObject()
+                    .startObject("metadata_object")
+                        .field("object_field", "value")
+                    .endObject()
+            .endObject();
+
+            originalBytes = builder.bytes();
+        }
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsedException);
+        assertEquals("Elasticsearch exception [type=custom_exception, reason=Custom reason]", parsedException.getMessage());
+        assertEquals(2, parsedException.getHeaderKeys().size());
+        assertThat(parsedException.getHeader("header_string"), hasItem("some header"));
+        assertThat(parsedException.getHeader("header_array_of_strings"), hasItems("foo", "bar", "baz"));
+        assertEquals(1, parsedException.getMetadataKeys().size());
+        assertThat(parsedException.getMetadata("es.metadata_other"), hasItem("some metadata"));
+    }
+
+    public void testThrowableToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        final Tuple<Throwable, ElasticsearchException> exceptions = randomExceptions();
+        final Throwable throwable = exceptions.v1();
+
+        BytesReference throwableBytes = XContentHelper.toXContent((builder, params) -> {
+            ElasticsearchException.generateThrowableXContent(builder, params, throwable);
+            return builder;
+        }, xContent.type(), randomBoolean());
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, throwableBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertDeepEquals(exceptions.v2(), parsedException);
+    }
+
+    public void testUnknownFailureToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        BytesReference failureBytes = XContentHelper.toXContent((builder, params) -> {
+            // Prints a null failure using generateFailureXContent()
+            ElasticsearchException.generateFailureXContent(builder, params, null, randomBoolean());
+            return builder;
+        }, xContent.type(), randomBoolean());
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+
+        // Failure was null, expecting a "unknown" reason
+        assertEquals("Elasticsearch exception [type=exception, reason=unknown]", parsedFailure.getMessage());
+        assertEquals(0, parsedFailure.getHeaders().size());
+        assertEquals(0, parsedFailure.getMetadata().size());
+    }
+
+    public void testFailureToAndFromXContentWithNoDetails() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        final Exception failure = (Exception) randomExceptions().v1();
+        BytesReference failureBytes = XContentHelper.toXContent((builder, params) -> {
+            ElasticsearchException.generateFailureXContent(builder, params, failure, false);
+            return builder;
+        }, xContent.type(), randomBoolean());
+
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            failureBytes = shuffleXContent(parser, randomBoolean()).bytes();
+        }
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertNotNull(parsedFailure);
+
+        String reason;
+        if (failure instanceof ElasticsearchException) {
+            reason = failure.getClass().getSimpleName() + "[" + failure.getMessage() + "]";
+        } else {
+            reason = "No ElasticsearchException found";
+        }
+        assertEquals(ElasticsearchException.buildMessage("exception", reason, null), parsedFailure.getMessage());
+        assertEquals(0, parsedFailure.getHeaders().size());
+        assertEquals(0, parsedFailure.getMetadata().size());
+        assertNull(parsedFailure.getCause());
+    }
+
+    public void testFailureToAndFromXContentWithDetails() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        Exception failure;
+        Throwable failureCause;
+        ElasticsearchException expected;
+        ElasticsearchException expectedCause;
+        ElasticsearchException suppressed;
+
+        switch (randomIntBetween(0, 6)) {
+            case 0: // Simple elasticsearch exception without cause
+                failure = new NoNodeAvailableException("A");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]");
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=no_node_available_exception, reason=A]"));
+                break;
+
+            case 1: // Simple elasticsearch exception with headers (other metadata of type number are not parsed)
+                failure = new CircuitBreakingException("B", 5_000, 2_000);
+                ((ElasticsearchException) failure).addHeader("header_name", "0", "1");
+                expected = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
+                expected.addHeader("header_name", "0", "1");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=B]");
+                suppressed.addHeader("header_name", "0", "1");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 2: // Elasticsearch exception with a cause, headers and parsable metadata
+                failureCause = new NullPointerException("var is null");
+                failure = new ScriptException("C", failureCause, singletonList("stack"), "test", "painless");
+                ((ElasticsearchException) failure).addHeader("script_name", "my_script");
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=null_pointer_exception, reason=var is null]");
+                expected = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]", expectedCause);
+                expected.addHeader("script_name", "my_script");
+                expected.addMetadata("es.lang", "painless");
+                expected.addMetadata("es.script", "test");
+                expected.addMetadata("es.script_stack", "stack");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=script_exception, reason=C]");
+                suppressed.addHeader("script_name", "my_script");
+                suppressed.addMetadata("es.lang", "painless");
+                suppressed.addMetadata("es.script", "test");
+                suppressed.addMetadata("es.script_stack", "stack");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 3: // JDK exception without cause
+                failure = new IllegalStateException("D");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=illegal_state_exception, reason=D]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 4: // JDK exception with cause
+                failureCause = new RoutingMissingException("idx", "type", "id");
+                failure = new RuntimeException("E", failureCause);
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=routing_missing_exception, " +
+                        "reason=routing is required for [idx]/[type]/[id]]");
+                expectedCause.addMetadata("es.index", "idx");
+                expectedCause.addMetadata("es.index_uuid", "_na_");
+                expected = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]", expectedCause);
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=E]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 5: // Wrapped exception with cause
+                failureCause = new FileAlreadyExistsException("File exists");
+                failure = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 5), "F", failureCause);
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=file_already_exists_exception, reason=File exists]");
+                // strangely, the wrapped exception appears as the root cause...
+                suppressed = new ElasticsearchException("Elasticsearch exception [type=broadcast_shard_operation_failed_exception, " +
+                        "reason=F]");
+                expected.addSuppressed(suppressed);
+                break;
+
+            case 6: // SearchPhaseExecutionException with cause and multiple failures
+                DiscoveryNode node = new DiscoveryNode("node_g", buildNewFakeTransportAddress(), Version.CURRENT);
+                failureCause = new NodeClosedException(node);
+                failureCause = new NoShardAvailableActionException(new ShardId("_index_g", "_uuid_g", 6), "node_g", failureCause);
+                ShardSearchFailure[] shardFailures = new ShardSearchFailure[]{
+                        new ShardSearchFailure(new ParsingException(0, 0, "Parsing g", null),
+                                new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61))),
+                        new ShardSearchFailure(new RepositoryException("repository_g", "Repo"),
+                                new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62))),
+                        new ShardSearchFailure(new SearchContextMissingException(0L), null)
+                };
+                failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
+
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=node_closed_exception, " +
+                        "reason=node closed " + node + "]");
+                expectedCause = new ElasticsearchException("Elasticsearch exception [type=no_shard_available_action_exception, " +
+                        "reason=node_g]", expectedCause);
+                expectedCause.addMetadata("es.index", "_index_g");
+                expectedCause.addMetadata("es.index_uuid", "_uuid_g");
+                expectedCause.addMetadata("es.shard", "6");
+
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " +
+                        "reason=G]", expectedCause);
+                expected.addMetadata("es.phase", "phase_g");
+
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=parsing_exception, reason=Parsing g]"));
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=repository_exception, " +
+                        "reason=[repository_g] Repo]"));
+                expected.addSuppressed(new ElasticsearchException("Elasticsearch exception [type=search_context_missing_exception, " +
+                        "reason=No search context found for id [0]]"));
+                break;
+            default:
+                throw new UnsupportedOperationException("Failed to generate randomized failure");
+        }
+
+        Exception finalFailure = failure;
+        BytesReference failureBytes = XContentHelper.toXContent((builder, params) -> {
+            ElasticsearchException.generateFailureXContent(builder, params, finalFailure, true);
+            return builder;
+        }, xContent.type(), randomBoolean());
+
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            failureBytes = shuffleXContent(parser, randomBoolean()).bytes();
+        }
+
+        ElasticsearchException parsedFailure;
+        try (XContentParser parser = createParser(xContent, failureBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedFailure = ElasticsearchException.failureFromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertDeepEquals(expected, parsedFailure);
     }
 
     /**
@@ -248,17 +812,166 @@ public class ElasticsearchExceptionTests extends ESTestCase {
      * By default, the stack trace of the exception is not rendered. The parameter `errorTrace` forces the stack trace to
      * be rendered like the REST API does when the "error_trace" parameter is set to true.
      */
-    private static void assertExceptionAsJson(ElasticsearchException e, boolean errorTrace, Matcher<String> expected)
-            throws IOException {
-        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        if (errorTrace) {
-            params = new ToXContent.MapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"));
+    private static void assertToXContentAsJson(ToXContent e, String expectedJson) throws IOException {
+        BytesReference actual = XContentHelper.toXContent(e, XContentType.JSON, randomBoolean());
+        assertToXContentEquivalent(new BytesArray(expectedJson), actual, XContentType.JSON);
+    }
+
+    private static void assertExceptionAsJson(Exception e, String expectedJson) throws IOException {
+        assertToXContentAsJson((builder, params) -> {
+            ElasticsearchException.generateThrowableXContent(builder, params, e);
+            return builder;
+        }, expectedJson);
+    }
+
+    public static void assertDeepEquals(ElasticsearchException expected, ElasticsearchException actual) {
+        do {
+            if (expected == null) {
+                assertNull(actual);
+            } else {
+                assertNotNull(actual);
+            }
+
+            assertEquals(expected.getMessage(), actual.getMessage());
+            assertEquals(expected.getHeaders(), actual.getHeaders());
+            assertEquals(expected.getMetadata(), actual.getMetadata());
+            assertEquals(expected.getResourceType(), actual.getResourceType());
+            assertEquals(expected.getResourceId(), actual.getResourceId());
+
+            Throwable[] expectedSuppressed = expected.getSuppressed();
+            Throwable[] actualSuppressed = actual.getSuppressed();
+
+            if (expectedSuppressed == null) {
+                assertNull(actualSuppressed);
+            } else {
+                assertNotNull(actualSuppressed);
+                assertEquals(expectedSuppressed.length, actualSuppressed.length);
+                for (int i = 0; i < expectedSuppressed.length; i++) {
+                    assertDeepEquals((ElasticsearchException) expectedSuppressed[i], (ElasticsearchException) actualSuppressed[i]);
+                }
+            }
+
+            expected = (ElasticsearchException) expected.getCause();
+            actual = (ElasticsearchException) actual.getCause();
+            if (expected == null) {
+                assertNull(actual);
+            }
+        } while (expected != null);
+    }
+
+    private static Tuple<Throwable, ElasticsearchException> randomExceptions() {
+        Throwable actual;
+        ElasticsearchException expected;
+
+        int type = randomIntBetween(0, 5);
+        switch (type) {
+            case 0:
+                actual = new ClusterBlockException(singleton(DiscoverySettings.NO_MASTER_BLOCK_WRITES));
+                expected = new ElasticsearchException("Elasticsearch exception [type=cluster_block_exception, " +
+                        "reason=blocked by: [SERVICE_UNAVAILABLE/2/no master];]");
+                break;
+            case 1:
+                actual = new CircuitBreakingException("Data too large", 123, 456);
+                expected = new ElasticsearchException("Elasticsearch exception [type=circuit_breaking_exception, reason=Data too large]");
+                break;
+            case 2:
+                actual = new SearchParseException(new TestSearchContext(null), "Parse failure", new XContentLocation(12, 98));
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_parse_exception, reason=Parse failure]");
+                break;
+            case 3:
+                actual = new IllegalArgumentException("Closed resource", new RuntimeException("Resource"));
+                expected = new ElasticsearchException("Elasticsearch exception [type=illegal_argument_exception, reason=Closed resource]",
+                                new ElasticsearchException("Elasticsearch exception [type=runtime_exception, reason=Resource]"));
+                break;
+            case 4:
+                actual = new SearchPhaseExecutionException("search", "all shards failed",
+                            new ShardSearchFailure[]{
+                                    new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                                            new SearchShardTarget("node_1", new Index("foo", "_na_"), 1))
+                            });
+                expected = new ElasticsearchException("Elasticsearch exception [type=search_phase_execution_exception, " +
+                        "reason=all shards failed]");
+                expected.addMetadata("es.phase", "search");
+                break;
+            case 5:
+                actual = new ElasticsearchException("Parsing failed",
+                            new ParsingException(9, 42, "Wrong state",
+                                new NullPointerException("Unexpected null value")));
+
+                ElasticsearchException expectedCause = new ElasticsearchException("Elasticsearch exception [type=parsing_exception, " +
+                        "reason=Wrong state]", new ElasticsearchException("Elasticsearch exception [type=null_pointer_exception, " +
+                        "reason=Unexpected null value]"));
+                expected = new ElasticsearchException("Elasticsearch exception [type=exception, reason=Parsing failed]", expectedCause);
+                break;
+            default:
+                throw new UnsupportedOperationException("No randomized exceptions generated for type [" + type + "]");
         }
-        try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
-            builder.startObject();
-            e.toXContent(builder, params);
-            builder.endObject();
-            assertThat(builder.bytes().utf8ToString(), expected);
+
+        if (actual instanceof ElasticsearchException) {
+            ElasticsearchException actualException = (ElasticsearchException) actual;
+            if (randomBoolean()) {
+                int nbHeaders = randomIntBetween(1, 5);
+                Map<String, List<String>> randomHeaders = new HashMap<>(nbHeaders);
+
+                for (int i = 0; i < nbHeaders; i++) {
+                    List<String> values = new ArrayList<>();
+
+                    int nbValues = randomIntBetween(1, 3);
+                    for (int j = 0; j < nbValues; j++) {
+                        values.add(frequently() ? randomAsciiOfLength(5) : "");
+                    }
+                    randomHeaders.put("header_" + i, values);
+                }
+
+                for (Map.Entry<String, List<String>> entry : randomHeaders.entrySet()) {
+                    actualException.addHeader(entry.getKey(), entry.getValue());
+                    expected.addHeader(entry.getKey(), entry.getValue());
+                }
+
+                if (rarely()) {
+                    // Empty or null headers are not printed out by the toXContent method
+                    actualException.addHeader("ignored", randomBoolean() ? emptyList() : null);
+                }
+            }
+
+            if (randomBoolean()) {
+                int nbMetadata = randomIntBetween(1, 5);
+                Map<String, List<String>> randomMetadata = new HashMap<>(nbMetadata);
+
+                for (int i = 0; i < nbMetadata; i++) {
+                    List<String> values = new ArrayList<>();
+
+                    int nbValues = randomIntBetween(1, 3);
+                    for (int j = 0; j < nbValues; j++) {
+                        values.add(frequently() ? randomAsciiOfLength(5) : "");
+                    }
+                    randomMetadata.put("es.metadata_" + i, values);
+                }
+
+                for (Map.Entry<String, List<String>> entry : randomMetadata.entrySet()) {
+                    actualException.addMetadata(entry.getKey(), entry.getValue());
+                    expected.addMetadata(entry.getKey(), entry.getValue());
+                }
+
+                if (rarely()) {
+                    // Empty or null metadata are not printed out by the toXContent method
+                    actualException.addMetadata("es.ignored", randomBoolean() ? emptyList() : null);
+                }
+            }
+
+            if (randomBoolean()) {
+                int nbResources = randomIntBetween(1, 5);
+                for (int i = 0; i < nbResources; i++) {
+                    String resourceType = "type_" + i;
+                    String[] resourceIds = new String[randomIntBetween(1, 3)];
+                    for (int j = 0; j < resourceIds.length; j++) {
+                        resourceIds[j] = frequently() ? randomAsciiOfLength(5) : "";
+                    }
+                    actualException.setResources(resourceType, resourceIds);
+                    expected.setResources(resourceType, resourceIds);
+                }
+            }
         }
+        return new Tuple<>(actual, expected);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 202224872b4..2cd87c0470e 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch;
 
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.TimestampParsingException;
@@ -33,8 +38,11 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
@@ -44,9 +52,6 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CancellableThreadsTests;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.env.ShardLockObtainFailedException;
@@ -81,6 +86,8 @@ import org.elasticsearch.transport.ActionTransportException;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TcpTransport;
 
+import java.io.EOFException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.AccessDeniedException;
@@ -97,6 +104,7 @@ import java.nio.file.NotDirectoryException;
 import java.nio.file.Path;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -107,6 +115,7 @@ import static java.lang.reflect.Modifier.isInterface;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -121,7 +130,6 @@ public class ExceptionSerializationTests extends ESTestCase {
         final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI())
                 .resolve("org").resolve("elasticsearch");
         final Set<? extends Class<?>> ignore = Sets.newHashSet(
-                org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException.class,
                 CancellableThreadsTests.CustomException.class,
                 org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class,
                 AbstractClientHeadersTestCase.InternalException.class);
@@ -512,30 +520,16 @@ public class ExceptionSerializationTests extends ESTestCase {
         assertEquals(1, ex.blocks().size());
     }
 
-    private String toXContent(ToXContent x) {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            builder.startObject();
-            x.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
-    }
-
     public void testNotSerializableExceptionWrapper() throws IOException {
         NotSerializableExceptionWrapper ex = serialize(new NotSerializableExceptionWrapper(new NullPointerException()));
-        assertEquals("{\"type\":\"null_pointer_exception\",\"reason\":\"null_pointer_exception: null\"}", toXContent(ex));
+        assertEquals("{\"type\":\"null_pointer_exception\",\"reason\":\"null_pointer_exception: null\"}", Strings.toString(ex));
         ex = serialize(new NotSerializableExceptionWrapper(new IllegalArgumentException("nono!")));
-        assertEquals("{\"type\":\"illegal_argument_exception\",\"reason\":\"illegal_argument_exception: nono!\"}", toXContent(ex));
+        assertEquals("{\"type\":\"illegal_argument_exception\",\"reason\":\"illegal_argument_exception: nono!\"}", Strings.toString(ex));
 
         class UnknownException extends Exception {
-
-            public UnknownException(final String message) {
+            UnknownException(final String message) {
                 super(message);
             }
-
         }
 
         Exception[] unknowns = new Exception[]{
@@ -560,28 +554,94 @@ public class ExceptionSerializationTests extends ESTestCase {
         }
     }
 
+    public void testUnknownException() throws IOException {
+        ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
+        final Exception ex = new UnknownException("eggplant", parsingException);
+        Exception exception = serialize(ex);
+        assertEquals("unknown_exception: eggplant", exception.getMessage());
+        assertTrue(exception instanceof ElasticsearchException);
+        ParsingException e = (ParsingException)exception.getCause();
+        assertEquals(parsingException.getIndex(), e.getIndex());
+        assertEquals(parsingException.getMessage(), e.getMessage());
+        assertEquals(parsingException.getLineNumber(), e.getLineNumber());
+        assertEquals(parsingException.getColumnNumber(), e.getColumnNumber());
+    }
+
+    public void testWriteThrowable() throws IOException {
+        final QueryShardException queryShardException = new QueryShardException(new Index("foo", "_na_"), "foobar", null);
+        final UnknownException unknownException = new UnknownException("this exception is unknown", queryShardException);
+
+        final Exception[] causes = new Exception[]{
+                new IllegalStateException("foobar"),
+                new IllegalArgumentException("alalaal"),
+                new NullPointerException("boom"),
+                new EOFException("dadada"),
+                new ElasticsearchSecurityException("nono!"),
+                new NumberFormatException("not a number"),
+                new CorruptIndexException("baaaam booom", "this is my resource"),
+                new IndexFormatTooNewException("tooo new", 1, 2, 3),
+                new IndexFormatTooOldException("tooo new", 1, 2, 3),
+                new IndexFormatTooOldException("tooo new", "very old version"),
+                new ArrayIndexOutOfBoundsException("booom"),
+                new StringIndexOutOfBoundsException("booom"),
+                new FileNotFoundException("booom"),
+                new NoSuchFileException("booom"),
+                new AlreadyClosedException("closed!!", new NullPointerException()),
+                new LockObtainFailedException("can't lock directory", new NullPointerException()),
+                unknownException};
+        for (final Exception cause : causes) {
+            ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
+            ElasticsearchException deserialized = serialize(ex);
+            assertEquals(deserialized.getMessage(), ex.getMessage());
+            assertTrue("Expected: " + deserialized.getCause().getMessage() + " to contain: " +
+                            ex.getCause().getClass().getName() + " but it didn't",
+                    deserialized.getCause().getMessage().contains(ex.getCause().getMessage()));
+            if (ex.getCause().getClass() != UnknownException.class) { // unknown exception is not directly mapped
+                assertEquals(deserialized.getCause().getClass(), ex.getCause().getClass());
+            } else {
+                assertEquals(deserialized.getCause().getClass(), NotSerializableExceptionWrapper.class);
+            }
+            assertArrayEquals(deserialized.getStackTrace(), ex.getStackTrace());
+            assertTrue(deserialized.getStackTrace().length > 1);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), cause);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), ex);
+            assertVersionSerializable(VersionUtils.randomVersion(random()), deserialized);
+        }
+    }
+
     public void testWithRestHeadersException() throws IOException {
-        ElasticsearchException ex = new ElasticsearchException("msg");
-        ex.addHeader("foo", "foo", "bar");
-        ex = serialize(ex);
-        assertEquals("msg", ex.getMessage());
-        assertEquals(2, ex.getHeader("foo").size());
-        assertEquals("foo", ex.getHeader("foo").get(0));
-        assertEquals("bar", ex.getHeader("foo").get(1));
+        {
+            ElasticsearchException ex = new ElasticsearchException("msg");
+            ex.addHeader("foo", "foo", "bar");
+            ex.addMetadata("es.foo_metadata", "value1", "value2");
+            ex = serialize(ex);
+            assertEquals("msg", ex.getMessage());
+            assertEquals(2, ex.getHeader("foo").size());
+            assertEquals("foo", ex.getHeader("foo").get(0));
+            assertEquals("bar", ex.getHeader("foo").get(1));
+            assertEquals(2, ex.getMetadata("es.foo_metadata").size());
+            assertEquals("value1", ex.getMetadata("es.foo_metadata").get(0));
+            assertEquals("value2", ex.getMetadata("es.foo_metadata").get(1));
+        }
+        {
+            RestStatus status = randomFrom(RestStatus.values());
+            // ensure we are carrying over the headers and metadata even if not serialized
+            UnknownHeaderException uhe = new UnknownHeaderException("msg", status);
+            uhe.addHeader("foo", "foo", "bar");
+            uhe.addMetadata("es.foo_metadata", "value1", "value2");
 
-        RestStatus status = randomFrom(RestStatus.values());
-        // ensure we are carrying over the headers even if not serialized
-        UnknownHeaderException uhe = new UnknownHeaderException("msg", status);
-        uhe.addHeader("foo", "foo", "bar");
-
-        ElasticsearchException serialize = serialize((ElasticsearchException) uhe);
-        assertTrue(serialize instanceof NotSerializableExceptionWrapper);
-        NotSerializableExceptionWrapper e = (NotSerializableExceptionWrapper) serialize;
-        assertEquals("unknown_header_exception: msg", e.getMessage());
-        assertEquals(2, e.getHeader("foo").size());
-        assertEquals("foo", e.getHeader("foo").get(0));
-        assertEquals("bar", e.getHeader("foo").get(1));
-        assertSame(status, e.status());
+            ElasticsearchException serialize = serialize((ElasticsearchException) uhe);
+            assertTrue(serialize instanceof NotSerializableExceptionWrapper);
+            NotSerializableExceptionWrapper e = (NotSerializableExceptionWrapper) serialize;
+            assertEquals("unknown_header_exception: msg", e.getMessage());
+            assertEquals(2, e.getHeader("foo").size());
+            assertEquals("foo", e.getHeader("foo").get(0));
+            assertEquals("bar", e.getHeader("foo").get(1));
+            assertEquals(2, e.getMetadata("es.foo_metadata").size());
+            assertEquals("value1", e.getMetadata("es.foo_metadata").get(0));
+            assertEquals("value2", e.getMetadata("es.foo_metadata").get(1));
+            assertSame(status, e.status());
+        }
     }
 
     public void testNoLongerPrimaryShardException() throws IOException {
@@ -595,7 +655,7 @@ public class ExceptionSerializationTests extends ESTestCase {
     public static class UnknownHeaderException extends ElasticsearchException {
         private final RestStatus status;
 
-        public UnknownHeaderException(String msg, RestStatus status) {
+        UnknownHeaderException(String msg, RestStatus status) {
             super(msg);
             this.status = status;
         }
@@ -760,7 +820,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(126, org.elasticsearch.index.mapper.MapperParsingException.class);
         ids.put(127, org.elasticsearch.search.SearchContextException.class);
         ids.put(128, org.elasticsearch.search.builder.SearchSourceBuilderException.class);
-        ids.put(129, org.elasticsearch.index.engine.EngineClosedException.class);
+        ids.put(129, null); // was org.elasticsearch.index.engine.EngineClosedException.class
         ids.put(130, org.elasticsearch.action.NoShardAvailableActionException.class);
         ids.put(131, org.elasticsearch.action.UnavailableShardsException.class);
         ids.put(132, org.elasticsearch.index.engine.FlushFailedEngineException.class);
@@ -779,6 +839,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(145, org.elasticsearch.ElasticsearchStatusException.class);
         ids.put(146, org.elasticsearch.tasks.TaskCancelledException.class);
         ids.put(147, org.elasticsearch.env.ShardLockObtainFailedException.class);
+        ids.put(148, org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {
@@ -857,5 +918,75 @@ public class ExceptionSerializationTests extends ESTestCase {
         assertEquals("shard_lock_obtain_failed_exception: [foo][1]: boom", ex.getMessage());
     }
 
+    public void testBWCHeadersAndMetadata() throws IOException {
+        //this is a request serialized with headers only, no metadata as they were added in 5.3.0
+        BytesReference decoded = new BytesArray(Base64.getDecoder().decode
+                ("AQ10ZXN0ICBtZXNzYWdlACYtb3JnLmVsYXN0aWNzZWFyY2guRXhjZXB0aW9uU2VyaWFsaXphdGlvblRlc3RzASBFeGNlcHRpb25TZXJpYWxpemF0aW9uVG" +
+                        "VzdHMuamF2YQR0ZXN03wYkc3VuLnJlZmxlY3QuTmF0aXZlTWV0aG9kQWNjZXNzb3JJbXBsAR1OYXRpdmVNZXRob2RBY2Nlc3NvckltcGwuamF2Y" +
+                        "QdpbnZva2Uw/v///w8kc3VuLnJlZmxlY3QuTmF0aXZlTWV0aG9kQWNjZXNzb3JJbXBsAR1OYXRpdmVNZXRob2RBY2Nlc3NvckltcGwuamF2YQZp" +
+                        "bnZva2U+KHN1bi5yZWZsZWN0LkRlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwBIURlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwuamF2YQZ" +
+                        "pbnZva2UrGGphdmEubGFuZy5yZWZsZWN0Lk1ldGhvZAELTWV0aG9kLmphdmEGaW52b2tl8QMzY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdG" +
+                        "VzdGluZy5SYW5kb21pemVkUnVubmVyARVSYW5kb21pemVkUnVubmVyLmphdmEGaW52b2tlsQ01Y29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkd" +
+                        "GVzdGluZy5SYW5kb21pemVkUnVubmVyJDgBFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQhldmFsdWF0ZYsHNWNvbS5jYXJyb3RzZWFyY2gucmFuZG9t" +
+                        "aXplZHRlc3RpbmcuUmFuZG9taXplZFJ1bm5lciQ5ARVSYW5kb21pemVkUnVubmVyLmphdmEIZXZhbHVhdGWvBzZjb20uY2Fycm90c2VhcmNoLnJ" +
+                        "hbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIkMTABFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQhldmFsdWF0Zb0HOWNvbS5jYXJyb3RzZW" +
+                        "FyY2gucmFuZG9taXplZHRlc3RpbmcucnVsZXMuU3RhdGVtZW50QWRhcHRlcgEVU3RhdGVtZW50QWRhcHRlci5qYXZhCGV2YWx1YXRlJDVvcmcuY" +
+                        "XBhY2hlLmx1Y2VuZS51dGlsLlRlc3RSdWxlU2V0dXBUZWFyZG93bkNoYWluZWQkMQEhVGVzdFJ1bGVTZXR1cFRlYXJkb3duQ2hhaW5lZC5qYXZh" +
+                        "CGV2YWx1YXRlMTBvcmcuYXBhY2hlLmx1Y2VuZS51dGlsLkFic3RyYWN0QmVmb3JlQWZ0ZXJSdWxlJDEBHEFic3RyYWN0QmVmb3JlQWZ0ZXJSdWx" +
+                        "lLmphdmEIZXZhbHVhdGUtMm9yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVUaHJlYWRBbmRUZXN0TmFtZSQxAR5UZXN0UnVsZVRocmVhZE" +
+                        "FuZFRlc3ROYW1lLmphdmEIZXZhbHVhdGUwN29yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVJZ25vcmVBZnRlck1heEZhaWx1cmVzJDEBI" +
+                        "1Rlc3RSdWxlSWdub3JlQWZ0ZXJNYXhGYWlsdXJlcy5qYXZhCGV2YWx1YXRlQCxvcmcuYXBhY2hlLmx1Y2VuZS51dGlsLlRlc3RSdWxlTWFya0Zh" +
+                        "aWx1cmUkMQEYVGVzdFJ1bGVNYXJrRmFpbHVyZS5qYXZhCGV2YWx1YXRlLzljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGV" +
+                        "zLlN0YXRlbWVudEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSREY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdG" +
+                        "luZy5UaHJlYWRMZWFrQ29udHJvbCRTdGF0ZW1lbnRSdW5uZXIBFlRocmVhZExlYWtDb250cm9sLmphdmEDcnVu7wI0Y29tLmNhcnJvdHNlYXJja" +
+                        "C5yYW5kb21pemVkdGVzdGluZy5UaHJlYWRMZWFrQ29udHJvbAEWVGhyZWFkTGVha0NvbnRyb2wuamF2YRJmb3JrVGltZW91dGluZ1Rhc2urBjZj" +
+                        "b20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlRocmVhZExlYWtDb250cm9sJDMBFlRocmVhZExlYWtDb250cm9sLmphdmEIZXZhbHV" +
+                        "hdGXOAzNjb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIBFVJhbmRvbWl6ZWRSdW5uZXIuamF2YQ1ydW" +
+                        "5TaW5nbGVUZXN0lAc1Y29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5SYW5kb21pemVkUnVubmVyJDUBFVJhbmRvbWl6ZWRSdW5uZ" +
+                        "XIuamF2YQhldmFsdWF0ZaIGNWNvbS5jYXJyb3RzZWFyY2gucmFuZG9taXplZHRlc3RpbmcuUmFuZG9taXplZFJ1bm5lciQ2ARVSYW5kb21pemVk" +
+                        "UnVubmVyLmphdmEIZXZhbHVhdGXUBjVjb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLlJhbmRvbWl6ZWRSdW5uZXIkNwEVUmFuZG9" +
+                        "taXplZFJ1bm5lci5qYXZhCGV2YWx1YXRl3wYwb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5BYnN0cmFjdEJlZm9yZUFmdGVyUnVsZSQxARxBYnN0cm" +
+                        "FjdEJlZm9yZUFmdGVyUnVsZS5qYXZhCGV2YWx1YXRlLTljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVud" +
+                        "EFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSQvb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZVN0b3JlQ2xhc3NO" +
+                        "YW1lJDEBG1Rlc3RSdWxlU3RvcmVDbGFzc05hbWUuamF2YQhldmFsdWF0ZSlOY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5ydWx" +
+                        "lcy5Ob1NoYWRvd2luZ09yT3ZlcnJpZGVzT25NZXRob2RzUnVsZSQxAShOb1NoYWRvd2luZ09yT3ZlcnJpZGVzT25NZXRob2RzUnVsZS5qYXZhCG" +
+                        "V2YWx1YXRlKE5jb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLk5vU2hhZG93aW5nT3JPdmVycmlkZXNPbk1ldGhvZHNSd" +
+                        "WxlJDEBKE5vU2hhZG93aW5nT3JPdmVycmlkZXNPbk1ldGhvZHNSdWxlLmphdmEIZXZhbHVhdGUoOWNvbS5jYXJyb3RzZWFyY2gucmFuZG9taXpl" +
+                        "ZHRlc3RpbmcucnVsZXMuU3RhdGVtZW50QWRhcHRlcgEVU3RhdGVtZW50QWRhcHRlci5qYXZhCGV2YWx1YXRlJDljb20uY2Fycm90c2VhcmNoLnJ" +
+                        "hbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVudEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSQ5Y29tLmNhcnJvdH" +
+                        "NlYXJjaC5yYW5kb21pemVkdGVzdGluZy5ydWxlcy5TdGF0ZW1lbnRBZGFwdGVyARVTdGF0ZW1lbnRBZGFwdGVyLmphdmEIZXZhbHVhdGUkM29yZ" +
+                        "y5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVBc3NlcnRpb25zUmVxdWlyZWQkMQEfVGVzdFJ1bGVBc3NlcnRpb25zUmVxdWlyZWQuamF2YQhl" +
+                        "dmFsdWF0ZTUsb3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZU1hcmtGYWlsdXJlJDEBGFRlc3RSdWxlTWFya0ZhaWx1cmUuamF2YQhldmF" +
+                        "sdWF0ZS83b3JnLmFwYWNoZS5sdWNlbmUudXRpbC5UZXN0UnVsZUlnbm9yZUFmdGVyTWF4RmFpbHVyZXMkMQEjVGVzdFJ1bGVJZ25vcmVBZnRlck" +
+                        "1heEZhaWx1cmVzLmphdmEIZXZhbHVhdGVAMW9yZy5hcGFjaGUubHVjZW5lLnV0aWwuVGVzdFJ1bGVJZ25vcmVUZXN0U3VpdGVzJDEBHVRlc3RSd" +
+                        "WxlSWdub3JlVGVzdFN1aXRlcy5qYXZhCGV2YWx1YXRlNjljb20uY2Fycm90c2VhcmNoLnJhbmRvbWl6ZWR0ZXN0aW5nLnJ1bGVzLlN0YXRlbWVu" +
+                        "dEFkYXB0ZXIBFVN0YXRlbWVudEFkYXB0ZXIuamF2YQhldmFsdWF0ZSREY29tLmNhcnJvdHNlYXJjaC5yYW5kb21pemVkdGVzdGluZy5UaHJlYWR" +
+                        "MZWFrQ29udHJvbCRTdGF0ZW1lbnRSdW5uZXIBFlRocmVhZExlYWtDb250cm9sLmphdmEDcnVu7wIQamF2YS5sYW5nLlRocmVhZAELVGhyZWFkLm" +
+                        "phdmEDcnVu6QUABAdoZWFkZXIyAQZ2YWx1ZTIKZXMuaGVhZGVyMwEGdmFsdWUzB2hlYWRlcjEBBnZhbHVlMQplcy5oZWFkZXI0AQZ2YWx1ZTQAA" +
+                        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +
+                        "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +
+                        "AAAAA"));
 
+        try (StreamInput in = decoded.streamInput()) {
+            //randomize the version across released and unreleased ones
+            Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+                    Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+            in.setVersion(version);
+            ElasticsearchException exception = new ElasticsearchException(in);
+            assertEquals("test  message", exception.getMessage());
+            //the headers received as part of a single set get split based on their prefix
+            assertEquals(2, exception.getHeaderKeys().size());
+            assertEquals("value1", exception.getHeader("header1").get(0));
+            assertEquals("value2", exception.getHeader("header2").get(0));
+            assertEquals(2, exception.getMetadataKeys().size());
+            assertEquals("value3", exception.getMetadata("es.header3").get(0));
+            assertEquals("value4", exception.getMetadata("es.header4").get(0));
+        }
+    }
+
+    private static class UnknownException extends Exception {
+        UnknownException(final String message, final Exception cause) {
+            super(message, cause);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/SpecialPermissionTests.java b/core/src/test/java/org/elasticsearch/SpecialPermissionTests.java
index a2cfc02c5d5..5834de32b2b 100644
--- a/core/src/test/java/org/elasticsearch/SpecialPermissionTests.java
+++ b/core/src/test/java/org/elasticsearch/SpecialPermissionTests.java
@@ -25,14 +25,18 @@ import java.security.AllPermission;
 
 /** Very simple sanity checks for {@link SpecialPermission} */
 public class SpecialPermissionTests extends ESTestCase {
-    
+
     public void testEquals() {
         assertEquals(new SpecialPermission(), new SpecialPermission());
+        assertEquals(SpecialPermission.INSTANCE, new SpecialPermission());
         assertFalse(new SpecialPermission().equals(new AllPermission()));
+        assertFalse(SpecialPermission.INSTANCE.equals(new AllPermission()));
     }
-    
+
     public void testImplies() {
-        assertTrue(new SpecialPermission().implies(new SpecialPermission()));
+        assertTrue(SpecialPermission.INSTANCE.implies(new SpecialPermission()));
+        assertTrue(SpecialPermission.INSTANCE.implies(SpecialPermission.INSTANCE));
         assertFalse(new SpecialPermission().implies(new AllPermission()));
+        assertFalse(SpecialPermission.INSTANCE.implies(new AllPermission()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java
index 1c9964547b0..35c36bee643 100644
--- a/core/src/test/java/org/elasticsearch/VersionTests.java
+++ b/core/src/test/java/org/elasticsearch/VersionTests.java
@@ -38,7 +38,10 @@ import static org.elasticsearch.Version.V_5_0_0_alpha1;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class VersionTests extends ESTestCase {
@@ -64,6 +67,10 @@ public class VersionTests extends ESTestCase {
         assertTrue(Version.fromString("5.0.0").onOrAfter(Version.fromString("5.0.0-beta2")));
         assertTrue(Version.fromString("5.0.0-rc1").onOrAfter(Version.fromString("5.0.0-beta24")));
         assertTrue(Version.fromString("5.0.0-alpha24").before(Version.fromString("5.0.0-beta0")));
+
+        assertThat(V_2_2_0, is(lessThan(V_5_0_0_alpha1)));
+        assertThat(V_2_2_0.compareTo(V_2_2_0), is(0));
+        assertThat(V_5_0_0_alpha1, is(greaterThan(V_2_2_0)));
     }
 
     public void testMin() {
diff --git a/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java b/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java
new file mode 100644
index 00000000000..6414c81058b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ActionListenerTests.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action;
+
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class ActionListenerTests extends ESTestCase {
+
+    public void testWrap() {
+        AtomicReference<Boolean> reference = new AtomicReference<>();
+        AtomicReference<Exception> exReference = new AtomicReference<>();
+
+        CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+            if (Boolean.FALSE.equals(o)) {
+                throw new IllegalArgumentException("must not be false");
+            }
+            reference.set(o);
+        };
+        ActionListener<Boolean> wrap = ActionListener.wrap(handler, exReference::set);
+        wrap.onResponse(Boolean.FALSE);
+        assertNull(reference.get());
+        assertNotNull(exReference.get());
+        assertEquals("must not be false", exReference.get().getMessage());
+        exReference.set(null);
+
+        wrap.onResponse(Boolean.TRUE);
+        assertTrue(reference.get());
+        assertNull(exReference.get());
+    }
+
+    public void testOnResponse() {
+        final int numListeners = randomIntBetween(1, 20);
+        List<AtomicReference<Boolean>> refList = new ArrayList<>();
+        List<AtomicReference<Exception>> excList = new ArrayList<>();
+        List<ActionListener<Boolean>> listeners = new ArrayList<>();
+        List<Boolean> failOnTrue = new ArrayList<>();
+        AtomicInteger exceptionCounter = new AtomicInteger(0);
+        for (int i = 0; i < numListeners; i++) {
+            boolean doFailOnTrue = rarely();
+            failOnTrue.add(doFailOnTrue);
+            AtomicReference<Boolean> reference = new AtomicReference<>();
+            AtomicReference<Exception> exReference = new AtomicReference<>();
+            refList.add(reference);
+            excList.add(exReference);
+            CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+                if (Boolean.FALSE.equals(o)) {
+                    throw new IllegalArgumentException("must not be false " + exceptionCounter.getAndIncrement());
+                }
+                if (doFailOnTrue) {
+                    throw new IllegalStateException("must not be true");
+                }
+                reference.set(o);
+            };
+            listeners.add(ActionListener.wrap(handler, exReference::set));
+        }
+
+        ActionListener.onResponse(listeners, Boolean.TRUE);
+        for (int i = 0; i < numListeners; i++) {
+            if (failOnTrue.get(i) == false) {
+                assertTrue("listener index " + i, refList.get(i).get());
+                refList.get(i).set(null);
+            } else {
+                assertNull("listener index " + i, refList.get(i).get());
+            }
+
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            if (failOnTrue.get(i) == false) {
+                assertNull("listener index " + i, excList.get(i).get());
+            } else {
+                assertEquals("listener index " + i, "must not be true", excList.get(i).get().getMessage());
+            }
+        }
+
+        ActionListener.onResponse(listeners, Boolean.FALSE);
+        for (int i = 0; i < numListeners; i++) {
+            assertNull("listener index " + i, refList.get(i).get());
+        }
+
+        assertEquals(numListeners, exceptionCounter.get());
+        for (int i = 0; i < numListeners; i++) {
+            assertNotNull(excList.get(i).get());
+            assertEquals("listener index " + i, "must not be false " + i, excList.get(i).get().getMessage());
+        }
+    }
+
+    public void testOnFailure() {
+        final int numListeners = randomIntBetween(1, 20);
+        List<AtomicReference<Boolean>> refList = new ArrayList<>();
+        List<AtomicReference<Exception>> excList = new ArrayList<>();
+        List<ActionListener<Boolean>> listeners = new ArrayList<>();
+
+        final int listenerToFail = randomBoolean() ? -1 : randomIntBetween(0, numListeners-1);
+        for (int i = 0; i < numListeners; i++) {
+            AtomicReference<Boolean> reference = new AtomicReference<>();
+            AtomicReference<Exception> exReference = new AtomicReference<>();
+            refList.add(reference);
+            excList.add(exReference);
+            boolean fail = i == listenerToFail;
+            CheckedConsumer<Boolean, ? extends Exception> handler = (o) -> {
+                reference.set(o);
+            };
+            listeners.add(ActionListener.wrap(handler, (e) -> {
+                exReference.set(e);
+                if (fail) {
+                    throw new RuntimeException("double boom");
+                }
+            }));
+        }
+
+        try {
+            ActionListener.onFailure(listeners, new Exception("booom"));
+            assertTrue("unexpected succces listener to fail: " + listenerToFail, listenerToFail == -1);
+        } catch (RuntimeException ex) {
+            assertTrue("listener to fail: " + listenerToFail, listenerToFail >= 0);
+            assertNotNull(ex.getCause());
+            assertEquals("double boom", ex.getCause().getMessage());
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            assertNull("listener index " + i, refList.get(i).get());
+        }
+
+        for (int i = 0; i < numListeners; i++) {
+            assertEquals("listener index " + i, "booom", excList.get(i).get().getMessage());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index a0437f05c4c..7c9d5443464 100644
--- a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -25,23 +25,33 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestRequest.Method;
 import org.elasticsearch.rest.action.RestMainAction;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.startsWith;
 
 public class ActionModuleTests extends ESTestCase {
     public void testSetupActionsContainsKnownBuiltin() {
@@ -99,32 +109,70 @@ public class ActionModuleTests extends ESTestCase {
     }
 
     public void testSetupRestHandlerContainsKnownBuiltin() {
-        assertThat(ActionModule.setupRestHandlers(emptyList()), hasItem(RestMainAction.class));
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), null, emptyList(), null,
+                null);
+        actionModule.initRestHandlers(null);
+        // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
+        Exception e = expectThrows(IllegalArgumentException.class, () ->
+            actionModule.getRestController().registerHandler(Method.GET, "/", null));
+        assertThat(e.getMessage(), startsWith("Path [/] already has a value [" + RestMainAction.class.getName()));
     }
 
-    public void testPluginCantOverwriteBuiltinRestHandler() {
+    public void testPluginCantOverwriteBuiltinRestHandler() throws IOException {
         ActionPlugin dupsMainAction = new ActionPlugin() {
             @Override
-            public List<Class<? extends RestHandler>> getRestHandlers() {
-                return singletonList(RestMainAction.class);
+            public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                    IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                    IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
+                return singletonList(new RestMainAction(settings, restController));
             }
         };
-        Exception e = expectThrows(IllegalArgumentException.class, () -> ActionModule.setupRestHandlers(singletonList(dupsMainAction)));
-        assertEquals("can't register the same [rest_handler] more than once for [" + RestMainAction.class.getName() + "]", e.getMessage());
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ThreadPool threadPool = new TestThreadPool(getTestName());
+        try {
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
+                    singletonList(dupsMainAction), null, null);
+            Exception e = expectThrows(IllegalArgumentException.class, () -> actionModule.initRestHandlers(null));
+            assertThat(e.getMessage(), startsWith("Path [/] already has a value [" + RestMainAction.class.getName()));
+        } finally {
+            threadPool.shutdown();
+        }
     }
 
     public void testPluginCanRegisterRestHandler() {
         class FakeHandler implements RestHandler {
+            FakeHandler(RestController restController) {
+                restController.registerHandler(Method.GET, "/_dummy", this);
+            }
             @Override
             public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
             }
         }
         ActionPlugin registersFakeHandler = new ActionPlugin() {
             @Override
-            public List<Class<? extends RestHandler>> getRestHandlers() {
-                return singletonList(FakeHandler.class);
+            public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                    IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                    IndexNameExpressionResolver indexNameExpressionResolver, Supplier<DiscoveryNodes> nodesInCluster) {
+                return singletonList(new FakeHandler(restController));
             }
         };
-        assertThat(ActionModule.setupRestHandlers(singletonList(registersFakeHandler)), hasItem(FakeHandler.class));
+
+        SettingsModule settings = new SettingsModule(Settings.EMPTY);
+        ThreadPool threadPool = new TestThreadPool(getTestName());
+        try {
+            ActionModule actionModule = new ActionModule(false, settings.getSettings(), new IndexNameExpressionResolver(Settings.EMPTY),
+                    settings.getIndexScopedSettings(), settings.getClusterSettings(), settings.getSettingsFilter(), threadPool,
+                    singletonList(registersFakeHandler), null, null);
+            actionModule.initRestHandlers(null);
+            // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail
+            Exception e = expectThrows(IllegalArgumentException.class, () ->
+                actionModule.getRestController().registerHandler(Method.GET, "/_dummy", null));
+            assertThat(e.getMessage(), startsWith("Path [/_dummy] already has a value [" + FakeHandler.class.getName()));
+        } finally {
+            threadPool.shutdown();
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
index 017ba128d50..356d74acb13 100644
--- a/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/DocWriteResponseTests.java
@@ -30,13 +30,14 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.not;
 
 public class DocWriteResponseTests extends ESTestCase {
-    public void testGetLocation() {
+    public void testGetLocation() throws URISyntaxException {
         DocWriteResponse response =
             new DocWriteResponse(
                 new ShardId("index", "uuid", 0),
@@ -51,6 +52,35 @@ public class DocWriteResponseTests extends ESTestCase {
         assertEquals("/index/type/id?routing=test_routing", response.getLocation("test_routing"));
     }
 
+    public void testGetLocationNonAscii() throws URISyntaxException {
+        DocWriteResponse response =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "type",
+                "❤",
+                SequenceNumbersService.UNASSIGNED_SEQ_NO,
+                0,
+                Result.CREATED) {
+            };
+        assertEquals("/index/type/%E2%9D%A4", response.getLocation(null));
+        assertEquals("/index/type/%E2%9D%A4?routing=%C3%A4", response.getLocation("%C3%A4"));
+    }
+
+    public void testInvalidGetLocation() {
+        String invalidPath = "!^*$(@!^!#@";
+        DocWriteResponse invalid =
+            new DocWriteResponse(
+                new ShardId("index", "uuid", 0),
+                "type",
+                invalidPath,
+                SequenceNumbersService.UNASSIGNED_SEQ_NO,
+                0,
+                Result.CREATED) {
+            };
+        Throwable exception = expectThrows(URISyntaxException.class, () -> invalid.getLocation(null));
+        assertTrue(exception.getMessage().contains(invalidPath));
+    }
+
     /**
      * Tests that {@link DocWriteResponse#toXContent(XContentBuilder, ToXContent.Params)} doesn't include {@code forced_refresh} unless it
      * is true. We can't assert this in the yaml tests because "not found" is also "false" there....
@@ -69,18 +99,14 @@ public class DocWriteResponseTests extends ESTestCase {
         response.setShardInfo(new ShardInfo(1, 1));
         response.setForcedRefresh(false);
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.startObject();
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
                 assertThat(parser.map(), not(hasKey("forced_refresh")));
             }
         }
         response.setForcedRefresh(true);
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
-            builder.startObject();
             response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
                 assertThat(parser.map(), hasEntry("forced_refresh", true));
             }
diff --git a/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java b/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java
index 1bc895095ba..adab07f52a3 100644
--- a/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ExplainRequestTests.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.test.ESTestCase;
@@ -41,9 +40,8 @@ import java.util.Collections;
 import java.util.List;
 
 public class ExplainRequestTests extends ESTestCase {
+    private NamedWriteableRegistry namedWriteableRegistry;
 
-    protected NamedWriteableRegistry namedWriteableRegistry;
-    protected SearchRequestParsers searchRequestParsers;
     public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
@@ -52,10 +50,8 @@ public class ExplainRequestTests extends ESTestCase {
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        searchRequestParsers = searchModule.getSearchRequestParsers();
     }
 
-
     public void testSerialize() throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             ExplainRequest request = new ExplainRequest("index", "type", "id");
diff --git a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
index e4a6eef33eb..cca9d73b78f 100644
--- a/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -56,7 +56,6 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.delete.DeleteAction;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.explain.ExplainAction;
 import org.elasticsearch.action.explain.ExplainRequest;
@@ -64,7 +63,6 @@ import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -77,9 +75,11 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -201,10 +201,11 @@ public class IndicesRequestIT extends ESIntegTestCase {
     }
 
     public void testIndex() {
-        String[] indexShardActions = new String[]{IndexAction.NAME, IndexAction.NAME + "[p]", IndexAction.NAME + "[r]"};
+        String[] indexShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(indexShardActions);
 
-        IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id").source("field", "value");
+        IndexRequest indexRequest = new IndexRequest(randomIndexOrAlias(), "type", "id")
+            .source(Requests.INDEX_CONTENT_TYPE, "field", "value");
         internalCluster().coordOnlyNodeClient().index(indexRequest).actionGet();
 
         clearInterceptedActions();
@@ -212,7 +213,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
     }
 
     public void testDelete() {
-        String[] deleteShardActions = new String[]{DeleteAction.NAME, DeleteAction.NAME + "[p]", DeleteAction.NAME + "[r]"};
+        String[] deleteShardActions = new String[]{BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(deleteShardActions);
 
         DeleteRequest deleteRequest = new DeleteRequest(randomIndexOrAlias(), "type", "id");
@@ -224,12 +225,12 @@ public class IndicesRequestIT extends ESIntegTestCase {
 
     public void testUpdate() {
         //update action goes to the primary, index op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
         client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get();
-        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1");
+        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult());
 
@@ -239,11 +240,12 @@ public class IndicesRequestIT extends ESIntegTestCase {
 
     public void testUpdateUpsert() {
         //update action goes to the primary, index op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", IndexAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
-        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert("field", "value").doc("field1", "value1");
+        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").upsert(Requests.INDEX_CONTENT_TYPE, "field", "value")
+            .doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertEquals(DocWriteResponse.Result.CREATED, updateResponse.getResult());
 
@@ -253,7 +255,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
 
     public void testUpdateDelete() {
         //update action goes to the primary, delete op gets executed locally, then replicated
-        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", DeleteAction.NAME + "[r]"};
+        String[] updateShardActions = new String[]{UpdateAction.NAME + "[s]", BulkAction.NAME + "[s][p]", BulkAction.NAME + "[s][r]"};
         interceptTransportActions(updateShardActions);
 
         String indexOrAlias = randomIndexOrAlias();
@@ -276,7 +278,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
         int numIndexRequests = iterations(1, 10);
         for (int i = 0; i < numIndexRequests; i++) {
             String indexOrAlias = randomIndexOrAlias();
-            bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value"));
+            bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
             indices.add(indexOrAlias);
         }
         int numDeleteRequests = iterations(1, 10);
@@ -288,7 +290,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
         int numUpdateRequests = iterations(1, 10);
         for (int i = 0; i < numUpdateRequests; i++) {
             String indexOrAlias = randomIndexOrAlias();
-            bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"));
+            bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc(Requests.INDEX_CONTENT_TYPE, "field1", "value1"));
             indices.add(indexOrAlias);
         }
 
@@ -599,27 +601,6 @@ public class IndicesRequestIT extends ESIntegTestCase {
         assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
     }
 
-    public void testSearchDfsQueryAndFetch() throws Exception {
-        interceptTransportActions(SearchTransportService.QUERY_QUERY_FETCH_ACTION_NAME,
-                SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-
-        String[] randomIndicesOrAliases = randomIndicesOrAliases();
-        for (int i = 0; i < randomIndicesOrAliases.length; i++) {
-            client().prepareIndex(randomIndicesOrAliases[i], "type", "id-" + i).setSource("field", "value").get();
-        }
-        refresh();
-
-        SearchRequest searchRequest = new SearchRequest(randomIndicesOrAliases).searchType(SearchType.DFS_QUERY_AND_FETCH);
-        SearchResponse searchResponse = internalCluster().coordOnlyNodeClient().search(searchRequest).actionGet();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), greaterThan(0L));
-
-        clearInterceptedActions();
-        assertSameIndices(searchRequest, SearchTransportService.QUERY_QUERY_FETCH_ACTION_NAME);
-        //free context messages are not necessarily sent, but if they are, check their indices
-        assertSameIndicesOptionalRequests(searchRequest, SearchTransportService.FREE_CONTEXT_ACTION_NAME);
-    }
-
     private static void assertSameIndices(IndicesRequest originalRequest, String... actions) {
         assertSameIndices(originalRequest, false, actions);
     }
@@ -743,7 +724,8 @@ public class IndicesRequestIT extends ESIntegTestCase {
         public static class TestPlugin extends Plugin implements NetworkPlugin {
             public final InterceptingTransportService instance = new InterceptingTransportService();
             @Override
-            public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+            public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                       ThreadContext threadContext) {
                 return Collections.singletonList(instance);
             }
         }
@@ -754,6 +736,7 @@ public class IndicesRequestIT extends ESIntegTestCase {
 
         @Override
         public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                        boolean forceExecution,
                                                                                         TransportRequestHandler<T> actualHandler) {
             return new InterceptingRequestHandler<>(action, actualHandler);
         }
diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
index 60e4b4d2622..d0e8d997a1b 100644
--- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -38,7 +39,7 @@ public class ListenerActionIT extends ESIntegTestCase {
         IndexRequest request = new IndexRequest("test", "type", "1");
         if (randomBoolean()) {
             // set the source, without it, we will have a verification failure
-            request.source("field1", "value1");
+            request.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
         }
 
         client.index(request, new ActionListener<IndexResponse>() {
diff --git a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
index e8db648111c..0aa84ad8069 100644
--- a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
@@ -80,7 +80,7 @@ public class RejectionActionIT extends ESIntegTestCase {
                     });
         }
         latch.await();
-        assertThat(responses.size(), equalTo(numberOfAsyncOps));
+
 
         // validate all responses
         for (Object response : responses) {
@@ -102,5 +102,6 @@ public class RejectionActionIT extends ESIntegTestCase {
                 }
             }
         }
+        assertThat(responses.size(), equalTo(numberOfAsyncOps));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java b/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
index c1d18146a08..55bc37df18a 100644
--- a/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ShardValidateQueryRequestTests.java
@@ -31,7 +31,6 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.test.ESTestCase;
 
@@ -42,9 +41,8 @@ import java.util.Collections;
 import java.util.List;
 
 public class ShardValidateQueryRequestTests extends ESTestCase {
-
     protected NamedWriteableRegistry namedWriteableRegistry;
-    protected SearchRequestParsers searchRequestParsers;
+
     public void setUp() throws Exception {
         super.setUp();
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
@@ -53,10 +51,8 @@ public class ShardValidateQueryRequestTests extends ESTestCase {
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        searchRequestParsers = searchModule.getSearchRequestParsers();
     }
 
-
     public void testSerialize() throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest("indices");
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
new file mode 100644
index 00000000000..0f8f71a3af8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainActionTests.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.allocation;
+
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.gateway.TestGatewayAllocator;
+
+import java.util.Collections;
+import java.util.Locale;
+
+import static org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction.findShardToExplain;
+
+/**
+ * Tests for the {@link TransportClusterAllocationExplainAction} class.
+ */
+public class ClusterAllocationExplainActionTests extends ESTestCase {
+
+    private static final AllocationDeciders NOOP_DECIDERS = new AllocationDeciders(Settings.EMPTY, Collections.emptyList());
+
+    public void testInitializingOrRelocatingShardExplanation() throws Exception {
+        ShardRoutingState shardRoutingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.RELOCATING);
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), shardRoutingState);
+        ShardRouting shard = clusterState.getRoutingTable().index("idx").shard(0).primaryShard();
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, Collections.emptyList()),
+            clusterState.getRoutingNodes(), clusterState, null, System.nanoTime(), randomBoolean());
+        ClusterAllocationExplanation cae = TransportClusterAllocationExplainAction.explainShard(shard, allocation, null, randomBoolean(),
+            new TestGatewayAllocator(), new ShardsAllocator() {
+                @Override
+                public void allocate(RoutingAllocation allocation) {
+                    // no-op
+                }
+
+                @Override
+                public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                    if (shard.initializing() || shard.relocating()) {
+                        return ShardAllocationDecision.NOT_TAKEN;
+                    } else {
+                        throw new UnsupportedOperationException("cannot explain");
+                    }
+                }
+            });
+
+        assertEquals(shard.currentNodeId(), cae.getCurrentNode().getId());
+        assertFalse(cae.getShardAllocationDecision().isDecisionTaken());
+        assertFalse(cae.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
+        assertFalse(cae.getShardAllocationDecision().getMoveDecision().isDecisionTaken());
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        String explanation;
+        if (shardRoutingState == ShardRoutingState.RELOCATING) {
+            explanation = "the shard is in the process of relocating from node [] to node [], wait until " +
+                              "relocation has completed";
+        } else {
+            explanation = "the shard is in the process of initializing on node [], " +
+                              "wait until initialization has completed";
+        }
+        assertEquals("{\"index\":\"idx\",\"shard\":0,\"primary\":true,\"current_state\":\"" +
+                         shardRoutingState.toString().toLowerCase(Locale.ROOT) + "\",\"current_node\":" +
+                         "{\"id\":\"" + cae.getCurrentNode().getId() + "\",\"name\":\"" + cae.getCurrentNode().getName() +
+                         "\",\"transport_address\":\"" + cae.getCurrentNode().getAddress() +
+                         "\"},\"explanation\":\"" + explanation + "\"}", builder.string());
+    }
+
+    public void testFindAnyUnassignedShardToExplain() {
+        // find unassigned primary
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.UNASSIGNED);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest();
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
+
+        // find unassigned replica
+        clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED);
+        request = new ClusterAllocationExplainRequest();
+        shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0), shard);
+
+        // no unassigned shard to explain
+        final ClusterState allStartedClusterState = ClusterStateCreationUtils.state("idx", randomBoolean(),
+            ShardRoutingState.STARTED, ShardRoutingState.STARTED);
+        final ClusterAllocationExplainRequest anyUnassignedShardsRequest = new ClusterAllocationExplainRequest();
+        expectThrows(IllegalStateException.class, () ->
+            findShardToExplain(anyUnassignedShardsRequest, routingAllocation(allStartedClusterState)));
+    }
+
+    public void testFindPrimaryShardToExplain() {
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), randomFrom(ShardRoutingState.values()));
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, true, null);
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).primaryShard(), shard);
+    }
+
+    public void testFindAnyReplicaToExplain() {
+        // prefer unassigned replicas to started replicas
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED,
+            ShardRoutingState.STARTED, ShardRoutingState.UNASSIGNED);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        ShardRouting shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards()
+                         .stream().filter(ShardRouting::unassigned).findFirst().get(), shard);
+
+        // prefer started replicas to initializing/relocating replicas
+        clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED,
+            randomFrom(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING), ShardRoutingState.STARTED);
+        request = new ClusterAllocationExplainRequest("idx", 0, false, null);
+        shard = findShardToExplain(request, routingAllocation(clusterState));
+        assertEquals(clusterState.getRoutingTable().index("idx").shard(0).replicaShards()
+                         .stream().filter(ShardRouting::started).findFirst().get(), shard);
+    }
+
+    public void testFindShardAssignedToNode() {
+        // find shard with given node
+        final boolean primary = randomBoolean();
+        ShardRoutingState[] replicaStates = new ShardRoutingState[0];
+        if (primary == false) {
+            replicaStates = new ShardRoutingState[] { ShardRoutingState.STARTED };
+        }
+        ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(), ShardRoutingState.STARTED, replicaStates);
+        ShardRouting shardToExplain = primary ? clusterState.getRoutingTable().index("idx").shard(0).primaryShard() :
+                                          clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0);
+        ClusterAllocationExplainRequest request = new ClusterAllocationExplainRequest("idx", 0, primary, shardToExplain.currentNodeId());
+        RoutingAllocation allocation = routingAllocation(clusterState);
+        ShardRouting foundShard = findShardToExplain(request, allocation);
+        assertEquals(shardToExplain, foundShard);
+
+        // shard is not assigned to given node
+        String explainNode = null;
+        for (RoutingNode routingNode : clusterState.getRoutingNodes()) {
+            if (routingNode.nodeId().equals(shardToExplain.currentNodeId()) == false) {
+                explainNode = routingNode.nodeId();
+                break;
+            }
+        }
+        final ClusterAllocationExplainRequest failingRequest = new ClusterAllocationExplainRequest("idx", 0, primary, explainNode);
+        expectThrows(IllegalStateException.class, () -> findShardToExplain(failingRequest, allocation));
+    }
+
+    private static RoutingAllocation routingAllocation(ClusterState clusterState) {
+        return new RoutingAllocation(NOOP_DECIDERS, clusterState.getRoutingNodes(), clusterState, null, System.nanoTime(), randomBoolean());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 33f683b458c..ebbf4f74468 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -19,148 +19,1383 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus;
+import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
+import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.test.InternalTestCluster;
 
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.isOneOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Tests for the cluster allocation explanation
  */
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public final class ClusterAllocationExplainIT extends ESIntegTestCase {
-    @TestLogging("_root:DEBUG")
-    public void testDelayShards() throws Exception {
+
+    public void testUnassignedPrimaryWithExistingIndex() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary, 0 replicas");
+        createIndexAndIndexData(1, 0);
+
+        logger.info("--> stopping the node with the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName()));
+        ensureStableCluster(1);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertTrue(unassignedInfo.getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
+                       || unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 1);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertTrue(allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY
+                       || allocateDecision.getAllocationDecision() == AllocationDecision.AWAITING_INFO);
+        if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
+            assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be " +
+                             "found on the nodes in the cluster", allocateDecision.getExplanation());
+        } else {
+            assertEquals("cannot allocate because information about existing shard data is still being retrieved from some of the nodes",
+                allocateDecision.getExplanation());
+        }
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+
+        if (allocateDecision.getAllocationDecision() == AllocationDecision.NO_VALID_SHARD_COPY) {
+            assertEquals(1, allocateDecision.getNodeDecisions().size());
+
+            // verify JSON output
+            try (XContentParser parser = getParser(explanation)) {
+                verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+                parser.nextToken();
+                assertEquals("can_allocate", parser.currentName());
+                parser.nextToken();
+                assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
+                parser.nextToken();
+                assertEquals("allocate_explanation", parser.currentName());
+                parser.nextToken();
+                assertEquals("cannot allocate because a previous copy of the primary shard existed but can no longer be found " +
+                                 "on the nodes in the cluster", parser.text());
+                verifyStaleShardCopyNodeDecisions(parser, 1, Collections.emptySet());
+            }
+        }
+    }
+
+    public void testUnassignedReplicaDelayedAllocation() throws Exception {
         logger.info("--> starting 3 nodes");
         internalCluster().startNodes(3);
 
-        // Wait for all 3 nodes to be up
-        logger.info("--> waiting for 3 nodes to be up");
-        ensureStableCluster(3);
-
-        logger.info("--> creating 'test' index");
-        assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1m")
-                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 5)
-                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL).get());
-
-        logger.info("--> stopping a random node");
-        assertTrue(internalCluster().stopRandomDataNode());
-
-        logger.info("--> waiting for the master to remove the stopped node from the cluster state");
+        logger.info("--> creating an index with 1 primary, 1 replica");
+        createIndexAndIndexData(1, 1);
+        logger.info("--> stopping the node with the replica");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode().getName()));
         ensureStableCluster(2);
+        assertBusy(() ->
+            // wait till we have passed any pending shard data fetching
+            assertEquals(AllocationDecision.ALLOCATION_DELAYED, client().admin().cluster().prepareAllocationExplain()
+                .setIndex("idx").setShard(0).setPrimary(false).get().getExplanation()
+                .getShardAllocationDecision().getAllocateDecision().getAllocationDecision())
+        );
 
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get();
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertThat(cae.getShard().getIndexName(), equalTo("test"));
-        assertFalse(cae.isPrimary());
-        assertFalse(cae.isAssigned());
-        assertThat("expecting a remaining delay, got: " + cae.getRemainingDelayMillis(), cae.getRemainingDelayMillis(), greaterThan(0L));
-    }
+        logger.info("--> observing delayed allocation...");
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
 
-    public void testUnassignedShards() throws Exception {
-        logger.info("--> starting 3 nodes");
-        String noAttrNode = internalCluster().startNode();
-        String barAttrNode = internalCluster().startNode(Settings.builder().put("node.attr.bar", "baz"));
-        String fooBarAttrNode = internalCluster().startNode(Settings.builder()
-                .put("node.attr.foo", "bar")
-                .put("node.attr.bar", "baz"));
+        ClusterAllocationExplanation explanation = runExplain(false, includeYesDecisions, includeDiskInfo);
 
-        // Wait for all 3 nodes to be up
-        logger.info("--> waiting for 3 nodes to be up");
-        client().admin().cluster().health(Requests.clusterHealthRequest().waitForNodes("3")).actionGet();
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
 
-        client().admin().indices().prepareCreate("anywhere")
-                .setSettings(Settings.builder()
-                        .put("index.number_of_shards", 5)
-                        .put("index.number_of_replicas", 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL)  // wait on all shards
-                .get();
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
 
-        client().admin().indices().prepareCreate("only-baz")
-                .setSettings(Settings.builder()
-                        .put("index.routing.allocation.include.bar", "baz")
-                        .put("index.number_of_shards", 5)
-                        .put("index.number_of_replicas", 1))
-                .setWaitForActiveShards(ActiveShardCount.ALL)
-                .get();
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
 
-        client().admin().indices().prepareCreate("only-foo")
-                .setSettings(Settings.builder()
-                        .put("index.routing.allocation.include.foo", "bar")
-                        .put("index.number_of_shards", 1)
-                        .put("index.number_of_replicas", 1))
-                .get();
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
 
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain()
-                .setIndex("only-foo")
-                .setShard(0)
-                .setPrimary(false)
-                .get();
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertThat(cae.getShard().getIndexName(), equalTo("only-foo"));
-        assertFalse(cae.isPrimary());
-        assertFalse(cae.isAssigned());
-        assertFalse(cae.isStillFetchingShardData());
-        assertThat(UnassignedInfo.Reason.INDEX_CREATED, equalTo(cae.getUnassignedInfo().getReason()));
-        assertThat("expecting no remaining delay: " + cae.getRemainingDelayMillis(), cae.getRemainingDelayMillis(), equalTo(0L));
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
 
-        Map<DiscoveryNode, NodeExplanation> explanations = cae.getNodeExplanations();
-
-        Float barAttrWeight = -1f;
-        Float fooBarAttrWeight = -1f;
-        for (Map.Entry<DiscoveryNode, NodeExplanation> entry : explanations.entrySet()) {
-            DiscoveryNode node = entry.getKey();
-            String nodeName = node.getName();
-            NodeExplanation explanation = entry.getValue();
-            ClusterAllocationExplanation.FinalDecision finalDecision = explanation.getFinalDecision();
-            ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
-            Decision d = explanation.getDecision();
-            float weight = explanation.getWeight();
-            IndicesShardStoresResponse.StoreStatus storeStatus = explanation.getStoreStatus();
-
-            assertEquals(d.type(), Decision.Type.NO);
-            if (noAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("node does not match [index.routing.allocation.include] filters [foo:\"bar\"]"));
-                assertNull(storeStatus);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-            } else if (barAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("node does not match [index.routing.allocation.include] filters [foo:\"bar\"]"));
-                barAttrWeight = weight;
-                assertNull(storeStatus);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-            } else if (fooBarAttrNode.equals(nodeName)) {
-                assertThat(d.toString(), containsString("the shard cannot be allocated to the same node"));
-                fooBarAttrWeight = weight;
-                assertEquals(storeStatus.getAllocationStatus(),
-                        IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY);
-                assertEquals(ClusterAllocationExplanation.FinalDecision.NO, finalDecision);
-                assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-                assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                        explanation.getFinalExplanation());
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.ALLOCATION_DELAYED, allocateDecision.getAllocationDecision());
+        assertThat(allocateDecision.getExplanation(), startsWith("cannot allocate because the cluster is still waiting"));
+        assertThat(allocateDecision.getExplanation(), containsString(
+            "despite being allowed to allocate the shard to at least one other node"));
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(60000L, allocateDecision.getConfiguredDelayInMillis());
+        assertThat(allocateDecision.getRemainingDelayInMillis(), greaterThan(0L));
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        String primaryNodeName = primaryNodeName();
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            boolean nodeHoldingPrimary = result.getNode().getName().equals(primaryNodeName);
+            if (nodeHoldingPrimary) {
+                // shouldn't be able to allocate to the same node as the primary, the same shard decider should say no
+                assertEquals(AllocationDecision.NO, result.getNodeDecision());
+                assertThat(result.getShardStoreInfo().getMatchingBytes(), greaterThan(0L));
             } else {
-                fail("unexpected node with name: " + nodeName +
-                     ", I have: " + noAttrNode + ", " + barAttrNode + ", " + fooBarAttrNode);
+                assertEquals(AllocationDecision.YES, result.getNodeDecision());
+                assertNull(result.getShardStoreInfo());
+            }
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                // if we are not including YES decisions, then the node holding the primary should have 1 NO decision,
+                // the other node should have zero NO decisions
+                assertEquals(nodeHoldingPrimary ? 1 : 0, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("same_shard") && nodeHoldingPrimary) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertThat(d.getExplanation(), startsWith(
+                        "the shard cannot be allocated to the same node on which a copy of the shard already exists"));
+                } else {
+                    assertEquals(Decision.Type.YES, d.type());
+                    assertNotNull(d.getExplanation());
+                }
             }
         }
-        assertFalse(barAttrWeight == fooBarAttrWeight);
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.ALLOCATION_DELAYED.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            assertThat(parser.text(), startsWith("cannot allocate because the cluster is still waiting"));
+            parser.nextToken();
+            assertEquals("configured_delay_in_millis", parser.currentName());
+            parser.nextToken();
+            assertEquals(60000L, parser.longValue());
+            parser.nextToken();
+            assertEquals("remaining_delay_in_millis", parser.currentName());
+            parser.nextToken();
+            assertThat(parser.longValue(), greaterThan(0L));
+            Map<String, AllocationDecision> nodes = new HashMap<>();
+            nodes.put(primaryNodeName, AllocationDecision.NO);
+            String[] currentNodes = internalCluster().getNodeNames();
+            nodes.put(currentNodes[0].equals(primaryNodeName) ? currentNodes[1] : currentNodes[0], AllocationDecision.YES);
+            verifyNodeDecisions(parser, nodes, includeYesDecisions, true);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
     }
+
+    public void testUnassignedReplicaWithPriorCopy() throws Exception {
+        logger.info("--> starting 3 nodes");
+        List<String> nodes = internalCluster().startNodes(3);
+
+        logger.info("--> creating an index with 1 primary and 1 replica");
+        createIndexAndIndexData(1, 1);
+        String primaryNodeName = primaryNodeName();
+        nodes.remove(primaryNodeName);
+
+        logger.info("--> shutting down all nodes except the one that holds the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes.get(1)));
+        ensureStableCluster(1);
+
+        logger.info("--> setting allocation filtering to only allow allocation on the currently running node");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", primaryNodeName)).get();
+
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startNode(Settings.builder().put("node.name", nodes.get(0)).build());
+        internalCluster().startNode(Settings.builder().put("node.name", nodes.get(1)).build());
+        ensureStableCluster(3);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(false, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        AllocationDecision decisionToAllocate = allocateDecision.getAllocationDecision();
+        assertTrue(decisionToAllocate == AllocationDecision.AWAITING_INFO || decisionToAllocate == AllocationDecision.NO);
+        if (decisionToAllocate == AllocationDecision.AWAITING_INFO) {
+            assertEquals("cannot allocate because information about existing shard data is still being retrieved from some of the nodes",
+                allocateDecision.getExplanation());
+        } else {
+            assertEquals("cannot allocate because allocation is not permitted to any of the nodes", allocateDecision.getExplanation());
+        }
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+        assertEquals(3, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            boolean nodeHoldingPrimary = result.getNode().getName().equals(primaryNodeName);
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("same_shard") && nodeHoldingPrimary) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertThat(d.getExplanation(), startsWith(
+                        "the shard cannot be allocated to the same node on which a copy of the shard already exists"));
+                } else if (d.label().equals("filter") && nodeHoldingPrimary == false) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertEquals("node does not match index setting [index.routing.allocation.include] " +
+                                     "filters [_name:\"" + primaryNodeName + "\"]", d.getExplanation());
+                } else {
+                    assertEquals(Decision.Type.YES, d.type());
+                    assertNotNull(d.getExplanation());
+                }
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            String allocationDecision = parser.text();
+            assertTrue(allocationDecision.equals(AllocationDecision.NO.toString())
+                           || allocationDecision.equals(AllocationDecision.AWAITING_INFO.toString()));
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            if (allocationDecision.equals("awaiting_info")) {
+                assertEquals("cannot allocate because information about existing shard data is still being retrieved " +
+                                 "from some of the nodes", parser.text());
+            } else {
+                assertEquals("cannot allocate because allocation is not permitted to any of the nodes", parser.text());
+            }
+            Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+            for (String nodeName : internalCluster().getNodeNames()) {
+                nodeDecisions.put(nodeName, AllocationDecision.NO);
+            }
+            verifyNodeDecisions(parser, nodeDecisions, includeYesDecisions, true);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAllocationFilteringOnIndexCreation() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary, 0 replicas, with allocation filtering so the primary can't be assigned");
+        createIndexAndIndexData(1, 0, Settings.builder().put("index.routing.allocation.include._name", "non_existent_node").build(),
+            ActiveShardCount.NONE);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertNotEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNull(currentNode);
+
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+        assertEquals(Reason.INDEX_CREATED, unassignedInfo.getReason());
+        assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.getLastAllocationStatus());
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision objects
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, allocateDecision.getAllocationDecision());
+        assertEquals("cannot allocate because allocation is not permitted to any of the nodes", allocateDecision.getExplanation());
+        assertNull(allocateDecision.getAllocationId());
+        assertNull(allocateDecision.getTargetNode());
+        assertEquals(0L, allocateDecision.getConfiguredDelayInMillis());
+        assertEquals(0L, allocateDecision.getRemainingDelayInMillis());
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : allocateDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.label().equals("filter")) {
+                    assertEquals(Decision.Type.NO, d.type());
+                    assertEquals("node does not match index setting [index.routing.allocation.include] filters " +
+                                     "[_name:\"non_existent_node\"]", d.getExplanation());
+                }
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            String allocationDecision = parser.text();
+            assertTrue(allocationDecision.equals(AllocationDecision.NO.toString())
+                           || allocationDecision.equals(AllocationDecision.AWAITING_INFO.toString()));
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            if (allocationDecision.equals("awaiting_info")) {
+                assertEquals("cannot allocate because information about existing shard data is still being retrieved " +
+                                 "from some of the nodes", parser.text());
+            } else {
+                assertEquals("cannot allocate because allocation is not permitted to any of the nodes", parser.text());
+            }
+            Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+            for (String nodeName : internalCluster().getNodeNames()) {
+                nodeDecisions.put(nodeName, AllocationDecision.NO);
+            }
+            verifyNodeDecisions(parser, nodeDecisions, includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAllocationFilteringPreventsShardMove() throws Exception {
+        logger.info("--> starting 2 nodes");
+        internalCluster().startNodes(2);
+
+        logger.info("--> creating an index with 1 primary and 0 replicas");
+        createIndexAndIndexData(1, 0);
+
+        logger.info("--> setting up allocation filtering to prevent allocation to both nodes");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", "non_existent_node")).get();
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot move shard to another node, even though it is not allowed to remain on its current node",
+            moveDecision.getExplanation());
+        assertFalse(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNull(moveDecision.getClusterRebalanceDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(0, moveDecision.getCurrentNodeRanking());
+        // verifying can remain decision object
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getCanRemainDecision().type());
+        for (Decision d : moveDecision.getCanRemainDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.NO, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+        } else {
+            assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_remain_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_move_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("move_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot move shard to another node, even though it is not allowed to remain on its current node", parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testRebalancingNotAllowed() throws Exception {
+        logger.info("--> starting a single node");
+        internalCluster().startNode();
+        ensureStableCluster(1);
+
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> disabling rebalancing on the index");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.rebalance.enable", "none")).get();
+
+        logger.info("--> starting another node, with rebalancing disabled, it should get no shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("rebalancing is not allowed, even though there is at least one node on which the shard can be allocated",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(2, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            if (d.label().equals("enable")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("no rebalancing is allowed due to index setting [index.routing.rebalance.enable=none]",
+                    d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.YES, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(0));
+        } else {
+            assertEquals(0, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("rebalancing is not allowed, even though there is at least one node on which the shard can be allocated",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.YES, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testWorseBalance() throws Exception {
+        logger.info("--> starting a single node");
+        internalCluster().startNode();
+        ensureStableCluster(1);
+
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> setting balancing threshold really high, so it won't be met");
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+            Settings.builder().put("cluster.routing.allocation.balance.threshold", 1000.0f)).get();
+
+        logger.info("--> starting another node, with the rebalance threshold so high, it should not get any shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertTrue(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(1, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.YES, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.WORSE_BALANCE, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(0));
+        } else {
+            assertEquals(0, result.getCanAllocateDecision().getDecisions().size());
+        }
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.WORSE_BALANCE, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testBetterBalanceButCannotAllocate() throws Exception {
+        logger.info("--> starting a single node");
+        String firstNode = internalCluster().startNode();
+        ensureStableCluster(1);
+
+        logger.info("--> creating an index with 5 shards, all allocated to the single node");
+        createIndexAndIndexData(5, 0);
+
+        logger.info("--> setting up allocation filtering to only allow allocation to the current node");
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", firstNode)).get();
+
+        logger.info("--> starting another node, with filtering not allowing allocation to the new node, it should not get any shards");
+        internalCluster().startNode();
+        ensureStableCluster(2);
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
+
+        // verify decision object
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+            moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertTrue(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        assertEquals(2, moveDecision.getCurrentNodeRanking());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.YES, moveDecision.getClusterRebalanceDecision().type());
+        for (Decision d : moveDecision.getClusterRebalanceDecision().getDecisions()) {
+            assertEquals(Decision.Type.YES, d.type());
+            assertNotNull(d.getExplanation());
+        }
+        // verify node decisions
+        assertEquals(1, moveDecision.getNodeDecisions().size());
+        NodeAllocationResult result = moveDecision.getNodeDecisions().get(0);
+        assertNotNull(result.getNode());
+        assertEquals(1, result.getWeightRanking());
+        assertEquals(AllocationDecision.NO, result.getNodeDecision());
+        if (includeYesDecisions) {
+            assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+        } else {
+            assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+        }
+        String primaryNodeName = primaryNodeName();
+        for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+            if (d.label().equals("filter")) {
+                assertEquals(Decision.Type.NO, d.type());
+                assertEquals("node does not match index setting [index.routing.allocation.include] filters [_name:\"" +
+                                 primaryNodeName + "\"]", d.getExplanation());
+            } else {
+                assertEquals(Decision.Type.YES, d.type());
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+                parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, true), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testAssignedReplicaOnSpecificNode() throws Exception {
+        logger.info("--> starting 3 nodes");
+        List<String> nodes = internalCluster().startNodes(3);
+
+        logger.info("--> creating an index with 1 primary and 2 replicas");
+        String excludedNode = nodes.get(randomIntBetween(0, 2));
+        createIndexAndIndexData(1, 2, Settings.builder().put("index.routing.allocation.exclude._name", excludedNode).build(),
+            ActiveShardCount.from(2));
+
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(false, replicaNode().getId(), includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        ClusterInfo clusterInfo = explanation.getClusterInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertFalse(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.STARTED, shardRoutingState);
+        assertNotNull(currentNode);
+        assertEquals(replicaNode().getName(), currentNode.getName());
+
+        // verify unassigned info
+        assertNull(unassignedInfo);
+
+        // verify cluster info
+        verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
+
+        // verify decision objects
+        assertFalse(allocateDecision.isDecisionTaken());
+        assertTrue(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO, moveDecision.getAllocationDecision());
+        assertEquals("rebalancing is not allowed", moveDecision.getExplanation());
+        assertTrue(moveDecision.canRemain());
+        assertFalse(moveDecision.forceMove());
+        assertFalse(moveDecision.canRebalanceCluster());
+        assertNotNull(moveDecision.getCanRemainDecision());
+        assertNull(moveDecision.getTargetNode());
+        // verifying cluster rebalance decision object
+        assertNotNull(moveDecision.getClusterRebalanceDecision());
+        assertEquals(Decision.Type.NO, moveDecision.getClusterRebalanceDecision().type());
+        // verify node decisions
+        assertEquals(2, moveDecision.getNodeDecisions().size());
+        for (NodeAllocationResult result : moveDecision.getNodeDecisions()) {
+            assertNotNull(result.getNode());
+            assertEquals(1, result.getWeightRanking());
+            assertEquals(AllocationDecision.NO, result.getNodeDecision());
+            if (includeYesDecisions) {
+                assertThat(result.getCanAllocateDecision().getDecisions().size(), greaterThan(1));
+            } else {
+                assertEquals(1, result.getCanAllocateDecision().getDecisions().size());
+            }
+            for (Decision d : result.getCanAllocateDecision().getDecisions()) {
+                if (d.type() == Decision.Type.NO) {
+                    assertThat(d.label(), isOneOf("filter", "same_shard"));
+                }
+                assertNotNull(d.getExplanation());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, false, includeDiskInfo, ShardRoutingState.STARTED);
+            parser.nextToken();
+            assertEquals("can_remain_on_current_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.YES.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("can_rebalance_cluster_decisions", parser.currentName());
+            verifyDeciders(parser, AllocationDecision.NO);
+            parser.nextToken();
+            assertEquals("can_rebalance_to_other_node", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("rebalance_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("rebalancing is not allowed", parser.text());
+            verifyNodeDecisions(parser, allNodeDecisions(AllocationDecision.NO, false), includeYesDecisions, false);
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+    }
+
+    public void testCannotAllocateStaleReplicaExplanation() throws Exception {
+        logger.info("--> starting 3 nodes");
+        final String masterNode = internalCluster().startNode();
+        // start replica node first, so it's path will be used first when we start a node after
+        // stopping all of them at end of test.
+        final String replicaNode = internalCluster().startNode();
+        final String primaryNode = internalCluster().startNode();
+
+        logger.info("--> creating an index with 1 primary and 1 replica");
+        createIndexAndIndexData(1, 1,
+            Settings.builder()
+                .put("index.routing.allocation.include._name", primaryNode)
+                .put("index.routing.allocation.exclude._name", masterNode)
+                .build(),
+            ActiveShardCount.ONE);
+
+        client().admin().indices().prepareUpdateSettings("idx").setSettings(
+            Settings.builder().put("index.routing.allocation.include._name", (String) null)).get();
+        ensureGreen();
+
+        assertThat(replicaNode().getName(), equalTo(replicaNode));
+        assertThat(primaryNodeName(), equalTo(primaryNode));
+
+        logger.info("--> stop node with the replica shard");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNode));
+
+        logger.info("--> index more data, now the replica is stale");
+        indexData();
+
+        logger.info("--> stop the node with the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNode));
+
+        logger.info("--> restart the node with the stale replica");
+        String restartedNode = internalCluster().startDataOnlyNode();
+        ensureClusterSizeConsistency(); // wait for the master to finish processing join.
+
+        // wait until the system has fetched shard data and we know there is no valid shard copy
+        assertBusy(() -> {
+            ClusterAllocationExplanation explanation = client().admin().cluster().prepareAllocationExplain()
+                .setIndex("idx").setShard(0).setPrimary(true).get().getExplanation();
+            assertTrue(explanation.getShardAllocationDecision().getAllocateDecision().isDecisionTaken());
+            assertEquals(AllocationDecision.NO_VALID_SHARD_COPY,
+                explanation.getShardAllocationDecision().getAllocateDecision().getAllocationDecision());
+        });
+        boolean includeYesDecisions = randomBoolean();
+        boolean includeDiskInfo = randomBoolean();
+        ClusterAllocationExplanation explanation = runExplain(true, includeYesDecisions, includeDiskInfo);
+
+        ShardId shardId = explanation.getShard();
+        boolean isPrimary = explanation.isPrimary();
+        ShardRoutingState shardRoutingState = explanation.getShardState();
+        DiscoveryNode currentNode = explanation.getCurrentNode();
+        UnassignedInfo unassignedInfo = explanation.getUnassignedInfo();
+        AllocateUnassignedDecision allocateDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+        MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+
+        // verify shard info
+        assertEquals("idx", shardId.getIndexName());
+        assertEquals(0, shardId.getId());
+        assertTrue(isPrimary);
+
+        // verify current node info
+        assertEquals(ShardRoutingState.UNASSIGNED, shardRoutingState);
+        assertNull(currentNode);
+        // verify unassigned info
+        assertNotNull(unassignedInfo);
+
+        // verify decision object
+        assertTrue(allocateDecision.isDecisionTaken());
+        assertFalse(moveDecision.isDecisionTaken());
+        assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, allocateDecision.getAllocationDecision());
+        assertEquals(2, allocateDecision.getNodeDecisions().size());
+        for (NodeAllocationResult nodeAllocationResult : allocateDecision.getNodeDecisions()) {
+            if (nodeAllocationResult.getNode().getName().equals(restartedNode)) {
+                assertNotNull(nodeAllocationResult.getShardStoreInfo());
+                assertNotNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
+                assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
+            } else {
+                assertNotNull(nodeAllocationResult.getShardStoreInfo());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getAllocationId());
+                assertFalse(nodeAllocationResult.getShardStoreInfo().isInSync());
+                assertNull(nodeAllocationResult.getShardStoreInfo().getStoreException());
+            }
+        }
+
+        // verify JSON output
+        try (XContentParser parser = getParser(explanation)) {
+            verifyShardInfo(parser, true, includeDiskInfo, ShardRoutingState.UNASSIGNED);
+            parser.nextToken();
+            assertEquals("can_allocate", parser.currentName());
+            parser.nextToken();
+            assertEquals(AllocationDecision.NO_VALID_SHARD_COPY.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("allocate_explanation", parser.currentName());
+            parser.nextToken();
+            assertEquals("cannot allocate because all found copies of the shard are either stale or corrupt", parser.text());
+            verifyStaleShardCopyNodeDecisions(parser, 2, Collections.singleton(restartedNode));
+        }
+    }
+
+    private void verifyClusterInfo(ClusterInfo clusterInfo, boolean includeDiskInfo, int numNodes) {
+        if (includeDiskInfo) {
+            assertThat(clusterInfo.getNodeMostAvailableDiskUsages().size(), greaterThanOrEqualTo(0));
+            assertThat(clusterInfo.getNodeLeastAvailableDiskUsages().size(), greaterThanOrEqualTo(0));
+            assertThat(clusterInfo.getNodeMostAvailableDiskUsages().size(), lessThanOrEqualTo(numNodes));
+            assertThat(clusterInfo.getNodeLeastAvailableDiskUsages().size(), lessThanOrEqualTo(numNodes));
+        } else {
+            assertNull(clusterInfo);
+        }
+    }
+
+    private ClusterAllocationExplanation runExplain(boolean primary, boolean includeYesDecisions, boolean includeDiskInfo)
+        throws Exception {
+
+        return runExplain(primary, null, includeYesDecisions, includeDiskInfo);
+    }
+
+    private ClusterAllocationExplanation runExplain(boolean primary, String nodeId, boolean includeYesDecisions, boolean includeDiskInfo)
+        throws Exception {
+
+        ClusterAllocationExplanation explanation = client().admin().cluster().prepareAllocationExplain()
+            .setIndex("idx").setShard(0).setPrimary(primary)
+            .setIncludeYesDecisions(includeYesDecisions)
+            .setIncludeDiskInfo(includeDiskInfo)
+            .setCurrentNode(nodeId)
+            .get().getExplanation();
+        if (logger.isDebugEnabled()) {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            builder.prettyPrint();
+            builder.humanReadable(true);
+            logger.debug("--> explain json output: \n{}", explanation.toXContent(builder, ToXContent.EMPTY_PARAMS).string());
+        }
+        return explanation;
+    }
+
+    private void createIndexAndIndexData(int numPrimaries, int numReplicas) {
+        createIndexAndIndexData(numPrimaries, numReplicas, Settings.EMPTY, ActiveShardCount.ALL);
+    }
+
+    private void createIndexAndIndexData(int numPrimaries, int numReplicas, Settings settings, ActiveShardCount activeShardCount) {
+        client().admin().indices().prepareCreate("idx")
+            .setSettings(Settings.builder()
+                             .put("index.number_of_shards", numPrimaries)
+                             .put("index.number_of_replicas", numReplicas)
+                             .put(settings))
+            .setWaitForActiveShards(activeShardCount)
+            .get();
+        if (activeShardCount != ActiveShardCount.NONE) {
+            indexData();
+        }
+    }
+
+    private void indexData() {
+        for (int i = 0; i < 10; i++) {
+            index("idx", "t", Integer.toString(i), Collections.singletonMap("f1", Integer.toString(i)));
+        }
+        flushAndRefresh("idx");
+    }
+
+    private String primaryNodeName() {
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        String nodeId = clusterState.getRoutingTable().index("idx").shard(0).primaryShard().currentNodeId();
+        return clusterState.getRoutingNodes().node(nodeId).node().getName();
+    }
+
+    private DiscoveryNode replicaNode() {
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        String nodeId = clusterState.getRoutingTable().index("idx").shard(0).replicaShards().get(0).currentNodeId();
+        return clusterState.getRoutingNodes().node(nodeId).node();
+    }
+
+    private XContentParser getParser(ClusterAllocationExplanation explanation) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        return createParser(explanation.toXContent(builder, ToXContent.EMPTY_PARAMS));
+    }
+
+    private void verifyShardInfo(XContentParser parser, boolean primary, boolean includeDiskInfo, ShardRoutingState state)
+        throws IOException {
+
+        parser.nextToken();
+        assertEquals(Token.START_OBJECT, parser.currentToken());
+        parser.nextToken();
+        assertEquals("index", parser.currentName());
+        parser.nextToken();
+        assertEquals("idx", parser.text());
+        parser.nextToken();
+        assertEquals("shard", parser.currentName());
+        parser.nextToken();
+        assertEquals(0, parser.intValue());
+        parser.nextToken();
+        assertEquals("primary", parser.currentName());
+        parser.nextToken();
+        assertEquals(primary, parser.booleanValue());
+        parser.nextToken();
+        assertEquals("current_state", parser.currentName());
+        parser.nextToken();
+        assertEquals(state.toString().toLowerCase(Locale.ROOT), parser.text());
+        if (state == ShardRoutingState.UNASSIGNED) {
+            parser.nextToken();
+            assertEquals("unassigned_info", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            Token token;
+            while ((token = parser.nextToken()) != Token.END_OBJECT) { // until we reach end of unassigned_info
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    assertNotEquals("delayed", parser.currentName()); // we should never display "delayed" from unassigned info
+                    if (parser.currentName().equals("last_allocation_status")) {
+                        parser.nextToken();
+                        assertThat(parser.text(), isOneOf(AllocationDecision.NO.toString(),
+                            AllocationDecision.NO_VALID_SHARD_COPY.toString(),
+                            AllocationDecision.AWAITING_INFO.toString(),
+                            AllocationDecision.NO_ATTEMPT.toString()));
+                    }
+                }
+            }
+        } else {
+            assertEquals(ShardRoutingState.STARTED, state);
+            parser.nextToken();
+            assertEquals("current_node", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            Token token;
+            while ((token = parser.nextToken()) != Token.END_OBJECT) { // until we reach end of current_node
+                if (token == Token.FIELD_NAME) {
+                    assertTrue(parser.currentName().equals("id")
+                                   || parser.currentName().equals("name")
+                                   || parser.currentName().equals("transport_address")
+                                   || parser.currentName().equals("weight_ranking"));
+                } else {
+                    assertTrue(token.isValue());
+                    assertNotNull(parser.text());
+                }
+            }
+        }
+        if (includeDiskInfo) {
+            // disk info is included, just verify the object is there
+            parser.nextToken();
+            assertEquals("cluster_info", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            int numObjects = 1;
+            while (numObjects > 0) {
+                Token token = parser.nextToken();
+                if (token == Token.START_OBJECT) {
+                    ++numObjects;
+                } else if (token == Token.END_OBJECT) {
+                    --numObjects;
+                }
+            }
+        }
+    }
+
+    private void verifyStaleShardCopyNodeDecisions(XContentParser parser, int numNodes, Set<String> foundStores) throws IOException {
+        parser.nextToken();
+        assertEquals("node_allocation_decisions", parser.currentName());
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        for (int i = 0; i < numNodes; i++) {
+            String nodeName = verifyNodeDecisionPrologue(parser);
+            assertEquals(AllocationDecision.NO.toString(), parser.text());
+            parser.nextToken();
+            assertEquals("store", parser.currentName());
+            assertEquals(Token.START_OBJECT, parser.nextToken());
+            parser.nextToken();
+            if (foundStores.contains(nodeName)) {
+                // shard data was found on the node, but it is stale
+                assertEquals("in_sync", parser.currentName());
+                parser.nextToken();
+                assertFalse(parser.booleanValue());
+                parser.nextToken();
+                assertEquals("allocation_id", parser.currentName());
+                parser.nextToken();
+                assertNotNull(parser.text());
+            } else {
+                // no shard data was found on the node
+                assertEquals("found", parser.currentName());
+                parser.nextToken();
+                assertFalse(parser.booleanValue());
+            }
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+            parser.nextToken();
+            assertEquals(Token.END_OBJECT, parser.currentToken());
+        }
+        assertEquals(Token.END_ARRAY, parser.nextToken());
+    }
+
+    private void verifyNodeDecisions(XContentParser parser, Map<String, AllocationDecision> expectedNodeDecisions,
+                                     boolean includeYesDecisions, boolean reuseStore) throws IOException {
+        parser.nextToken();
+        assertEquals("node_allocation_decisions", parser.currentName());
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        boolean encounteredNo = false;
+        final int numNodes = expectedNodeDecisions.size();
+        for (int i = 0; i < numNodes; i++) {
+            String nodeName = verifyNodeDecisionPrologue(parser);
+            AllocationDecision allocationDecision = expectedNodeDecisions.get(nodeName);
+            assertEquals(allocationDecision.toString(), parser.text());
+            if (allocationDecision != AllocationDecision.YES) {
+                encounteredNo = true;
+            } else {
+                assertFalse("encountered a YES node decision after a NO node decision - sort order is wrong", encounteredNo);
+            }
+            parser.nextToken();
+            if ("store".equals(parser.currentName())) {
+                assertTrue("store info should not be present", reuseStore);
+                assertEquals(Token.START_OBJECT, parser.nextToken());
+                parser.nextToken();
+                assertEquals("matching_size_in_bytes", parser.currentName());
+                parser.nextToken();
+                assertThat(parser.longValue(), greaterThan(0L));
+                assertEquals(Token.END_OBJECT, parser.nextToken());
+                parser.nextToken();
+            }
+            if (reuseStore == false) {
+                assertEquals("weight_ranking", parser.currentName());
+                parser.nextToken();
+                assertThat(parser.intValue(), greaterThan(0));
+                parser.nextToken();
+            }
+            if (allocationDecision == AllocationDecision.NO || allocationDecision == AllocationDecision.THROTTLED || includeYesDecisions) {
+                assertEquals("deciders", parser.currentName());
+                boolean atLeastOneMatchingDecisionFound = verifyDeciders(parser, allocationDecision);
+                parser.nextToken();
+                if (allocationDecision == AllocationDecision.NO || allocationDecision == AllocationDecision.THROTTLED) {
+                    assertTrue("decision was " + allocationDecision + " but found no node's with that decision",
+                        atLeastOneMatchingDecisionFound);
+                }
+            }
+            assertEquals(Token.END_OBJECT, parser.currentToken());
+        }
+        assertEquals(Token.END_ARRAY, parser.nextToken());
+    }
+
+    private String verifyNodeDecisionPrologue(XContentParser parser) throws IOException {
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        parser.nextToken();
+        assertEquals("node_id", parser.currentName());
+        parser.nextToken();
+        assertNotNull(parser.text());
+        parser.nextToken();
+        assertEquals("node_name", parser.currentName());
+        parser.nextToken();
+        String nodeName = parser.text();
+        assertNotNull(nodeName);
+        parser.nextToken();
+        assertEquals("transport_address", parser.currentName());
+        parser.nextToken();
+        assertNotNull(parser.text());
+        parser.nextToken();
+        assertEquals("node_decision", parser.currentName());
+        parser.nextToken();
+        return nodeName;
+    }
+
+    private boolean verifyDeciders(XContentParser parser, AllocationDecision allocationDecision) throws IOException {
+        assertEquals(Token.START_ARRAY, parser.nextToken());
+        boolean atLeastOneMatchingDecisionFound = false;
+        while (parser.nextToken() != Token.END_ARRAY) {
+            assertEquals(Token.START_OBJECT, parser.currentToken());
+            parser.nextToken();
+            assertEquals("decider", parser.currentName());
+            parser.nextToken();
+            assertNotNull(parser.text());
+            parser.nextToken();
+            assertEquals("decision", parser.currentName());
+            parser.nextToken();
+            String decisionText = parser.text();
+            if ((allocationDecision == AllocationDecision.NO && decisionText.equals("NO")
+                     || (allocationDecision == AllocationDecision.THROTTLED && decisionText.equals("THROTTLE")))) {
+                atLeastOneMatchingDecisionFound = true;
+            }
+            assertNotNull(decisionText);
+            parser.nextToken();
+            assertEquals("explanation", parser.currentName());
+            parser.nextToken();
+            assertNotNull(parser.text());
+            assertEquals(Token.END_OBJECT, parser.nextToken());
+        }
+        return atLeastOneMatchingDecisionFound;
+    }
+
+    private Map<String, AllocationDecision> allNodeDecisions(AllocationDecision allocationDecision, boolean removePrimary) {
+        Map<String, AllocationDecision> nodeDecisions = new HashMap<>();
+        Set<String> allNodes = Sets.newHashSet(internalCluster().getNodeNames());
+        allNodes.remove(removePrimary ? primaryNodeName() : replicaNode().getName());
+        for (String nodeName : allNodes) {
+            nodeDecisions.put(nodeName, allocationDecision);
+        }
+        return nodeDecisions;
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
index 926de8b253e..4067eb1b1ed 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequestTests.java
@@ -26,7 +26,8 @@ public class ClusterAllocationExplainRequestTests extends ESTestCase {
 
     public void testSerialization() throws Exception {
         ClusterAllocationExplainRequest request =
-                new ClusterAllocationExplainRequest(randomAsciiOfLength(4), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean());
+                new ClusterAllocationExplainRequest(randomAsciiOfLength(4), randomIntBetween(0, Integer.MAX_VALUE), randomBoolean(),
+                                                       randomBoolean() ? randomAsciiOfLength(5) : null);
         request.includeYesDecisions(randomBoolean());
         request.includeDiskInfo(randomBoolean());
         BytesStreamOutput output = new BytesStreamOutput();
@@ -39,6 +40,7 @@ public class ClusterAllocationExplainRequestTests extends ESTestCase {
         assertEquals(request.isPrimary(), actual.isPrimary());
         assertEquals(request.includeYesDecisions(), actual.includeYesDecisions());
         assertEquals(request.includeDiskInfo(), actual.includeDiskInfo());
+        assertEquals(request.getCurrentNode(), actual.getCurrentNode());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
deleted file mode 100644
index 329cc3805ab..00000000000
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.allocation;
-
-import org.elasticsearch.cluster.routing.allocation.decider.Decision;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-
-/**
- * Tests for the cluster allocation explanation
- */
-public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
-
-    public void testShardExplain() throws Exception {
-        client().admin().indices().prepareCreate("test")
-                .setSettings("index.number_of_shards", 1, "index.number_of_replicas", 1).get();
-        ClusterAllocationExplainResponse resp = client().admin().cluster().prepareAllocationExplain()
-                .setIndex("test").setShard(0).setPrimary(false).get();
-
-        ClusterAllocationExplanation cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(false, cae.isPrimary());
-        assertNull(cae.getAssignedNodeId());
-        assertFalse(cae.isStillFetchingShardData());
-        assertNotNull(cae.getUnassignedInfo());
-        NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next();
-        ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision();
-        ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
-        Decision d = explanation.getDecision();
-        assertNotNull("should have a decision", d);
-        assertEquals(Decision.Type.NO, d.type());
-        assertEquals(ClusterAllocationExplanation.FinalDecision.NO, fd);
-        assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-        assertTrue(d.toString(), d.toString().contains("NO(the shard cannot be allocated to the same node"));
-        assertTrue(d instanceof Decision.Multi);
-        Decision.Multi md = (Decision.Multi) d;
-        Decision ssd = md.getDecisions().get(0);
-        assertEquals(Decision.Type.NO, ssd.type());
-        assertTrue(ssd.toString(), ssd.toString().contains("NO(the shard cannot be allocated to the same node"));
-        Float weight = explanation.getWeight();
-        assertNotNull("should have a weight", weight);
-
-        resp = client().admin().cluster().prepareAllocationExplain().setIndex("test").setShard(0).setPrimary(true).get();
-
-        cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(true, cae.isPrimary());
-        assertFalse(cae.isStillFetchingShardData());
-        assertNotNull("shard should have assigned node id", cae.getAssignedNodeId());
-        assertNull("assigned shard should not have unassigned info", cae.getUnassignedInfo());
-        explanation = cae.getNodeExplanations().values().iterator().next();
-        d = explanation.getDecision();
-        fd = explanation.getFinalDecision();
-        storeCopy = explanation.getStoreCopy();
-        assertNotNull("should have a decision", d);
-        assertEquals(Decision.Type.NO, d.type());
-        assertEquals(ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, fd);
-        assertEquals(ClusterAllocationExplanation.StoreCopy.AVAILABLE, storeCopy);
-        assertTrue(d.toString(), d.toString().contains(
-            "NO(the shard cannot be allocated to the node on which it already exists [[test][0]"));
-        assertTrue(d instanceof Decision.Multi);
-        md = (Decision.Multi) d;
-        ssd = md.getDecisions().get(0);
-        assertEquals(Decision.Type.NO, ssd.type());
-        assertTrue(ssd.toString(), ssd.toString().contains(
-            "NO(the shard cannot be allocated to the node on which it already exists [[test][0]"));
-        weight = explanation.getWeight();
-        assertNotNull("should have a weight", weight);
-
-        resp = client().admin().cluster().prepareAllocationExplain().useAnyUnassignedShard().get();
-        cae = resp.getExplanation();
-        assertNotNull("should always have an explanation", cae);
-        assertEquals("test", cae.getShard().getIndexName());
-        assertEquals(0, cae.getShard().getId());
-        assertEquals(false, cae.isPrimary());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index d656702f9cd..bce9afd1c1f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -19,21 +19,19 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import org.apache.lucene.index.CorruptIndexException;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
-import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.elasticsearch.cluster.routing.allocation.AllocationDecision;
+import org.elasticsearch.cluster.routing.allocation.MoveDecision;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -41,12 +39,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 
@@ -55,123 +47,6 @@ import static java.util.Collections.emptySet;
  */
 public final class ClusterAllocationExplanationTests extends ESTestCase {
 
-    private Index i = new Index("foo", "uuid");
-    private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE,
-        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-    private IndexMetaData indexMetaData = IndexMetaData.builder("foo")
-            .settings(Settings.builder()
-                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                    .put(IndexMetaData.SETTING_INDEX_UUID, "uuid"))
-            .putInSyncAllocationIds(0, Sets.newHashSet("aid1", "aid2"))
-            .numberOfShards(1)
-            .numberOfReplicas(1)
-            .build();
-    private DiscoveryNode node = new DiscoveryNode("node-0", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
-    private static Decision.Multi yesDecision = new Decision.Multi();
-    private static Decision.Multi noDecision = new Decision.Multi();
-
-    static {
-        yesDecision.add(Decision.single(Decision.Type.YES, "yes label", "yes please"));
-        noDecision.add(Decision.single(Decision.Type.NO, "no label", "no thanks"));
-    }
-
-    private void assertExplanations(NodeExplanation ne, String finalExplanation, ClusterAllocationExplanation.FinalDecision finalDecision,
-                                    ClusterAllocationExplanation.StoreCopy storeCopy) {
-        assertEquals(finalExplanation, ne.getFinalExplanation());
-        assertEquals(finalDecision, ne.getFinalDecision());
-        assertEquals(storeCopy, ne.getStoreCopy());
-    }
-
-    public void testDecisionAndExplanation() {
-        Exception e = new IOException("stuff's broke, yo");
-        Exception corruptE = new CorruptIndexException("stuff's corrupt, yo", "");
-        Float nodeWeight = randomFloat();
-        Set<String> activeAllocationIds = new HashSet<>();
-        activeAllocationIds.add("eggplant");
-        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-        ShardRouting replicaStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), false, PeerRecoverySource.INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
-
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, e);
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard cannot be read",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.IO_ERROR);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                null, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, null, "", activeAllocationIds, false);
-        assertExplanations(ne, "there is no copy of the shard available",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                null, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, corruptE);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard is corrupt",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.CORRUPT);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.STALE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the copy of the shard is stale, allocation ids do not match",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.STALE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "node-0", activeAllocationIds, false);
-        assertExplanations(ne, "the shard is already assigned to this node",
-                ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds, false);
-        assertExplanations(ne, "the shard can be assigned and the node contains a valid copy of the shard data",
-                ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, true);
-        assertExplanations(ne, "the shard's state is still being fetched so it cannot be allocated",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-
-        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null);
-        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(replicaStartedShard, indexMetaData, node, noDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds, true);
-        assertExplanations(ne, "the shard cannot be assigned because allocation deciders return a NO decision",
-                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-    }
-
     public void testDecisionEquality() {
         Decision.Multi d = new Decision.Multi();
         Decision.Multi d2 = new Decision.Multi();
@@ -185,67 +60,53 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
     }
 
     public void testExplanationSerialization() throws Exception {
-        ShardId shard = new ShardId("test", "uuid", 0);
-        long allocationDelay = randomIntBetween(0, 500);
-        long remainingDelay = randomIntBetween(0, 500);
-        Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
-        Float nodeWeight = randomFloat();
-        Set<String> activeAllocationIds = new HashSet<>();
-        activeAllocationIds.add("eggplant");
-
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
-        nodeExplanations.put(ne.getNode(), ne);
-        ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true,
-                "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations, null);
+        ClusterAllocationExplanation cae = randomClusterAllocationExplanation(randomBoolean());
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
         StreamInput in = out.bytes().streamInput();
         ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in);
-        assertEquals(shard, cae2.getShard());
+        assertEquals(cae.getShard(), cae2.getShard());
+        assertEquals(cae.isPrimary(), cae2.isPrimary());
         assertTrue(cae2.isPrimary());
-        assertTrue(cae2.isAssigned());
-        assertEquals("assignedNode", cae2.getAssignedNodeId());
-        assertNull(cae2.getUnassignedInfo());
-        assertEquals(allocationDelay, cae2.getAllocationDelayMillis());
-        assertEquals(remainingDelay, cae2.getRemainingDelayMillis());
-        for (Map.Entry<DiscoveryNode, NodeExplanation> entry : cae2.getNodeExplanations().entrySet()) {
-            NodeExplanation explanation = entry.getValue();
-            assertNotNull(explanation.getStoreStatus());
-            assertNotNull(explanation.getDecision());
-            assertEquals(nodeWeight, explanation.getWeight());
+        assertEquals(cae.getUnassignedInfo(), cae2.getUnassignedInfo());
+        assertEquals(cae.getCurrentNode(), cae2.getCurrentNode());
+        assertEquals(cae.getShardState(), cae2.getShardState());
+        if (cae.getClusterInfo() == null) {
+            assertNull(cae2.getClusterInfo());
+        } else {
+            assertNotNull(cae2.getClusterInfo());
+            assertEquals(cae.getClusterInfo().getNodeMostAvailableDiskUsages().size(),
+                cae2.getClusterInfo().getNodeMostAvailableDiskUsages().size());
         }
+        assertEquals(cae.getShardAllocationDecision().getAllocateDecision(), cae2.getShardAllocationDecision().getAllocateDecision());
+        assertEquals(cae.getShardAllocationDecision().getMoveDecision(), cae2.getShardAllocationDecision().getMoveDecision());
     }
 
     public void testExplanationToXContent() throws Exception {
-        ShardId shardId = new ShardId("foo", "uuid", 0);
-        Decision.Multi d = new Decision.Multi();
-        d.add(Decision.single(Decision.Type.NO, "no label", "because I said no"));
-        d.add(Decision.single(Decision.Type.YES, "yes label", "yes please"));
-        d.add(Decision.single(Decision.Type.THROTTLE, "throttle label", "wait a sec"));
-        Float nodeWeight = 1.5f;
-        Set<String> allocationIds = new HashSet<>();
-        allocationIds.add("bar");
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, new ElasticsearchException("stuff's broke, yo"));
-        NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                d, nodeWeight, storeStatus, "node-0", allocationIds, false);
-        Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
-        nodeExplanations.put(ne.getNode(), ne);
-        ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shardId, true,
-                "assignedNode", 42, 42, null, false, nodeExplanations, null);
+        ClusterAllocationExplanation cae = randomClusterAllocationExplanation(true);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertEquals("{\"shard\":{\"index\":\"foo\",\"index_uuid\":\"uuid\",\"id\":0,\"primary\":true},\"assigned\":true," +
-                        "\"assigned_node_id\":\"assignedNode\",\"shard_state_fetch_pending\":false,\"nodes\":{\"node-0\":" +
-                        "{\"node_name\":\"\",\"node_attributes\":{},\"store\":{\"shard_copy\":\"IO_ERROR\",\"store_except" +
-                        "ion\":\"ElasticsearchException[stuff's broke, yo]\"},\"final_decision\":\"ALREADY_ASSIGNED\",\"f" +
-                        "inal_explanation\":\"the shard is already assigned to this node\",\"weight\":1.5,\"decisions\":[" +
-                        "{\"decider\":\"no label\",\"decision\":\"NO\",\"explanation\":\"because I said no\"},{\"decider" +
-                        "\":\"yes label\",\"decision\":\"YES\",\"explanation\":\"yes please\"},{\"decider\":\"throttle la" +
-                        "bel\",\"decision\":\"THROTTLE\",\"explanation\":\"wait a sec\"}]}}}",
-                builder.string());
+        assertEquals("{\"index\":\"idx\",\"shard\":0,\"primary\":true,\"current_state\":\"started\",\"current_node\":" +
+                         "{\"id\":\"node-0\",\"name\":\"\",\"transport_address\":\"" + cae.getCurrentNode().getAddress() +
+                         "\",\"weight_ranking\":3},\"can_remain_on_current_node\":\"yes\",\"can_rebalance_cluster\":\"yes\"," +
+                         "\"can_rebalance_to_other_node\":\"no\",\"rebalance_explanation\":\"cannot rebalance as no target node exists " +
+                         "that can both allocate this shard and improve the cluster balance\"}", builder.string());
+    }
+
+    private static ClusterAllocationExplanation randomClusterAllocationExplanation(boolean assignedShard) {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(new Index("idx", "123"), 0),
+            assignedShard ? "node-0" : null, true, assignedShard ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED);
+        DiscoveryNode node = assignedShard ? new DiscoveryNode("node-0", buildNewFakeTransportAddress(), emptyMap(), emptySet(),
+                                                                  Version.CURRENT) : null;
+        ShardAllocationDecision shardAllocationDecision;
+        if (assignedShard) {
+            MoveDecision moveDecision = MoveDecision.cannotRebalance(Decision.YES, AllocationDecision.NO, 3, null)
+                                            .withRemainDecision(Decision.YES);
+            shardAllocationDecision = new ShardAllocationDecision(AllocateUnassignedDecision.NOT_TAKEN, moveDecision);
+        } else {
+            AllocateUnassignedDecision allocateDecision = AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.DECIDERS_NO, null);
+            shardAllocationDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN);
+        }
+        return new ClusterAllocationExplanation(shardRouting, node, null, null, shardAllocationDecision);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 6913abd81a4..d3d156ee2cd 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -291,41 +291,51 @@ public class NodeStatsTests extends ESTestCase {
                     new OsStats.Swap(randomLong(), randomLong()),
                     new OsStats.Cgroup(
                         randomAsciiOfLength(8),
-                        randomPositiveLong(),
+                        randomNonNegativeLong(),
                         randomAsciiOfLength(8),
-                        randomPositiveLong(),
-                        randomPositiveLong(),
-                        new OsStats.Cgroup.CpuStat(randomPositiveLong(), randomPositiveLong(), randomPositiveLong())));
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())));
         }
-        ProcessStats processStats = frequently() ? new ProcessStats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                new ProcessStats.Cpu(randomShort(), randomPositiveLong()),
-                new ProcessStats.Mem(randomPositiveLong())) : null;
+        ProcessStats processStats = frequently() ?
+            new ProcessStats(
+                randomNonNegativeLong(),
+                randomNonNegativeLong(),
+                randomNonNegativeLong(),
+                new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()),
+                new ProcessStats.Mem(randomNonNegativeLong())) :
+            null;
         JvmStats jvmStats = null;
         if (frequently()) {
             int numMemoryPools = randomIntBetween(0, 10);
             List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
             for (int i = 0; i < numMemoryPools; i++) {
-                memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong()));
+                memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
             }
             JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
             int numGarbageCollectors = randomIntBetween(0, 10);
             JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[numGarbageCollectors];
             for (int i = 0; i < numGarbageCollectors; i++) {
                 garbageCollectorsArray[i] = new JvmStats.GarbageCollector(randomAsciiOfLengthBetween(3, 10),
-                        randomPositiveLong(), randomPositiveLong());
+                        randomNonNegativeLong(), randomNonNegativeLong());
             }
             JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray);
             int numBufferPools = randomIntBetween(0, 10);
             List<JvmStats.BufferPool> bufferPoolList = new ArrayList<>();
             for (int i = 0; i < numBufferPools; i++) {
-                bufferPoolList.add(new JvmStats.BufferPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong()));
+                bufferPoolList.add(
+                    new JvmStats.BufferPool(
+                        randomAsciiOfLengthBetween(3, 10),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong(),
+                        randomNonNegativeLong()));
             }
-            JvmStats.Classes classes = new JvmStats.Classes(randomPositiveLong(), randomPositiveLong(), randomPositiveLong());
-            jvmStats = frequently() ? new JvmStats(randomPositiveLong(), randomPositiveLong(), new JvmStats.Mem(randomPositiveLong(),
-                    randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), memoryPools), threads,
-                    garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null;
+            JvmStats.Classes classes = new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
+            jvmStats =
+                frequently() ? new JvmStats(randomNonNegativeLong(), randomNonNegativeLong(), new JvmStats.Mem(randomNonNegativeLong(),
+                    randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), memoryPools),
+                    threads, garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null;
         }
         ThreadPoolStats threadPoolStats = null;
         if (frequently()) {
@@ -333,7 +343,7 @@ public class NodeStatsTests extends ESTestCase {
             List<ThreadPoolStats.Stats> threadPoolStatsList = new ArrayList<>();
             for (int i = 0; i < numThreadPoolStats; i++) {
                 threadPoolStatsList.add(new ThreadPoolStats.Stats(randomAsciiOfLengthBetween(3, 10), randomIntBetween(1, 1000),
-                        randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomPositiveLong(),
+                        randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomNonNegativeLong(),
                         randomIntBetween(1, 1000), randomIntBetween(1, 1000)));
             }
             threadPoolStats = new ThreadPoolStats(threadPoolStatsList);
@@ -345,50 +355,51 @@ public class NodeStatsTests extends ESTestCase {
             for (int i = 0; i < numDeviceStats; i++) {
                 FsInfo.DeviceStats previousDeviceStats = randomBoolean() ? null :
                         new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10),
-                                randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), null);
-                deviceStatsArray[i] = new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), previousDeviceStats);
+                                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), null);
+                deviceStatsArray[i] =
+                    new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), previousDeviceStats);
             }
             FsInfo.IoStats ioStats = new FsInfo.IoStats(deviceStatsArray);
             int numPaths = randomIntBetween(0, 10);
             FsInfo.Path[] paths = new FsInfo.Path[numPaths];
             for (int i = 0; i < numPaths; i++) {
                 paths[i] = new FsInfo.Path(randomAsciiOfLengthBetween(3, 10), randomBoolean() ? randomAsciiOfLengthBetween(3, 10) : null,
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong());
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
             }
-            fsInfo = new FsInfo(randomPositiveLong(), ioStats, paths);
+            fsInfo = new FsInfo(randomNonNegativeLong(), ioStats, paths);
         }
-        TransportStats transportStats = frequently() ? new TransportStats(randomPositiveLong(), randomPositiveLong(),
-                randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) : null;
-        HttpStats httpStats = frequently() ? new HttpStats(randomPositiveLong(), randomPositiveLong()) : null;
+        TransportStats transportStats = frequently() ? new TransportStats(randomNonNegativeLong(), randomNonNegativeLong(),
+                randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
+        HttpStats httpStats = frequently() ? new HttpStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
         AllCircuitBreakerStats allCircuitBreakerStats = null;
         if (frequently()) {
             int numCircuitBreakerStats = randomIntBetween(0, 10);
             CircuitBreakerStats[] circuitBreakerStatsArray = new CircuitBreakerStats[numCircuitBreakerStats];
             for (int i = 0; i < numCircuitBreakerStats; i++) {
-                circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(),
-                        randomPositiveLong(), randomDouble(), randomPositiveLong());
+                circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomDouble(), randomNonNegativeLong());
             }
             allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray);
         }
-        ScriptStats scriptStats = frequently() ? new ScriptStats(randomPositiveLong(), randomPositiveLong()) : null;
+        ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
         DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(),
                 randomInt(), randomInt()) : null) : null;
         IngestStats ingestStats = null;
         if (frequently()) {
-            IngestStats.Stats totalStats = new IngestStats.Stats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                    randomPositiveLong());
+            IngestStats.Stats totalStats = new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                    randomNonNegativeLong());
 
             int numStatsPerPipeline = randomIntBetween(0, 10);
             Map<String, IngestStats.Stats> statsPerPipeline = new HashMap<>();
             for (int i = 0; i < numStatsPerPipeline; i++) {
-                statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomPositiveLong(),
-                        randomPositiveLong(), randomPositiveLong(), randomPositiveLong()));
+                statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
             }
             ingestStats = new IngestStats(totalStats, statsPerPipeline);
         }
         //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet
-        return new NodeStats(node, randomPositiveLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo,
+        return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo,
                 transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, ingestStats);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 6d0a0824490..decff2ffc37 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -51,6 +51,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -91,7 +92,12 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return false;
+                }
+            };
         }
     }
 
@@ -126,7 +132,12 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return true;
+                }
+            };
         }
     }
 
@@ -289,6 +300,54 @@ public class CancellableTasksTests extends TaskManagerTestCase {
         });
     }
 
+    public void testChildTasksCancellation() throws Exception {
+        setupTestNodes(Settings.EMPTY);
+        connectNodes(testNodes);
+        CountDownLatch responseLatch = new CountDownLatch(1);
+        final AtomicReference<NodesResponse> responseReference = new AtomicReference<>();
+        final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
+        Task mainTask = startCancellableTestNodesAction(true, nodesCount, new ActionListener<NodesResponse>() {
+            @Override
+            public void onResponse(NodesResponse listTasksResponse) {
+                responseReference.set(listTasksResponse);
+                responseLatch.countDown();
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throwableReference.set(e);
+                responseLatch.countDown();
+            }
+        });
+
+        // Cancel all child tasks without cancelling the main task, which should quit on its own
+        CancelTasksRequest request = new CancelTasksRequest();
+        request.setReason("Testing Cancellation");
+        request.setParentTaskId(new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()));
+        // And send the cancellation request to a random node
+        CancelTasksResponse response = testNodes[randomIntBetween(1, testNodes.length - 1)].transportCancelTasksAction.execute(request)
+            .get();
+
+        // Awaiting for the main task to finish
+        responseLatch.await();
+
+        // Should have cancelled tasks on all nodes
+        assertThat(response.getTasks().size(), equalTo(testNodes.length));
+
+        assertBusy(() -> {
+            try {
+            // Make sure that main task is no longer running
+                ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)]
+                    .transportListTasksAction.execute(new ListTasksRequest().setTaskId(
+                        new TaskId(testNodes[0].discoveryNode.getId(), mainTask.getId()))).get();
+                assertEquals(0, listTasksResponse.getTasks().size());
+
+            } catch (ExecutionException | InterruptedException ex) {
+                throw new RuntimeException(ex);
+            }
+        });
+    }
+
     public void testTaskCancellationOnCoordinatingNodeLeavingTheCluster() throws Exception {
         setupTestNodes(Settings.EMPTY);
         connectNodes(testNodes);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 43adf182c8b..7366ce22523 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -171,8 +172,9 @@ public abstract class TaskManagerTestCase extends ESTestCase {
             clusterService = createClusterService(threadPool);
             transportService = new TransportService(settings,
                     new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE, new NoneCircuitBreakerService(),
-                            new NamedWriteableRegistry(Collections.emptyList()), new NetworkService(settings, Collections.emptyList())),
-                    threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null) {
+                        new NamedWriteableRegistry(ClusterModule.getNamedWriteables()),
+                        new NetworkService(settings, Collections.emptyList())),
+                    threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null) {
                 @Override
                 protected TaskManager createTaskManager() {
                     if (MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING.get(settings)) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java
new file mode 100644
index 00000000000..d3f10f2b038
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskTests.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.node.tasks;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+public class TaskTests extends ESTestCase {
+
+    public void testTaskInfoToString() {
+        String nodeId = randomAsciiOfLength(10);
+        long taskId = randomIntBetween(0, 100000);
+        long startTime = randomNonNegativeLong();
+        long runningTime = randomNonNegativeLong();
+        boolean cancellable = randomBoolean();
+        TaskInfo taskInfo = new TaskInfo(new TaskId(nodeId, taskId), "test_type",
+            "test_action", "test_description", null, startTime, runningTime, cancellable, TaskId.EMPTY_TASK_ID);
+        String taskInfoString = taskInfo.toString();
+        Map<String, Object> map = XContentHelper.convertToMap(new BytesArray(taskInfoString.getBytes(StandardCharsets.UTF_8)), true).v2();
+        assertEquals(((Number)map.get("id")).longValue(), taskId);
+        assertEquals(map.get("type"), "test_type");
+        assertEquals(map.get("action"), "test_action");
+        assertEquals(map.get("description"), "test_description");
+        assertEquals(((Number)map.get("start_time_in_millis")).longValue(), startTime);
+        assertEquals(((Number)map.get("running_time_in_nanos")).longValue(), runningTime);
+        assertEquals(map.get("cancellable"), cancellable);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 7f490ebab90..580d8149fd5 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -50,6 +50,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -300,7 +301,8 @@ public class TasksIT extends ESIntegTestCase {
         registerTaskManageListeners(BulkAction.NAME + "[s][r]");  // shard task on replica
         createIndex("test");
         ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks
-        client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}")).get();
+        client().prepareBulk().add(client().prepareIndex("test", "doc", "test_id")
+            .setSource("{\"foo\": \"bar\"}", XContentType.JSON)).get();
 
         // the bulk operation should produce one main task
         List<TaskInfo> topTask = findEvents(BulkAction.NAME, Tuple::v1);
@@ -352,7 +354,7 @@ public class TasksIT extends ESIntegTestCase {
         registerTaskManageListeners(SearchAction.NAME + "[*]");  // shard task
         createIndex("test");
         ensureGreen("test"); // Make sure all shards are allocated to catch replication tasks
-        client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}")
+        client().prepareIndex("test", "doc", "test_id").setSource("{\"foo\": \"bar\"}", XContentType.JSON)
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
 
         assertSearchResponse(client().prepareSearch("test").setTypes("doc").setQuery(QueryBuilders.matchAllQuery()).get());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 28aff0d3bda..b4ba0354ed7 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -79,10 +79,15 @@ public class TestTaskPlugin extends Plugin implements ActionPlugin {
 
         private volatile boolean blocked = true;
 
-        public TestTask(long id, String type, String action, String description, TaskId parentTaskId) {
+        TestTask(long id, String type, String action, String description, TaskId parentTaskId) {
             super(id, type, action, description, parentTaskId);
         }
 
+        @Override
+        public boolean shouldCancelChildrenOnCancellation() {
+            return false;
+        }
+
         public boolean isBlocked() {
             return blocked;
         }
@@ -242,7 +247,12 @@ public class TestTaskPlugin extends Plugin implements ActionPlugin {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-            return new CancellableTask(id, type, action, getDescription(), parentTaskId);
+            return new CancellableTask(id, type, action, getDescription(), parentTaskId) {
+                @Override
+                public boolean shouldCancelChildrenOnCancellation() {
+                    return true;
+                }
+            };
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index fbe0302e5d9..07859070d10 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -190,7 +190,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         private final String status;
 
-        public TestTaskResponse(StreamInput in) throws IOException {
+        TestTaskResponse(StreamInput in) throws IOException {
             status = in.readString();
         }
 
@@ -199,7 +199,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             out.writeString(status);
         }
 
-        public TestTaskResponse(String status) {
+        TestTaskResponse(String status) {
             this.status = status;
         }
 
@@ -216,11 +216,11 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
 
         private List<TestTaskResponse> tasks;
 
-        public TestTasksResponse() {
+        TestTasksResponse() {
 
         }
 
-        public TestTasksResponse(List<TestTaskResponse> tasks, List<TaskOperationFailure> taskFailures,
+        TestTasksResponse(List<TestTaskResponse> tasks, List<TaskOperationFailure> taskFailures,
                 List<? extends FailedNodeException> nodeFailures) {
             super(taskFailures, nodeFailures);
             this.tasks = tasks == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(tasks));
@@ -801,6 +801,6 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         builder.endObject();
         builder.flush();
         logger.info(builder.string());
-        return XContentHelper.convertToMap(builder.bytes(), false).v2();
+        return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
index e1205840976..4b531267d3a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -25,15 +25,14 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimary
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -71,10 +70,8 @@ public class ClusterRerouteRequestTests extends ESTestCase {
             () -> new MoveAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
                     randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10))));
     private final NamedWriteableRegistry namedWriteableRegistry;
-    private final AllocationCommandRegistry allocationCommandRegistry;
 
     public ClusterRerouteRequestTests() {
-        allocationCommandRegistry = NetworkModule.getAllocationCommandRegistry();
         namedWriteableRegistry = new NamedWriteableRegistry(NetworkModule.getNamedWriteables());
     }
 
@@ -176,10 +173,10 @@ public class ClusterRerouteRequestTests extends ESTestCase {
 
     private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest original) throws IOException {
         RestRequest restRequest = toRestRequest(original);
-        return RestClusterRerouteAction.createRequest(restRequest, allocationCommandRegistry, ParseFieldMatcher.STRICT);
+        return RestClusterRerouteAction.createRequest(restRequest);
     }
 
-    private static RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
+    private RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
         Map<String, String> params = new HashMap<>();
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         boolean hasBody = false;
@@ -209,11 +206,16 @@ public class ClusterRerouteRequestTests extends ESTestCase {
         }
         builder.endObject();
 
-        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder();
+        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
         requestBuilder.withParams(params);
         if (hasBody) {
-            requestBuilder.withContent(builder.bytes());
+            requestBuilder.withContent(builder.bytes(), builder.contentType());
         }
         return requestBuilder.build();
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(NetworkModule.getNamedXContents());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
new file mode 100644
index 00000000000..5f3e1079421
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.storedscripts;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Base64;
+
+public class PutStoredScriptRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        PutStoredScriptRequest storedScriptRequest = new PutStoredScriptRequest("foo", "bar", new BytesArray("{}"), XContentType.JSON);
+
+        assertEquals(XContentType.JSON, storedScriptRequest.xContentType());
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            storedScriptRequest.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                PutStoredScriptRequest serialized = new PutStoredScriptRequest();
+                serialized.readFrom(in);
+                assertEquals(XContentType.JSON, serialized.xContentType());
+                assertEquals(storedScriptRequest.lang(), serialized.lang());
+                assertEquals(storedScriptRequest.id(), serialized.id());
+            }
+        }
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] rawStreamBytes = Base64.getDecoder().decode("ADwDCG11c3RhY2hlAQZzY3JpcHQCe30A");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(rawStreamBytes)) {
+            in.setVersion(version);
+            PutStoredScriptRequest serialized = new PutStoredScriptRequest();
+            serialized.readFrom(in);
+            assertEquals(XContentType.JSON, serialized.xContentType());
+            assertEquals("mustache", serialized.lang());
+            assertEquals("script", serialized.id());
+            assertEquals(new BytesArray("{}"), serialized.content());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                serialized.writeTo(out);
+                out.flush();
+                assertArrayEquals(rawStreamBytes, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
index b374ce7135c..8cd3f271b7b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/alias/AliasActionsTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.admin.indices.alias;
 
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -142,7 +141,7 @@ public class AliasActionsTests extends ESTestCase {
         b.endObject();
         b = shuffleXContent(b, "filter");
         try (XContentParser parser = createParser(b)) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.ADD, action.actionType());
             assertThat(action.indices(), equalTo(indices));
             assertThat(action.aliases(), equalTo(aliases));
@@ -179,7 +178,7 @@ public class AliasActionsTests extends ESTestCase {
         b.endObject();
         b = shuffleXContent(b);
         try (XContentParser parser = createParser(b)) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.ADD, action.actionType());
             assertThat(action.indices(), arrayContaining(index));
             assertThat(action.aliases(), arrayContaining(alias));
@@ -210,7 +209,7 @@ public class AliasActionsTests extends ESTestCase {
         b.endObject();
         b = shuffleXContent(b);
         try (XContentParser parser = createParser(b)) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.REMOVE, action.actionType());
             assertThat(action.indices(), equalTo(indices));
             assertThat(action.aliases(), equalTo(aliases));
@@ -233,7 +232,7 @@ public class AliasActionsTests extends ESTestCase {
         b.endObject();
         b = shuffleXContent(b);
         try (XContentParser parser = createParser(b)) {
-            AliasActions action = AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            AliasActions action = AliasActions.PARSER.apply(parser, null);
             assertEquals(AliasActions.Type.REMOVE_INDEX, action.actionType());
             assertArrayEquals(indices, action.indices());
             assertThat(action.aliases(), arrayWithSize(0));
@@ -252,7 +251,7 @@ public class AliasActionsTests extends ESTestCase {
         }
         b.endObject();
         try (XContentParser parser = createParser(b)) {
-            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT));
+            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, null));
             assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
             assertEquals("Only one of [index] and [indices] is supported", e.getCause().getCause().getMessage());
         }
@@ -270,7 +269,7 @@ public class AliasActionsTests extends ESTestCase {
         }
         b.endObject();
         try (XContentParser parser = createParser(b)) {
-            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT));
+            Exception e = expectThrows(ParsingException.class, () -> AliasActions.PARSER.apply(parser, null));
             assertThat(e.getCause().getCause(), instanceOf(IllegalArgumentException.class));
             assertEquals("Only one of [alias] and [aliases] is supported", e.getCause().getCause().getMessage());
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 9d2e56f25bb..f461be77e0c 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -21,27 +21,19 @@ package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
-import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
-import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -49,12 +41,12 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import java.util.HashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -64,6 +56,7 @@ import static org.hamcrest.core.IsNull.notNullValue;
 
 @ClusterScope(scope = Scope.TEST)
 public class CreateIndexIT extends ESIntegTestCase {
+
     public void testCreationDateGivenFails() {
         try {
             prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4L)).get();
@@ -277,15 +270,8 @@ public class CreateIndexIT extends ESIntegTestCase {
                 .startObject("text")
                     .field("type", "text")
                 .endObject().endObject().endObject());
-        try {
-            b.get();
-        } catch (MapperParsingException e) {
-            StringBuilder messages = new StringBuilder();
-            for (Exception rootCause: e.guessRootCauses()) {
-                messages.append(rootCause.getMessage());
-            }
-            assertThat(messages.toString(), containsString("mapper [text] is used by multiple types"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> b.get());
+        assertThat(e.getMessage(), containsString("mapper [text] is used by multiple types"));
     }
 
     public void testRestartIndexCreationAfterFullClusterRestart() throws Exception {
@@ -296,192 +282,6 @@ public class CreateIndexIT extends ESIntegTestCase {
         ensureGreen("test");
     }
 
-    public void testCreateShrinkIndexToN() {
-        int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
-        int[] shardSplits = randomFrom(possibleShardSplits);
-        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
-        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String mergeNode = discoveryNodes[0].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a 4 shard index
-        assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink")
-            .setSettings(Settings.builder()
-                .put("index.number_of_replicas", 0)
-                .put("index.number_of_shards", shardSplits[1]).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        for (int i = 0; i < 20; i++) { // now update
-            client().prepareIndex("first_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("first_shrink")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a 2 shard index
-        assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink")
-            .setSettings(Settings.builder()
-                .put("index.number_of_replicas", 0)
-                .put("index.number_of_shards", shardSplits[2]).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        // let it be allocated anywhere and bump replicas
-        client().admin().indices().prepareUpdateSettings("second_shrink")
-            .setSettings(Settings.builder()
-                .putNull("index.routing.allocation.include._id")
-                .put("index.number_of_replicas", 1)).get();
-        ensureGreen();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        for (int i = 0; i < 20; i++) { // now update
-            client().prepareIndex("second_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-    }
-
-    public void testCreateShrinkIndex() {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7))).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String mergeNode = discoveryNodes[0].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-        // now merge source into a single shard index
-
-        final boolean createWithReplicas = randomBoolean();
-        assertAcked(client().admin().indices().prepareShrinkIndex("source", "target")
-            .setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).build()).get());
-        ensureGreen();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-        if (createWithReplicas == false) {
-            // bump replicas
-            client().admin().indices().prepareUpdateSettings("target")
-                .setSettings(Settings.builder()
-                    .put("index.number_of_replicas", 1)).get();
-            ensureGreen();
-            assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        }
-
-        for (int i = 20; i < 40; i++) {
-            client().prepareIndex("target", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40);
-        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-
-    }
-    /**
-     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
-     */
-    public void testCreateShrinkIndexFails() throws Exception {
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
-            .put("number_of_shards", randomIntBetween(2, 7))
-            .put("number_of_replicas", 0)).get();
-        for (int i = 0; i < 20; i++) {
-            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
-        }
-        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
-            .getDataNodes();
-        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
-        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
-        String spareNode = discoveryNodes[0].getName();
-        String mergeNode = discoveryNodes[1].getName();
-        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
-        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
-        // to the require._name below.
-        ensureGreen();
-        // relocate all shards to one node such that we can merge it.
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen();
-
-        // now merge source into a single shard index
-        client().admin().indices().prepareShrinkIndex("source", "target")
-            .setSettings(Settings.builder()
-            .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
-            .put("index.number_of_replicas", 0)
-            .put("index.allocation.max_retries", 1).build()).get();
-
-        // now we move all shards away from the merge node
-        client().admin().indices().prepareUpdateSettings("source")
-            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode)
-                .put("index.blocks.write", true)).get();
-        ensureGreen("source");
-
-        client().admin().indices().prepareUpdateSettings("target") // erase the forcefully fuckup!
-            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name")).get();
-        // wait until it fails
-        assertBusy(() -> {
-            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
-            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
-            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
-            assertEquals(UnassignedInfo.Reason.ALLOCATION_FAILED,
-                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason());
-            assertEquals(1,
-                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
-        });
-        client().admin().indices().prepareUpdateSettings("source") // now relocate them all to the right node
-            .setSettings(Settings.builder()
-                .put("index.routing.allocation.require._name", mergeNode)).get();
-        ensureGreen("source");
-
-        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class,
-            internalCluster().getMasterName());
-        infoService.refresh();
-        // kick off a retry and wait until it's done!
-        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
-        long expectedShardSize = clusterRerouteResponse.getState().routingTable().index("target")
-            .shard(0).getShards().get(0).getExpectedShardSize();
-        // we support the expected shard size in the allocator to sum up over the source index shards
-        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
-        ensureGreen();
-        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-    }
-
     /**
      * This test ensures that index creation adheres to the {@link IndexMetaData#SETTING_WAIT_FOR_ACTIVE_SHARDS}.
      */
@@ -508,4 +308,31 @@ public class CreateIndexIT extends ESIntegTestCase {
                        .build();
         assertFalse(client().admin().indices().prepareCreate("test-idx-3").setSettings(settings).setTimeout("100ms").get().isShardsAcked());
     }
+
+    public void testInvalidPartitionSize() {
+        BiFunction<Integer, Integer, Boolean> createPartitionedIndex = (shards, partitionSize) -> {
+            CreateIndexResponse response;
+
+            try {
+                response = prepareCreate("test_" + shards + "_" + partitionSize)
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", shards)
+                        .put("index.routing_partition_size", partitionSize))
+                    .execute().actionGet();
+            } catch (IllegalStateException | IllegalArgumentException e) {
+                return false;
+            }
+
+            return response.isAcknowledged();
+        };
+
+        assertFalse(createPartitionedIndex.apply(3, 6));
+        assertFalse(createPartitionedIndex.apply(3, 0));
+        assertFalse(createPartitionedIndex.apply(3, 3));
+
+        assertTrue(createPartitionedIndex.apply(3, 1));
+        assertTrue(createPartitionedIndex.apply(3, 2));
+
+        assertTrue(createPartitionedIndex.apply(1, 1));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
index ad17689bb78..1ae60f42eb6 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilderTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.create;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
 import org.junit.After;
@@ -57,7 +58,7 @@ public class CreateIndexRequestBuilderTests extends ESTestCase {
      */
     public void testSetSource() throws IOException {
         CreateIndexRequestBuilder builder = new CreateIndexRequestBuilder(this.testClient, CreateIndexAction.INSTANCE);
-        builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}");
+        builder.setSource("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         XContentBuilder xContent = XContentFactory.jsonBuilder().startObject().field(KEY, VALUE).endObject();
@@ -68,7 +69,7 @@ public class CreateIndexRequestBuilderTests extends ESTestCase {
         ByteArrayOutputStream docOut = new ByteArrayOutputStream();
         XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field(KEY, VALUE).endObject();
         doc.close();
-        builder.setSource(docOut.toByteArray());
+        builder.setSource(docOut.toByteArray(), XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         Map<String, String> settingsMap = new HashMap<>();
@@ -85,7 +86,7 @@ public class CreateIndexRequestBuilderTests extends ESTestCase {
         builder.setSettings(KEY, VALUE);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
-        builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}");
+        builder.setSettings("{\""+KEY+"\" : \""+VALUE+"\"}", XContentType.JSON);
         assertEquals(VALUE, builder.request().settings().get(KEY));
 
         builder.setSettings(Settings.builder().put(KEY, VALUE));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
new file mode 100644
index 00000000000..590eba36666
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Base64;
+
+public class CreateIndexRequestTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        CreateIndexRequest request = new CreateIndexRequest("foo");
+        String mapping = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().string();
+        request.mapping("my_type", mapping, XContentType.JSON);
+
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            request.writeTo(output);
+
+            try (StreamInput in = output.bytes().streamInput()) {
+                CreateIndexRequest serialized = new CreateIndexRequest();
+                serialized.readFrom(in);
+                assertEquals(request.index(), serialized.index());
+                assertEquals(mapping, serialized.mappings().get("my_type"));
+            }
+        }
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAANmb28APAMBB215X3R5cGULeyJ0eXBlIjp7fX0AAAD////+AA==");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            CreateIndexRequest serialized = new CreateIndexRequest();
+            serialized.readFrom(in);
+            assertEquals("foo", serialized.index());
+            BytesReference bytesReference = JsonXContent.contentBuilder().startObject().startObject("type").endObject().endObject().bytes();
+            assertEquals(bytesReference.utf8ToString(), serialized.mappings().get("my_type"));
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                serialized.writeTo(out);
+                out.flush();
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
new file mode 100644
index 00000000000..9ab6551d6fd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.create;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.InternalClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+
+public class ShrinkIndexIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class);
+    }
+
+    public void testCreateShrinkIndexToN() {
+        int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
+        int[] shardSplits = randomFrom(possibleShardSplits);
+        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
+        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 4 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[1]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("first_shrink", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 2 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[2]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        // let it be allocated anywhere and bump replicas
+        client().admin().indices().prepareUpdateSettings("second_shrink")
+            .setSettings(Settings.builder()
+                .putNull("index.routing.allocation.include._id")
+                .put("index.number_of_replicas", 1)).get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("second_shrink", "t1", Integer.toString(i))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+
+    public void testCreateShrinkIndex() {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        Version version = VersionUtils.randomVersion(random());
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
+            .put("number_of_shards", randomIntBetween(2, 7))
+            .put("index.version.created", version)
+        ).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a single shard index
+
+        final boolean createWithReplicas = randomBoolean();
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "target")
+            .setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        if (createWithReplicas == false) {
+            // bump replicas
+            client().admin().indices().prepareUpdateSettings("target")
+                .setSettings(Settings.builder()
+                    .put("index.number_of_replicas", 1)).get();
+            ensureGreen();
+            assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        }
+
+        for (int i = 20; i < 40; i++) {
+            client().prepareIndex("target", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get();
+        assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null));
+    }
+    /**
+     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
+     */
+    public void testCreateShrinkIndexFails() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
+            .put("number_of_shards", randomIntBetween(2, 7))
+            .put("number_of_replicas", 0)).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3"))
+                .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String spareNode = discoveryNodes[0].getName();
+        String mergeNode = discoveryNodes[1].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+
+        // now merge source into a single shard index
+        client().admin().indices().prepareShrinkIndex("source", "target")
+            .setWaitForActiveShards(ActiveShardCount.NONE)
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
+                .put("index.number_of_replicas", 0)
+                .put("index.allocation.max_retries", 1).build()).get();
+        client().admin().cluster().prepareHealth("target").setWaitForEvents(Priority.LANGUID).get();
+
+        // now we move all shards away from the merge node
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen("source");
+
+        client().admin().indices().prepareUpdateSettings("target") // erase the forcefully fuckup!
+            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name")).get();
+        // wait until it fails
+        assertBusy(() -> {
+            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
+            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
+            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
+            assertEquals(UnassignedInfo.Reason.ALLOCATION_FAILED,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason());
+            assertEquals(1,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
+        });
+        client().admin().indices().prepareUpdateSettings("source") // now relocate them all to the right node
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)).get();
+        ensureGreen("source");
+
+        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class,
+            internalCluster().getMasterName());
+        infoService.refresh();
+        // kick off a retry and wait until it's done!
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        long expectedShardSize = clusterRerouteResponse.getState().routingTable().index("target")
+            .shard(0).getShards().get(0).getExpectedShardSize();
+        // we support the expected shard size in the allocator to sum up over the source index shards
+        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
index 0ca2bd23386..3ba349ffca8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/FlushBlocksIT.java
@@ -46,7 +46,7 @@ public class FlushBlocksIT extends ESIntegTestCase {
         }
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 FlushResponse response = client().admin().indices().prepareFlush("test").execute().actionGet();
@@ -56,28 +56,5 @@ public class FlushBlocksIT extends ESIntegTestCase {
                 disableIndexBlock("test", blockSetting);
             }
         }
-
-        // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
-            try {
-                enableIndexBlock("test", blockSetting);
-                FlushResponse flushResponse = client().admin().indices().prepareFlush("test").get();
-                assertBlocked(flushResponse);
-            } finally {
-                disableIndexBlock("test", blockSetting);
-            }
-        }
-
-        // Flushing all indices is blocked when the cluster is read-only
-        try {
-            FlushResponse response = client().admin().indices().prepareFlush().execute().actionGet();
-            assertNoFailures(response);
-            assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards));
-
-            setClusterReadOnly(true);
-            assertBlocked(client().admin().indices().prepareFlush().get());
-        } finally {
-            setClusterReadOnly(false);
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
new file mode 100644
index 00000000000..4dc396323c0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.mapping.get;
+
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class GetFieldMappingsResponseTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        Map<String, Map<String, Map<String, FieldMappingMetaData>>> mappings = new HashMap<>();
+        FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}"));
+        mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData)));
+        GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            response.writeTo(out);
+            GetFieldMappingsResponse serialized = new GetFieldMappingsResponse();
+            try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) {
+                serialized.readFrom(in);
+                FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field");
+                assertNotNull(metaData);
+                assertEquals(new BytesArray("{}"), metaData.getSource());
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
index 9c93e5c73d9..fd7f830e592 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingRequestTests.java
@@ -19,10 +19,20 @@
 
 package org.elasticsearch.action.admin.indices.mapping.put;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+import java.util.Base64;
+
 public class PutMappingRequestTests extends ESTestCase {
 
     public void testValidation() {
@@ -41,12 +51,12 @@ public class PutMappingRequestTests extends ESTestCase {
         assertNotNull("source validation should fail", ex);
         assertTrue(ex.getMessage().contains("source is missing"));
 
-        r.source("");
+        r.source("", XContentType.JSON);
         ex = r.validate();
         assertNotNull("source validation should fail", ex);
         assertTrue(ex.getMessage().contains("source is empty"));
 
-        r.source("somevalidmapping");
+        r.source("somevalidmapping", XContentType.JSON);
         ex = r.validate();
         assertNull("validation should succeed", ex);
 
@@ -64,4 +74,33 @@ public class PutMappingRequestTests extends ESTestCase {
                 () -> PutMappingRequest.buildFromSimplifiedDef("type", "only_field"));
         assertEquals("mapping source must be pairs of fieldnames and properties definition.", e.getMessage());
     }
+
+    public void testPutMappingRequestSerialization() throws IOException {
+        PutMappingRequest request = new PutMappingRequest("foo");
+        String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+        request.source(mapping, XContentType.YAML);
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source());
+
+        BytesStreamOutput bytesStreamOutput = new BytesStreamOutput();
+        request.writeTo(bytesStreamOutput);
+        StreamInput in = StreamInput.wrap(bytesStreamOutput.bytes().toBytesRef().bytes);
+        PutMappingRequest serialized = new PutMappingRequest();
+        serialized.readFrom(in);
+
+        String source = serialized.source();
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), source);
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAQNmb28MAA8tLS0KZm9vOiAiYmFyIgoAPAMAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutMappingRequest request = new PutMappingRequest();
+            request.readFrom(in);
+            String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+            assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.source());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
index 33e03010d59..cc74f7c7341 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/refresh/RefreshBlocksIT.java
@@ -42,7 +42,7 @@ public class RefreshBlocksIT extends ESIntegTestCase {
         NumShards numShards = getNumShards("test");
 
         // Request is not blocked
-        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE)) {
+        for (String blockSetting : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
             try {
                 enableIndexBlock("test", blockSetting);
                 RefreshResponse response = client().admin().indices().prepareRefresh("test").execute().actionGet();
@@ -52,27 +52,5 @@ public class RefreshBlocksIT extends ESIntegTestCase {
                 disableIndexBlock("test", blockSetting);
             }
         }
-
-        // Request is blocked
-        for (String blockSetting : Arrays.asList(SETTING_READ_ONLY, SETTING_BLOCKS_METADATA)) {
-            try {
-                enableIndexBlock("test", blockSetting);
-                assertBlocked(client().admin().indices().prepareRefresh("test").get());
-            } finally {
-                disableIndexBlock("test", blockSetting);
-            }
-        }
-
-        // Refreshing all indices is blocked when the cluster is read-only
-        try {
-            RefreshResponse response = client().admin().indices().prepareRefresh().execute().actionGet();
-            assertNoFailures(response);
-            assertThat(response.getSuccessfulShards(), equalTo(numShards.totalNumShards));
-
-            setClusterReadOnly(true);
-            assertBlocked(client().admin().indices().prepareRefresh().get());
-        } finally {
-            setClusterReadOnly(false);
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
index 999d16fdcb5..213616eec68 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -19,11 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Set;
@@ -36,12 +34,12 @@ public class RolloverRequestTests extends ESTestCase {
         final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
-            .startObject("conditions")
-                .field("max_age", "10d")
-                .field("max_docs", 100)
-            .endObject()
+                .startObject("conditions")
+                    .field("max_age", "10d")
+                    .field("max_docs", 100)
+                .endObject()
             .endObject();
-        RolloverRequest.PARSER.parse(XContentHelper.createParser(builder.bytes()), request, () -> ParseFieldMatcher.EMPTY);
+        RolloverRequest.PARSER.parse(createParser(builder), request, null);
         Set<Condition> conditions = request.getConditions();
         assertThat(conditions.size(), equalTo(2));
         for (Condition condition : conditions) {
@@ -61,28 +59,28 @@ public class RolloverRequestTests extends ESTestCase {
         final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
-            .startObject("conditions")
-                .field("max_age", "10d")
-                .field("max_docs", 100)
-            .endObject()
-            .startObject("mappings")
-                .startObject("type1")
-                    .startObject("properties")
-                        .startObject("field1")
-                            .field("type", "string")
-                            .field("index", "not_analyzed")
+                .startObject("conditions")
+                    .field("max_age", "10d")
+                    .field("max_docs", 100)
+                .endObject()
+                .startObject("mappings")
+                    .startObject("type1")
+                        .startObject("properties")
+                            .startObject("field1")
+                                .field("type", "string")
+                                .field("index", "not_analyzed")
+                            .endObject()
                         .endObject()
                     .endObject()
                 .endObject()
-            .endObject()
-            .startObject("settings")
-                .field("number_of_shards", 10)
-            .endObject()
-            .startObject("aliases")
-                .startObject("alias1").endObject()
-            .endObject()
+                .startObject("settings")
+                    .field("number_of_shards", 10)
+                .endObject()
+                .startObject("aliases")
+                    .startObject("alias1").endObject()
+                .endObject()
             .endObject();
-        RolloverRequest.PARSER.parse(XContentHelper.createParser(builder.bytes()), request, () -> ParseFieldMatcher.EMPTY);
+        RolloverRequest.PARSER.parse(createParser(builder), request, null);
         Set<Condition> conditions = request.getConditions();
         assertThat(conditions.size(), equalTo(2));
         assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index 44fb991af94..7cd18f0254e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -219,7 +219,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
     private static final class IndexNodePredicate implements Predicate<Settings> {
         private final Set<String> nodesWithShard;
 
-        public IndexNodePredicate(String index) {
+        IndexNodePredicate(String index) {
             this.nodesWithShard = findNodesWithShard(index);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index dfc10169e70..859e9f6e25d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -19,7 +19,11 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
+import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.CommitStats;
@@ -111,6 +115,32 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         }
     }
 
+    public void testRefreshListeners() throws Exception {
+        // Create an index without automatic refreshes
+        createIndex("test", Settings.builder().put("refresh_interval", -1).build());
+
+        // Index a document asynchronously so the request will only return when document is refreshed
+        ListenableActionFuture<IndexResponse> index = client().prepareIndex("test", "test", "test").setSource("test", "test")
+                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).execute();
+
+        // Wait for the refresh listener to appear in the stats
+        assertBusy(() -> {
+            IndicesStatsResponse stats = client().admin().indices().prepareStats("test").clear().setRefresh(true).get();
+            CommonStats common = stats.getIndices().get("test").getTotal();
+            assertEquals(1, common.refresh.getListeners());
+        });
+
+        // Refresh the index and wait for the request to come back
+        client().admin().indices().prepareRefresh("test").get();
+        index.get();
+
+        // The document should appear in the statistics and the refresh listener should be gone
+        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").clear().setRefresh(true).setDocs(true).get();
+        CommonStats common = stats.getIndices().get("test").getTotal();
+        assertEquals(1, common.docs.getCount());
+        assertEquals(0, common.refresh.getListeners());
+    }
+
     /**
      * Gives access to package private IndicesStatsResponse constructor for test purpose.
      **/
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
index 69c6731aa15..3615ea10c36 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/BWCTemplateTests.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.template;
 
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 
 /**
  * Rudimentary tests that the templates used by Logstash and Beats
@@ -29,25 +30,27 @@ import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
  */
 public class BWCTemplateTests extends ESSingleNodeTestCase {
     public void testBeatsTemplatesBWC() throws Exception {
-        String metricBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json");
-        String packetBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json");
-        String fileBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json");
-        String winLogBeat = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json");
-        client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat).get();
-        client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat).get();
-        client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat).get();
-        client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat).get();
+        byte[] metricBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json");
+        byte[] packetBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json");
+        byte[] fileBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json");
+        byte[] winLogBeat = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json");
+        client().admin().indices().preparePutTemplate("metricbeat").setSource(metricBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("packetbeat").setSource(packetBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("filebeat").setSource(fileBeat, XContentType.JSON).get();
+        client().admin().indices().preparePutTemplate("winlogbeat").setSource(winLogBeat, XContentType.JSON).get();
 
         client().prepareIndex("metricbeat-foo", "doc", "1").setSource("message", "foo").get();
         client().prepareIndex("packetbeat-foo", "doc", "1").setSource("message", "foo").get();
         client().prepareIndex("filebeat-foo", "doc", "1").setSource("message", "foo").get();
         client().prepareIndex("winlogbeat-foo", "doc", "1").setSource("message", "foo").get();
+        assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
     }
 
     public void testLogstashTemplatesBWC() throws Exception {
-        String ls5x = copyToStringFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json");
-        client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x).get();
+        byte[] ls5x = copyToBytesFromClasspath("/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json");
+        client().admin().indices().preparePutTemplate("logstash-5x").setSource(ls5x, XContentType.JSON).get();
         client().prepareIndex("logstash-foo", "doc", "1").setSource("message", "foo").get();
+        assertWarnings("Deprecated field [template] used, replaced by [index_patterns]");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index 9779ce83a65..48598ecb2ec 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutReques
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.indices.IndicesService;
@@ -58,7 +59,7 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         map.put("index.shard.check_on_startup", "blargh");
         request.settings(Settings.builder().put(map).build());
 
-        List<Throwable> throwables = putTemplate(request);
+        List<Throwable> throwables = putTemplate(xContentRegistry(), request);
         assertEquals(throwables.size(), 1);
         assertThat(throwables.get(0), instanceOf(InvalidIndexTemplateException.class));
         assertThat(throwables.get(0).getMessage(),
@@ -76,7 +77,7 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         map.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "0");
         request.settings(Settings.builder().put(map).build());
 
-        List<Throwable> throwables = putTemplate(request);
+        List<Throwable> throwables = putTemplate(xContentRegistry(), request);
         assertEquals(throwables.size(), 1);
         assertThat(throwables.get(0), instanceOf(InvalidIndexTemplateException.class));
         assertThat(throwables.get(0).getMessage(), containsString("name must not contain a space"));
@@ -90,7 +91,7 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
         request.patterns(Arrays.asList("foo", "foobar"));
         request.aliases(Collections.singleton(new Alias("foobar")));
 
-        List<Throwable> errors = putTemplate(request);
+        List<Throwable> errors = putTemplate(xContentRegistry(), request);
         assertThat(errors.size(), equalTo(1));
         assertThat(errors.get(0), instanceOf(IllegalArgumentException.class));
         assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as any pattern in [foo, foobar]"));
@@ -158,17 +159,17 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
     }
 
 
-    private static List<Throwable> putTemplate(PutRequest request) {
+    private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) {
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
                 Settings.EMPTY,
                 null,
                 null,
                 null,
                 null,
-                null, null, null);
+                null, null, null, xContentRegistry);
         MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService,
                 new AliasValidator(Settings.EMPTY), null,
-                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS));
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry);
 
         final List<Throwable> throwables = new ArrayList<>();
         service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
@@ -196,10 +197,11 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             null,
             null,
             null,
-            null);
+            null,
+            xContentRegistry());
         MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
                 Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService,
-                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS));
+                new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry());
 
         final List<Throwable> throwables = new ArrayList<>();
         final CountDownLatch latch = new CountDownLatch(1);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
index 076245ad76d..48b2ae79cf3 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestTests.java
@@ -22,6 +22,9 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -66,4 +69,36 @@ public class PutIndexTemplateRequestTests extends ESTestCase {
         }
     }
 
+    public void testPutIndexTemplateRequestSerializationXContent() throws IOException {
+        PutIndexTemplateRequest request = new PutIndexTemplateRequest("foo");
+        String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+        request.patterns(Collections.singletonList("foo"));
+        request.mapping("bar", mapping, XContentType.YAML);
+        assertNotEquals(mapping, request.mappings().get("bar"));
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar"));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        PutIndexTemplateRequest serialized = new PutIndexTemplateRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), serialized.mappings().get("bar"));
+    }
+
+    public void testPutIndexTemplateRequestSerializationXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDAANmb28IdGVtcGxhdGUAAAAAAAABA2Jhcg8tLS0KZm9vOiAiYmFyIgoAAAAAAAAAAAAAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutIndexTemplateRequest request = new PutIndexTemplateRequest();
+            request.readFrom(in);
+            String mapping = YamlXContent.contentBuilder().startObject().field("foo", "bar").endObject().string();
+            assertNotEquals(mapping, request.mappings().get("bar"));
+            assertEquals(XContentHelper.convertToJson(new BytesArray(mapping), false, XContentType.YAML), request.mappings().get("bar"));
+            assertEquals("foo", request.name());
+            assertEquals("template", request.patterns().get(0));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
index 4300a629fbd..22377ea1769 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java
@@ -21,6 +21,7 @@
 package org.elasticsearch.action.bulk;
 
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.nio.charset.StandardCharsets;
@@ -31,7 +32,7 @@ public class BulkIntegrationIT extends ESIntegTestCase {
     public void testBulkIndexCreatesMapping() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/bulk-log.json");
         BulkRequestBuilder bulkBuilder = client().prepareBulk();
-        bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         bulkBuilder.get();
         assertBusy(new Runnable() {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
index 48d989209cb..ca1630b0064 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.bulk;
 
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -36,9 +37,9 @@ public class BulkProcessorClusterSettingsIT extends ESIntegTestCase {
         client().admin().cluster().prepareHealth("willwork").setWaitForGreenStatus().execute().actionGet();
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}"));
-        bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}"));
-        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}"));
+        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "1").setSource("{\"foo\":1}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("wontwork", "type1", "2").setSource("{\"foo\":2}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("willwork", "type1", "3").setSource("{\"foo\":3}", XContentType.JSON));
         BulkResponse br = bulkRequestBuilder.get();
         BulkItemResponse[] responses = br.getItems();
         assertEquals(3, responses.length);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
index 6bac7c2f8a4..7e608815c43 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -254,11 +255,13 @@ public class BulkProcessorIT extends ESIntegTestCase {
             for (int i = 1; i <= numDocs; i++) {
                 if (randomBoolean()) {
                     testDocs++;
-                    processor.add(new IndexRequest("test", "test", Integer.toString(testDocs)).source("field", "value"));
+                    processor.add(new IndexRequest("test", "test", Integer.toString(testDocs))
+                        .source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                     multiGetRequestBuilder.add("test", "test", Integer.toString(testDocs));
                 } else {
                     testReadOnlyDocs++;
-                    processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs)).source("field", "value"));
+                    processor.add(new IndexRequest("test-ro", "test", Integer.toString(testReadOnlyDocs))
+                        .source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
                 }
             }
         }
@@ -296,7 +299,8 @@ public class BulkProcessorIT extends ESIntegTestCase {
     private static MultiGetRequestBuilder indexDocs(Client client, BulkProcessor processor, int numDocs) {
         MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
         for (int i = 1; i <= numDocs; i++) {
-            processor.add(new IndexRequest("test", "test", Integer.toString(i)).source("field", randomRealisticUnicodeOfLengthBetween(1, 30)));
+            processor.add(new IndexRequest("test", "test", Integer.toString(i))
+                .source(Requests.INDEX_CONTENT_TYPE, "field", randomRealisticUnicodeOfLengthBetween(1, 30)));
             multiGetRequestBuilder.add("test", "test", Integer.toString(i));
         }
         return multiGetRequestBuilder;
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
index 4a41499d0d1..b6242e6d5fc 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestModifierTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
@@ -44,7 +45,7 @@ public class BulkRequestModifierTests extends ESTestCase {
         int numRequests = scaledRandomIntBetween(8, 64);
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
-            bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}"));
+            bulkRequest.add(new IndexRequest("_index", "_type", String.valueOf(i)).source("{}", XContentType.JSON));
         }
         CaptureActionListener actionListener = new CaptureActionListener();
         TransportBulkAction.BulkRequestModifier bulkRequestModifier = new TransportBulkAction.BulkRequestModifier(bulkRequest);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index a4a5f6f5ba6..a35a82ff096 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -30,9 +30,15 @@ import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -55,7 +61,7 @@ public class BulkRequestTests extends ESTestCase {
             bulkAction = Strings.replace(bulkAction, "\r\n", "\n");
         }
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
         assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }")));
         assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class));
@@ -65,21 +71,21 @@ public class BulkRequestTests extends ESTestCase {
     public void testSimpleBulk2() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
     }
 
     public void testSimpleBulk3() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk3.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
     }
 
     public void testSimpleBulk4() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk4.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(4));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2));
@@ -101,21 +107,21 @@ public class BulkRequestTests extends ESTestCase {
     public void testBulkAllowExplicitIndex() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json");
         try {
-            new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false);
+            new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), null, null, false, XContentType.JSON);
             fail();
         } catch (Exception e) {
 
         }
 
         bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk5.json");
-        new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false);
+        new BulkRequest().add(new BytesArray(bulkAction.getBytes(StandardCharsets.UTF_8)), "test", null, false, XContentType.JSON);
     }
 
     public void testBulkAddIterable() {
         BulkRequest bulkRequest = Requests.bulkRequest();
         List<DocWriteRequest> requests = new ArrayList<>();
-        requests.add(new IndexRequest("test", "test", "id").source("field", "value"));
-        requests.add(new UpdateRequest("test", "test", "id").doc("field", "value"));
+        requests.add(new IndexRequest("test", "test", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"));
+        requests.add(new UpdateRequest("test", "test", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value"));
         requests.add(new DeleteRequest("test", "test", "id"));
         bulkRequest.add(requests);
         assertThat(bulkRequest.requests().size(), equalTo(3));
@@ -128,7 +134,7 @@ public class BulkRequestTests extends ESTestCase {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk6.json");
         BulkRequest bulkRequest = new BulkRequest();
         ParsingException exc = expectThrows(ParsingException.class,
-            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null));
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
         assertThat(exc.getMessage(), containsString("Unknown key for a VALUE_STRING in [hello]"));
     }
 
@@ -136,7 +142,7 @@ public class BulkRequestTests extends ESTestCase {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk7.json");
         BulkRequest bulkRequest = new BulkRequest();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
-            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null));
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
         assertThat(exc.getMessage(),
             containsString("Malformed action/metadata line [5], expected a simple value for field [_unkown] but found [START_ARRAY]"));
     }
@@ -145,7 +151,7 @@ public class BulkRequestTests extends ESTestCase {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk8.json");
         BulkRequest bulkRequest = new BulkRequest();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
-            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null));
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
         assertThat(exc.getMessage(), containsString("Action/metadata line [3] contains an unknown parameter [_foo]"));
     }
 
@@ -153,14 +159,14 @@ public class BulkRequestTests extends ESTestCase {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk9.json");
         BulkRequest bulkRequest = new BulkRequest();
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
-            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null));
+            () -> bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON));
         assertThat(exc.getMessage(), containsString("Malformed action/metadata line [3], expected START_OBJECT or END_OBJECT but found [START_ARRAY]"));
     }
 
     public void testSimpleBulk10() throws Exception {
         String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk10.json");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         assertThat(bulkRequest.numberOfActions(), equalTo(9));
     }
 
@@ -172,8 +178,8 @@ public class BulkRequestTests extends ESTestCase {
         // We force here a "type is missing" validation error
         bulkRequest.add(new DeleteRequest("index", null, "id"));
         bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
-        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
-        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}", XContentType.JSON).setRefreshPolicy(RefreshPolicy.IMMEDIATE));
         ActionRequestValidationException validate = bulkRequest.validate();
         assertThat(validate, notNullValue());
         assertThat(validate.validationErrors(), not(empty()));
@@ -196,7 +202,8 @@ public class BulkRequestTests extends ESTestCase {
         assertThat(validate.validationErrors(), not(empty()));
         assertThat(validate.validationErrors(), contains(
                 "script or doc is missing",
-                "source is missing"));
+                "source is missing",
+                "content type is missing"));
     }
 
     public void testCannotAddNullRequests() throws Exception {
@@ -205,4 +212,41 @@ public class BulkRequestTests extends ESTestCase {
         expectThrows(NullPointerException.class, () -> bulkRequest.add((UpdateRequest) null));
         expectThrows(NullPointerException.class, () -> bulkRequest.add((DeleteRequest) null));
     }
+
+    public void testSmileIsSupported() throws IOException {
+        XContentType xContentType = XContentType.SMILE;
+        BytesReference data;
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            try(XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+                builder.startObject();
+                builder.startObject("index");
+                builder.field("_index", "index");
+                builder.field("_type", "type");
+                builder.field("_id", "test");
+                builder.endObject();
+                builder.endObject();
+            }
+            out.write(xContentType.xContent().streamSeparator());
+            try(XContentBuilder builder = XContentFactory.contentBuilder(xContentType, out)) {
+                builder.startObject();
+                builder.field("field", "value");
+                builder.endObject();
+            }
+            out.write(xContentType.xContent().streamSeparator());
+            data = out.bytes();
+        }
+
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(data, null, null);
+        assertEquals(1, bulkRequest.requests().size());
+        DocWriteRequest docWriteRequest = bulkRequest.requests().get(0);
+        assertEquals(DocWriteRequest.OpType.INDEX, docWriteRequest.opType());
+        assertEquals("index", docWriteRequest.index());
+        assertEquals("type", docWriteRequest.type());
+        assertEquals("test", docWriteRequest.id());
+        assertThat(docWriteRequest, instanceOf(IndexRequest.class));
+        IndexRequest request = (IndexRequest) docWriteRequest;
+        assertEquals(1, request.sourceAsMap().size());
+        assertEquals("value", request.sourceAsMap().get("field"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
index 82eee3554e8..b31426e3978 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
@@ -29,8 +29,10 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
@@ -209,9 +211,9 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
         assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L));
 
         bulkResponse = client().prepareBulk()
-                .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc("field", "2"))
-                .add(client().prepareUpdate("test", "type", "2").setDoc("field", "2"))
-                .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc("field", "3")).get();
+                .add(client().prepareUpdate("test", "type", "1").setVersion(4L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2"))
+                .add(client().prepareUpdate("test", "type", "2").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2"))
+                .add(client().prepareUpdate("test", "type", "1").setVersion(2L).setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3")).get();
 
         assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict"));
         assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(2L));
@@ -235,9 +237,9 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
 
         bulkResponse = client().prepareBulk()
                 .add(client().prepareUpdate("test", "type", "e1")
-                        .setDoc("field", "2").setVersion(10)) // INTERNAL
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "2").setVersion(10)) // INTERNAL
                 .add(client().prepareUpdate("test", "type", "e1")
-                        .setDoc("field", "3").setVersion(13).setVersionType(VersionType.INTERNAL))
+                        .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "3").setVersion(13).setVersionType(VersionType.INTERNAL))
                 .get();
 
         assertThat(bulkResponse.getItems()[0].getFailureMessage(), containsString("version conflict"));
@@ -493,8 +495,8 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
                 "}" +
                 "\n").array();
 
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChild, 0, addChild.length);
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChild, 0, addChild.length, XContentType.JSON);
 
         BulkResponse bulkResponse = builder.get();
         assertThat(bulkResponse.getItems().length, equalTo(2));
@@ -576,9 +578,9 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
                 "}" +
                 "\n").array();
 
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChild1, 0, addChild1.length);
-        builder.add(addChild2, 0, addChild2.length);
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChild1, 0, addChild1.length, XContentType.JSON);
+        builder.add(addChild2, 0, addChild2.length, XContentType.JSON);
 
         BulkResponse bulkResponse = builder.get();
         assertThat(bulkResponse.getItems().length, equalTo(3));
@@ -650,10 +652,10 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
                 "}" +
                 "\n").array();
 
-        builder.add(addParent, 0, addParent.length);
-        builder.add(addChildOK, 0, addChildOK.length);
-        builder.add(addChildMissingRouting, 0, addChildMissingRouting.length);
-        builder.add(addChildOK, 0, addChildOK.length);
+        builder.add(addParent, 0, addParent.length, XContentType.JSON);
+        builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON);
+        builder.add(addChildMissingRouting, 0, addChildMissingRouting.length, XContentType.JSON);
+        builder.add(addChildOK, 0, addChildOK.length, XContentType.JSON);
 
         BulkResponse bulkResponse = builder.get();
         assertThat(bulkResponse.getItems().length, equalTo(4));
@@ -680,7 +682,8 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
                     return;
                 }
                 BulkRequestBuilder requestBuilder = client().prepareBulk();
-                requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1).setDoc("field", threadID));
+                requestBuilder.add(client().prepareUpdate("test", "type", "1").setVersion(1)
+                    .setDoc(Requests.INDEX_CONTENT_TYPE, "field", threadID));
                 responses[threadID] = requestBuilder.get();
 
             });
@@ -733,19 +736,19 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
     // issue 6630
     public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception {
         BulkResponse indexBulkItemResponse = client().prepareBulk()
-                .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }"))
-                .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }"))
-                .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }"))
-                .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }"))
+                .add(new IndexRequest("test", "type", "3").source("{ \"title\" : \"Great Title of doc 3\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }", XContentType.JSON))
                 .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                 .get();
         assertNoFailures(indexBulkItemResponse);
 
         BulkResponse bulkItemResponse = client().prepareBulk()
-                .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }"))
-                .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }"))
-                .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}"))
-                .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}"))
+                .add(new IndexRequest("test", "type", "1").source("{ \"title\" : \"Great Title of doc 1\" }", XContentType.JSON))
+                .add(new IndexRequest("test", "type", "2").source("{ \"title\" : \"Great Title of doc 2\" }", XContentType.JSON))
+                .add(new UpdateRequest("test", "type", "3").doc("{ \"date\" : \"2014-01-30T23:59:57\"}", XContentType.JSON))
+                .add(new UpdateRequest("test", "type", "4").doc("{ \"date\" : \"2014-13-30T23:59:57\"}", XContentType.JSON))
                 .add(new DeleteRequest("test", "type", "5"))
                 .add(new DeleteRequest("test", "type", "6"))
         .get();
@@ -768,10 +771,10 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
     public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{
         createIndex("bulkindex1", "bulkindex2");
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1"))
-                   .add(new IndexRequest("bulkindex2", "index2_type", "1").source("text", "hallo2"))
-                   .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2"))
-                   .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar"))
+        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1"))
+                   .add(new IndexRequest("bulkindex2", "index2_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2"))
+                   .add(new IndexRequest("bulkindex2", "index2_type").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo2"))
+                   .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar"))
                    .add(new DeleteRequest("bulkindex2", "index2_type", "3"))
                    .setRefreshPolicy(RefreshPolicy.IMMEDIATE);
 
@@ -794,8 +797,8 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareClose("bulkindex1"));
 
         BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1"))
-                .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar"))
+        bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source(Requests.INDEX_CONTENT_TYPE, "text", "hallo1"))
+                .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc(Requests.INDEX_CONTENT_TYPE, "foo", "bar"))
                 .add(new DeleteRequest("bulkindex1", "index1_type", "1"));
 
         BulkResponse bulkResponse = client().bulk(bulkRequest).get();
@@ -810,8 +813,10 @@ public class BulkWithUpdatesIT extends ESIntegTestCase {
     // issue 9821
     public void testInvalidIndexNamesCorrectOpType() {
         BulkResponse bulkResponse = client().prepareBulk()
-                .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1").setSource("field", 1))
-                .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1").setDoc("field", randomInt()))
+                .add(client().prepareIndex().setIndex("INVALID.NAME").setType("type1").setId("1")
+                    .setSource(Requests.INDEX_CONTENT_TYPE, "field", 1))
+                .add(client().prepareUpdate().setIndex("INVALID.NAME").setType("type1").setId("1")
+                    .setDoc(Requests.INDEX_CONTENT_TYPE, "field", randomInt()))
                 .add(client().prepareDelete().setIndex("INVALID.NAME").setType("type1").setId("1")).get();
         assertThat(bulkResponse.getItems().length, is(3));
         assertThat(bulkResponse.getItems()[0].getOpType(), is(OpType.INDEX));
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
index 132b770ca15..b2483bd9306 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java
@@ -21,21 +21,27 @@ package org.elasticsearch.action.bulk;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineExecutionService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 import org.junit.Before;
@@ -48,6 +54,7 @@ import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.sameInstance;
@@ -84,6 +91,9 @@ public class TransportBulkActionIngestTests extends ESTestCase {
     /** The actual action we want to test, with real indexing mocked */
     TestTransportBulkAction action;
 
+    /** Single item bulk write action that wraps index requests */
+    TestSingleItemBulkWriteAction singleItemBulkWriteAction;
+
     /** True if the next call to the index action should act as an ingest node */
     boolean localIngest;
 
@@ -110,6 +120,20 @@ public class TransportBulkActionIngestTests extends ESTestCase {
         }
     }
 
+    class TestSingleItemBulkWriteAction extends TransportSingleItemBulkWriteAction<IndexRequest, IndexResponse> {
+
+        TestSingleItemBulkWriteAction(TestTransportBulkAction bulkAction) {
+            super(Settings.EMPTY, IndexAction.NAME, transportService, TransportBulkActionIngestTests.this.clusterService,
+                    null, null, null, new ActionFilters(Collections.emptySet()), null,
+                    IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX, bulkAction, null);
+        }
+
+        @Override
+        protected IndexResponse newResponseInstance() {
+            return new IndexResponse();
+        }
+    }
+
     @Before
     public void setupAction() {
         // initialize captors, which must be members to use @Capture because of generics
@@ -142,6 +166,7 @@ public class TransportBulkActionIngestTests extends ESTestCase {
         executionService = mock(PipelineExecutionService.class);
         when(ingestService.getPipelineExecutionService()).thenReturn(executionService);
         action = new TestTransportBulkAction();
+        singleItemBulkWriteAction = new TestSingleItemBulkWriteAction(action);
         reset(transportService); // call on construction of action
     }
 
@@ -157,6 +182,16 @@ public class TransportBulkActionIngestTests extends ESTestCase {
         verifyZeroInteractions(ingestService);
     }
 
+    public void testSingleItemBulkActionIngestSkipped() throws Exception {
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(response -> {}, exception -> {
+            throw new AssertionError(exception);
+        }));
+        assertTrue(action.isExecuted);
+        verifyZeroInteractions(ingestService);
+    }
+
     public void testIngestLocal() throws Exception {
         Exception exception = new Exception("fake exception");
         BulkRequest bulkRequest = new BulkRequest();
@@ -200,6 +235,38 @@ public class TransportBulkActionIngestTests extends ESTestCase {
         verifyZeroInteractions(transportService);
     }
 
+    public void testSingleItemBulkActionIngestLocal() throws Exception {
+        Exception exception = new Exception("fake exception");
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        indexRequest.setPipeline("testpipeline");
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        AtomicBoolean failureCalled = new AtomicBoolean(false);
+        singleItemBulkWriteAction.execute(null, indexRequest, ActionListener.wrap(
+                response -> {
+                    responseCalled.set(true);
+                },
+                e -> {
+                    assertThat(e, sameInstance(exception));
+                    failureCalled.set(true);
+                }));
+
+        // check failure works, and passes through to the listener
+        assertFalse(action.isExecuted); // haven't executed yet
+        assertFalse(responseCalled.get());
+        assertFalse(failureCalled.get());
+        verify(executionService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture());
+        completionHandler.getValue().accept(exception);
+        assertTrue(failureCalled.get());
+
+        // now check success
+        indexRequest.setPipeline(null); // this is done by the real pipeline execution service when processing
+        completionHandler.getValue().accept(null);
+        assertTrue(action.isExecuted);
+        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
+        verifyZeroInteractions(transportService);
+    }
+
     public void testIngestForward() throws Exception {
         localIngest = false;
         BulkRequest bulkRequest = new BulkRequest();
@@ -246,5 +313,51 @@ public class TransportBulkActionIngestTests extends ESTestCase {
         }
     }
 
+    public void testSingleItemBulkActionIngestForward() throws Exception {
+        localIngest = false;
+        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
+        indexRequest.source(Collections.emptyMap());
+        indexRequest.setPipeline("testpipeline");
+        IndexResponse indexResponse = mock(IndexResponse.class);
+        AtomicBoolean responseCalled = new AtomicBoolean(false);
+        ActionListener<IndexResponse> listener = ActionListener.wrap(
+                response -> {
+                    responseCalled.set(true);
+                    assertSame(indexResponse, response);
+                },
+                e -> {
+                    throw new AssertionError(e);
+                });
+        singleItemBulkWriteAction.execute(null, indexRequest, listener);
+
+        // should not have executed ingest locally
+        verify(executionService, never()).executeBulkRequest(any(), any(), any());
+        // but instead should have sent to a remote node with the transport service
+        ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        boolean usedNode1 = node.getValue() == remoteNode1; // make sure we used one of the nodes
+        if (usedNode1 == false) {
+            assertSame(remoteNode2, node.getValue());
+        }
+        assertFalse(action.isExecuted); // no local index execution
+        assertFalse(responseCalled.get()); // listener not called yet
+
+        BulkItemResponse itemResponse = new BulkItemResponse(0, DocWriteRequest.OpType.CREATE, indexResponse);
+        BulkItemResponse[] bulkItemResponses = new BulkItemResponse[1];
+        bulkItemResponses[0] = itemResponse;
+        remoteResponseHandler.getValue().handleResponse(new BulkResponse(bulkItemResponses, 0)); // call the listener for the remote node
+        assertTrue(responseCalled.get()); // now the listener we passed should have been delegated to by the remote listener
+        assertFalse(action.isExecuted); // still no local index execution
+
+        // now make sure ingest nodes are rotated through with a subsequent request
+        reset(transportService);
+        singleItemBulkWriteAction.execute(null, indexRequest, listener);
+        verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
+        if (usedNode1) {
+            assertSame(remoteNode2, node.getValue());
+        } else {
+            assertSame(remoteNode1, node.getValue());
+        }
+    }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index b8c29bbf174..0e3d96b5e8e 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
@@ -88,7 +89,8 @@ public class TransportBulkActionTookTests extends ESTestCase {
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
         TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         IndexNameExpressionResolver resolver = new Resolver(Settings.EMPTY);
@@ -116,11 +118,6 @@ public class TransportBulkActionTookTests extends ESTestCase {
                     resolver,
                     null,
                     expected::get) {
-                @Override
-                public void executeBulk(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-                    expected.set(1000000);
-                    super.executeBulk(bulkRequest, listener);
-                }
 
                 @Override
                 void executeBulk(
@@ -145,12 +142,6 @@ public class TransportBulkActionTookTests extends ESTestCase {
                     resolver,
                     null,
                     System::nanoTime) {
-                @Override
-                public void executeBulk(BulkRequest bulkRequest, ActionListener<BulkResponse> listener) {
-                    long elapsed = spinForAtLeastOneMillisecond();
-                    expected.set(elapsed);
-                    super.executeBulk(bulkRequest, listener);
-                }
 
                 @Override
                 void executeBulk(
@@ -184,7 +175,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
             bulkAction = Strings.replace(bulkAction, "\r\n", "\n");
         }
         BulkRequest bulkRequest = new BulkRequest();
-        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
+        bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null, XContentType.JSON);
         AtomicLong expected = new AtomicLong();
         TransportBulkAction action = createAction(controlled, expected);
         action.doExecute(null, bulkRequest, new ActionListener<BulkResponse>() {
@@ -209,7 +200,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
     }
 
     static class Resolver extends IndexNameExpressionResolver {
-        public Resolver(Settings settings) {
+        Resolver(Settings settings) {
             super(settings);
         }
 
@@ -221,7 +212,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
 
     static class TestTransportBulkAction extends TransportBulkAction {
 
-        public TestTransportBulkAction(
+        TestTransportBulkAction(
                 Settings settings,
                 ThreadPool threadPool,
                 TransportService transportService,
@@ -260,7 +251,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
 
     static class TestTransportCreateIndexAction extends TransportCreateIndexAction {
 
-        public TestTransportCreateIndexAction(
+        TestTransportCreateIndexAction(
                 Settings settings,
                 TransportService transportService,
                 ClusterService clusterService,
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java
similarity index 88%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java
index 932d4de9174..6b3ca0d2815 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/AsyncBulkByScrollActionTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
@@ -34,6 +34,13 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource;
+import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Hit;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -55,15 +62,15 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.FilterClient;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.internal.InternalSearchHit;
@@ -119,7 +126,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     private MyMockClient client;
     private DummyAbstractBulkByScrollRequest testRequest;
     private SearchRequest firstSearchRequest;
-    private PlainActionFuture<BulkIndexByScrollResponse> listener;
+    private PlainActionFuture<BulkByScrollResponse> listener;
     private String scrollId;
     private TaskManager taskManager;
     private WorkingBulkByScrollTask testTask;
@@ -150,7 +157,6 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             client.close();
         }
         client = new MyMockClient(new NoOpClient(threadPool));
-        client.threadPool().getThreadContext().newStoredContext();
         client.threadPool().getThreadContext().putHeader(expectedHeaders);
     }
 
@@ -171,7 +177,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testStartRetriesOnRejectionAndSucceeds() throws Exception {
         client.searchesToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.start();
         assertBusy(() -> assertEquals(client.searchesToReject + 1, client.searchAttempts.get()));
         if (listener.isDone()) {
@@ -184,7 +190,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
         client.searchesToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.start();
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.searchAttempts.get()));
         assertBusy(() -> assertTrue(listener.isDone()));
@@ -196,7 +202,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testStartNextScrollRetriesOnRejectionAndSucceeds() throws Exception {
         client.scrollsToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
         action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
         assertBusy(() -> assertEquals(client.scrollsToReject + 1, client.scrollAttempts.get()));
@@ -210,7 +216,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testStartNextScrollRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
         client.scrollsToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
         action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.scrollAttempts.get()));
@@ -227,7 +233,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
         long total = randomIntBetween(0, Integer.MAX_VALUE);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
         assertEquals(total, testTask.getStatus().getTotal());
     }
 
@@ -239,7 +245,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         for (int batches = 1; batches < maxBatches; batches++) {
             Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
             ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
-            DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+            DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
             simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
 
             // Use assert busy because the update happens on another thread
@@ -292,7 +298,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
                     opType,
                     new IndexResponse(shardId, "type", "id" + i, randomInt(20), randomInt(), createdResponse));
             }
-            new DummyAbstractAsyncBulkByScrollAction().onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(responses, 0));
+            new DummyAsyncBulkByScrollAction().onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(responses, 0));
             assertEquals(versionConflicts, testTask.getStatus().getVersionConflicts());
             assertEquals(updated, testTask.getStatus().getUpdated());
             assertEquals(created, testTask.getStatus().getCreated());
@@ -317,7 +323,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             }
         });
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(e.getMessage(), equalTo("EsRejectedExecutionException[test]"));
         assertThat(client.scrollsCleared, contains(scrollId));
@@ -334,8 +340,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         SearchFailure shardFailure = new SearchFailure(new RuntimeException("test"));
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(false, singletonList(shardFailure), 0,
                 emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), contains(shardFailure));
         assertFalse(response.isTimedOut());
@@ -348,8 +354,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      */
     public void testSearchTimeoutsAbortRequest() throws Exception {
         ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(true, emptyList(), 0, emptyList(), null);
-        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        simulateScrollResponse(new DummyAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), empty());
         assertTrue(response.isTimedOut());
@@ -362,11 +368,11 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      */
     public void testBulkFailuresAbortRequest() throws Exception {
         Failure failure = new Failure("index", "type", "id", new RuntimeException("test"));
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[]
             {new BulkItemResponse(0, DocWriteRequest.OpType.CREATE, failure)}, randomLong());
         action.onBulkResponse(timeValueNanos(System.nanoTime()), bulkResponse);
-        BulkIndexByScrollResponse response = listener.get();
+        BulkByScrollResponse response = listener.get();
         assertThat(response.getBulkFailures(), contains(failure));
         assertThat(response.getSearchFailures(), empty());
         assertNull(response.getReasonCancelled());
@@ -375,14 +381,15 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     /**
      * Mimicks script failures or general wrongness by implementers.
      */
-    public void testListenerReceiveBuildBulkExceptions() throws Exception {
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction() {
+    public void testBuildRequestThrowsException() throws Exception {
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction() {
             @Override
-            protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
+            protected AbstractAsyncBulkByScrollAction.RequestWrapper<?> buildRequest(Hit doc) {
                 throw new RuntimeException("surprise");
             }
         };
-        Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+        ScrollableHitSource.BasicHit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+        hit.setSource(new BytesArray("{}"), XContentType.JSON);
         ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
         simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
@@ -427,7 +434,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             }
         });
 
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         action.setScroll(scrollId());
 
         // Set the base for the scroll to wait - this is added to the figure we calculate below
@@ -479,7 +486,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
          * deal with it. We just wait for it to happen.
          */
         CountDownLatch successLatch = new CountDownLatch(1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
+        DummyAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
             @Override
             void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
                 successLatch.countDown();
@@ -491,7 +498,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
         action.sendBulkRequest(timeValueNanos(System.nanoTime()), request);
         if (failWithRejection) {
-            BulkIndexByScrollResponse response = listener.get();
+            BulkByScrollResponse response = listener.get();
             assertThat(response.getBulkFailures(), hasSize(1));
             assertEquals(response.getBulkFailures().get(0).getStatus(), RestStatus.TOO_MANY_REQUESTS);
             assertThat(response.getSearchFailures(), empty());
@@ -505,7 +512,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * The default retry time matches what we say it is in the javadoc for the request.
      */
     public void testDefaultRetryTimes() {
-        Iterator<TimeValue> policy = new DummyAbstractAsyncBulkByScrollAction().buildBackoffPolicy().iterator();
+        Iterator<TimeValue> policy = new DummyAsyncBulkByScrollAction().buildBackoffPolicy().iterator();
         long millis = 0;
         while (policy.hasNext()) {
             millis += policy.next().millis();
@@ -538,7 +545,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         if (refresh != null) {
             testRequest.setRefresh(refresh);
         }
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         if (addDestinationIndexes) {
             action.addDestinationIndices(singleton("foo"));
         }
@@ -551,34 +558,34 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     }
 
     public void testCancelBeforeInitialSearch() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.start());
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.start());
     }
 
     public void testCancelBeforeScrollResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
                 null));
     }
 
     public void testCancelBeforeSendBulkRequest() throws Exception {
         // We bail so early we don't need to pass in a half way valid request.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.sendBulkRequest(timeValueNanos(System.nanoTime()), null));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.sendBulkRequest(timeValueNanos(System.nanoTime()), null));
     }
 
     public void testCancelBeforeOnBulkResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) ->
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) ->
                 action.onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(new BulkItemResponse[0], 0)));
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(timeValueNanos(System.nanoTime()), 0));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.startNextScroll(timeValueNanos(System.nanoTime()), 0));
     }
 
     public void testCancelBeforeRefreshAndFinish() throws Exception {
         // Refresh or not doesn't matter - we don't try to refresh.
         testRequest.setRefresh(usually());
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.refreshAndFinish(emptyList(), emptyList(), false));
+        cancelTaskCase((DummyAsyncBulkByScrollAction action) -> action.refreshAndFinish(emptyList(), emptyList(), false));
         assertNull("No refresh was attempted", client.lastRefreshRequest.get());
     }
 
@@ -604,14 +611,14 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
                  * that good stuff.
                  */
                 if (delay.nanos() > 0) {
-                    generic().execute(() -> taskManager.cancel(testTask, reason, (Set<String> s) -> {}));
+                    generic().execute(() -> taskManager.cancel(testTask, reason, () -> {}));
                 }
                 return super.schedule(delay, name, command);
             }
         });
 
         // Send the scroll response which will trigger the custom thread pool above, canceling the request before running the response
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         boolean previousScrollSet = usually();
         if (previousScrollSet) {
             action.setScroll(scrollId());
@@ -630,14 +637,14 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
     }
 
-    private void cancelTaskCase(Consumer<DummyAbstractAsyncBulkByScrollAction> testMe) throws Exception {
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
+    private void cancelTaskCase(Consumer<DummyAsyncBulkByScrollAction> testMe) throws Exception {
+        DummyAsyncBulkByScrollAction action = new DummyAsyncBulkByScrollAction();
         boolean previousScrollSet = usually();
         if (previousScrollSet) {
             action.setScroll(scrollId());
         }
         String reason = randomSimpleString(random());
-        taskManager.cancel(testTask, reason, (Set<String> s) -> {});
+        taskManager.cancel(testTask, reason, () -> {});
         testMe.accept(action);
         assertEquals(reason, listener.get().getReasonCancelled());
         if (previousScrollSet) {
@@ -649,17 +656,16 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     /**
      * Simulate a scroll response by setting the scroll id and firing the onScrollResponse method.
      */
-    private void simulateScrollResponse(DummyAbstractAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
+    private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
             ScrollableHitSource.Response response) {
         action.setScroll(scrollId());
         action.onScrollResponse(lastBatchTime, lastBatchSize, response);
     }
 
-    private class DummyAbstractAsyncBulkByScrollAction
-            extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
-        public DummyAbstractAsyncBulkByScrollAction() {
+    private class DummyAsyncBulkByScrollAction extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
+        DummyAsyncBulkByScrollAction() {
             super(testTask, AsyncBulkByScrollActionTests.this.logger, new ParentTaskAssigningClient(client, localNode, testTask),
-                    client.threadPool(), testRequest, listener);
+                    client.threadPool(), testRequest, null, null, listener);
         }
 
         @Override
@@ -668,15 +674,15 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
 
         @Override
-        protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
-            return new BulkRequest();
+        protected AbstractAsyncBulkByScrollAction.RequestWrapper<?> buildRequest(Hit doc) {
+            throw new UnsupportedOperationException("Use another override to test this.");
         }
     }
 
     /**
-     * An extension to {@linkplain DummyAbstractAsyncBulkByScrollAction} that uses a 0 delaying backoff policy.
+     * An extension to {@linkplain DummyAsyncBulkByScrollAction} that uses a 0 delaying backoff policy.
      */
-    private class DummyActionWithoutBackoff extends DummyAbstractAsyncBulkByScrollAction {
+    private class DummyActionWithoutBackoff extends DummyAsyncBulkByScrollAction {
         @Override
         BackoffPolicy buildBackoffPolicy() {
             // Force a backoff time of 0 to prevent sleeping
@@ -685,12 +691,12 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     }
 
     private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScrollRequest<DummyAbstractBulkByScrollRequest> {
-        public DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) {
+        DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) {
             super(searchRequest, true);
         }
 
         @Override
-        DummyAbstractBulkByScrollRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+        protected DummyAbstractBulkByScrollRequest forSlice(TaskId slicingTask, SearchRequest slice) {
             throw new UnsupportedOperationException();
         }
 
@@ -723,7 +729,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         private int searchesToReject = 0;
         private int scrollsToReject = 0;
 
-        public MyMockClient(Client in) {
+        MyMockClient(Client in) {
             super(in);
         }
 
@@ -850,7 +856,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         private final Request request;
         private final ActionListener<Response> listener;
 
-        public RequestAndListener(Request request, ActionListener<Response> listener) {
+        RequestAndListener(Request request, ActionListener<Response> listener) {
             this.request = request;
             this.listener = listener;
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParallelizationHelperTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelperTests.java
similarity index 89%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParallelizationHelperTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelperTests.java
index f1effa70ca8..e9291aa381d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParallelizationHelperTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollParallelizationHelperTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.index.mapper.UidFieldMapper;
@@ -27,17 +27,18 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.IOException;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.index.reindex.ReindexParallelizationHelper.sliceIntoSubRequests;
+import static org.elasticsearch.action.bulk.byscroll.BulkByScrollParallelizationHelper.sliceIntoSubRequests;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchSourceBuilder;
 
-public class ReindexParallelizationHelperTests extends ESTestCase {
+public class BulkByScrollParallelizationHelperTests extends ESTestCase {
     public void testSliceIntoSubRequests() throws IOException {
         SearchRequest searchRequest = randomSearchRequest(() -> randomSearchSourceBuilder(
                 () -> null,
                 () -> null,
                 () -> null,
-                () -> emptyList()));
+                () -> emptyList(),
+                () -> null));
         if (searchRequest.source() != null) {
             // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is.
             searchRequest.source().slice(null);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponseTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponseTests.java
new file mode 100644
index 00000000000..bb5d8bf26f6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollResponseTests.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk.byscroll;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public class BulkByScrollResponseTests extends ESTestCase {
+
+    public void testRountTrip() throws IOException {
+        BulkByScrollResponse response = new BulkByScrollResponse(timeValueMillis(randomNonNegativeLong()),
+                BulkByScrollTaskStatusTests.randomStatus(), randomIndexingFailures(), randomSearchFailures(), randomBoolean());
+        BulkByScrollResponse tripped = new BulkByScrollResponse();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            response.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                tripped.readFrom(in);
+            }
+        }
+        assertResponseEquals(response, tripped);
+    }
+
+    private List<Failure> randomIndexingFailures() {
+        return usually() ? emptyList()
+                : singletonList(new Failure(randomSimpleString(random()), randomSimpleString(random()),
+                        randomSimpleString(random()), new IllegalArgumentException("test")));
+    }
+
+    private List<SearchFailure> randomSearchFailures() {
+        if (randomBoolean()) {
+            return emptyList();
+        }
+        String index = null;
+        Integer shardId = null;
+        String nodeId = null;
+        if (randomBoolean()) {
+            index = randomAsciiOfLength(5);
+            shardId = randomInt();
+            nodeId = usually() ? randomAsciiOfLength(5) : null;
+        }
+        return singletonList(new SearchFailure(new ElasticsearchException("foo"), index, shardId, nodeId));
+    }
+
+    private void assertResponseEquals(BulkByScrollResponse expected, BulkByScrollResponse actual) {
+        assertEquals(expected.getTook(), actual.getTook());
+        BulkByScrollTaskStatusTests.assertTaskStatusEquals(Version.CURRENT, expected.getStatus(), actual.getStatus());
+        assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
+        for (int i = 0; i < expected.getBulkFailures().size(); i++) {
+            Failure expectedFailure = expected.getBulkFailures().get(i);
+            Failure actualFailure = actual.getBulkFailures().get(i);
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getType(), actualFailure.getType());
+            assertEquals(expectedFailure.getId(), actualFailure.getId());
+            assertEquals(expectedFailure.getMessage(), actualFailure.getMessage());
+            assertEquals(expectedFailure.getStatus(), actualFailure.getStatus());
+        }
+        assertEquals(expected.getSearchFailures().size(), actual.getSearchFailures().size());
+        for (int i = 0; i < expected.getSearchFailures().size(); i++) {
+            SearchFailure expectedFailure = expected.getSearchFailures().get(i);
+            SearchFailure actualFailure = actual.getSearchFailures().get(i);
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getShardId(), actualFailure.getShardId());
+            assertEquals(expectedFailure.getNodeId(), actualFailure.getNodeId());
+            assertEquals(expectedFailure.getReason().getClass(), actualFailure.getReason().getClass());
+            assertEquals(expectedFailure.getReason().getMessage(), actualFailure.getReason().getMessage());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskStatusTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskStatusTests.java
new file mode 100644
index 00000000000..b621264e7ea
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskStatusTests.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk.byscroll;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.IntStream;
+
+import static java.lang.Math.abs;
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.hamcrest.Matchers.hasSize;
+
+public class BulkByScrollTaskStatusTests extends ESTestCase {
+    public void testBulkByTaskStatus() throws IOException {
+        BulkByScrollTask.Status status = randomStatus();
+        BytesStreamOutput out = new BytesStreamOutput();
+        status.writeTo(out);
+        BulkByScrollTask.Status tripped = new BulkByScrollTask.Status(out.bytes().streamInput());
+        assertTaskStatusEquals(out.getVersion(), status, tripped);
+
+        // Also check round tripping pre-5.1 which is the first version to support parallelized scroll
+        out = new BytesStreamOutput();
+        out.setVersion(Version.V_5_0_0_rc1); // This can be V_5_0_0
+        status.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        in.setVersion(Version.V_5_0_0_rc1);
+        tripped = new BulkByScrollTask.Status(in);
+        assertTaskStatusEquals(Version.V_5_0_0_rc1, status, tripped);
+    }
+
+    /**
+     * Assert that two task statuses are equal after serialization.
+     * @param version the version at which expected was serialized
+     */
+    public static void assertTaskStatusEquals(Version version, BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
+        assertEquals(expected.getTotal(), actual.getTotal());
+        assertEquals(expected.getUpdated(), actual.getUpdated());
+        assertEquals(expected.getCreated(), actual.getCreated());
+        assertEquals(expected.getDeleted(), actual.getDeleted());
+        assertEquals(expected.getBatches(), actual.getBatches());
+        assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
+        assertEquals(expected.getNoops(), actual.getNoops());
+        assertEquals(expected.getBulkRetries(), actual.getBulkRetries());
+        assertEquals(expected.getSearchRetries(), actual.getSearchRetries());
+        assertEquals(expected.getThrottled(), actual.getThrottled());
+        assertEquals(expected.getRequestsPerSecond(), actual.getRequestsPerSecond(), 0f);
+        assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
+        assertEquals(expected.getThrottledUntil(), actual.getThrottledUntil());
+        if (version.onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+            assertThat(actual.getSliceStatuses(), hasSize(expected.getSliceStatuses().size()));
+            for (int i = 0; i < expected.getSliceStatuses().size(); i++) {
+                BulkByScrollTask.StatusOrException sliceStatus = expected.getSliceStatuses().get(i);
+                if (sliceStatus == null) {
+                    assertNull(actual.getSliceStatuses().get(i));
+                } else if (sliceStatus.getException() == null) {
+                    assertNull(actual.getSliceStatuses().get(i).getException());
+                    assertTaskStatusEquals(version, sliceStatus.getStatus(), actual.getSliceStatuses().get(i).getStatus());
+                } else {
+                    assertNull(actual.getSliceStatuses().get(i).getStatus());
+                    // Just check the message because we're not testing exception serialization in general here.
+                    assertEquals(sliceStatus.getException().getMessage(), actual.getSliceStatuses().get(i).getException().getMessage());
+                }
+            }
+        } else {
+            assertEquals(emptyList(), actual.getSliceStatuses());
+        }
+    }
+
+    public static BulkByScrollTask.Status randomStatus() {
+        if (randomBoolean()) {
+            return randomWorkingStatus(null);
+        }
+        boolean canHaveNullStatues = randomBoolean();
+        List<BulkByScrollTask.StatusOrException> statuses = IntStream.range(0, between(0, 10))
+                .mapToObj(i -> {
+                    if (canHaveNullStatues && LuceneTestCase.rarely()) {
+                        return null;
+                    }
+                    if (randomBoolean()) {
+                        return new BulkByScrollTask.StatusOrException(new ElasticsearchException(randomAsciiOfLength(5)));
+                    }
+                    return new BulkByScrollTask.StatusOrException(randomWorkingStatus(i));
+                })
+                .collect(toList());
+        return new BulkByScrollTask.Status(statuses, randomBoolean() ? "test" : null);
+    }
+
+    private static BulkByScrollTask.Status randomWorkingStatus(Integer sliceId) {
+        // These all should be believably small because we sum them if we have multiple workers
+        int total = between(0, 10000000);
+        int updated = between(0, total);
+        int created = between(0, total - updated);
+        int deleted = between(0, total - updated - created);
+        int noops = total - updated - created - deleted;
+        int batches = between(0, 10000);
+        long versionConflicts = between(0, total);
+        long bulkRetries = between(0, 10000000);
+        long searchRetries = between(0, 100000);
+        return new BulkByScrollTask.Status(sliceId, total, updated, created, deleted, batches, versionConflicts, noops, bulkRetries,
+                searchRetries, parseTimeValue(randomPositiveTimeValue(), "test"), abs(Randomness.get().nextFloat()),
+                randomBoolean() ? null : randomSimpleString(Randomness.get()), parseTimeValue(randomPositiveTimeValue(), "test"));
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskTests.java
similarity index 98%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskTests.java
index 54e1706fd03..0d353e426ec 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/BulkByScrollTaskTests.java
@@ -17,8 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequestTests.java
similarity index 93%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequestTests.java
index 187dfb847f6..ac7976e95c5 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/DeleteByQueryRequestTests.java
@@ -17,8 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestTestCase;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTaskTests.java
similarity index 74%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTaskTests.java
index f764cc60917..715fcaaad54 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ParentBulkByScrollTaskTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/ParentBulkByScrollTaskTests.java
@@ -17,20 +17,21 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
+import org.mockito.ArgumentCaptor;
 
 import java.util.Arrays;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.index.reindex.TransportRethrottleActionTests.captureResponse;
-import static org.elasticsearch.index.reindex.TransportRethrottleActionTests.neverCalled;
+import static org.mockito.Mockito.atMost;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
 public class ParentBulkByScrollTaskTests extends ESTestCase {
     private int slices;
@@ -88,9 +89,9 @@ public class ParentBulkByScrollTaskTests extends ESTestCase {
             sliceStatuses.set(slice, new BulkByScrollTask.StatusOrException(sliceStatus));
 
             @SuppressWarnings("unchecked")
-            ActionListener<BulkIndexByScrollResponse> listener = slice < slices - 1 ? neverCalled() : mock(ActionListener.class);
+            ActionListener<BulkByScrollResponse> listener = slice < slices - 1 ? neverCalled() : mock(ActionListener.class);
             task.onSliceResponse(listener, slice,
-                    new BulkIndexByScrollResponse(timeValueMillis(10), sliceStatus, emptyList(), emptyList(), false));
+                    new BulkByScrollResponse(timeValueMillis(10), sliceStatus, emptyList(), emptyList(), false));
 
             status = task.getStatus();
             assertEquals(total, status.getTotal());
@@ -104,7 +105,7 @@ public class ParentBulkByScrollTaskTests extends ESTestCase {
 
             if (slice == slices - 1) {
                 // The whole thing succeeded so we should have got the success
-                status = captureResponse(BulkIndexByScrollResponse.class, listener).getStatus();
+                status = captureResponse(BulkByScrollResponse.class, listener).getStatus();
                 assertEquals(total, status.getTotal());
                 assertEquals(created, status.getCreated());
                 assertEquals(updated, status.getUpdated());
@@ -117,5 +118,31 @@ public class ParentBulkByScrollTaskTests extends ESTestCase {
         }
     }
 
+    private <T> ActionListener<T> neverCalled() {
+        return new ActionListener<T>() {
+            @Override
+            public void onResponse(T response) {
+                throw new RuntimeException("Expected no interactions but got [" + response + "]");
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                throw new RuntimeException("Expected no interations but was received a failure", e);
+            }
+        };
+    }
+
+
+    private <T> T captureResponse(Class<T> responseClass, ActionListener<T> listener) {
+        ArgumentCaptor<Exception> failure = ArgumentCaptor.forClass(Exception.class);
+        // Rethrow any failures just so we get a nice exception if there were any. We don't expect any though.
+        verify(listener, atMost(1)).onFailure(failure.capture());
+        if (false == failure.getAllValues().isEmpty()) {
+            throw new AssertionError(failure.getValue());
+        }
+        ArgumentCaptor<T> response = ArgumentCaptor.forClass(responseClass);
+        verify(listener).onResponse(response.capture());
+        return response.getValue();
+    }
 
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTaskTests.java
similarity index 98%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java
rename to core/src/test/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTaskTests.java
index 389b5a29142..27d7a24312b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/WorkingBulkByScrollTaskTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/byscroll/WorkingBulkByScrollTaskTests.java
@@ -17,8 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.tasks.TaskId;
diff --git a/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java b/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
new file mode 100644
index 00000000000..c78fec37f88
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/delete/DeleteResponseTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.delete;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.action.index.IndexResponseTests.assertDocWriteResponse;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+
+public class DeleteResponseTests extends ESTestCase {
+
+    public void testToXContent() {
+        {
+            DeleteResponse response = new DeleteResponse(new ShardId("index", "index_uuid", 0), "type", "id", 3, 5, true);
+            String output = Strings.toString(response);
+            assertEquals("{\"found\":true,\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":5,\"result\":\"deleted\"," +
+                "\"_shards\":null,\"_seq_no\":3}", output);
+        }
+        {
+            DeleteResponse response = new DeleteResponse(new ShardId("index", "index_uuid", 0), "type", "id", -1, 7, true);
+            response.setForcedRefresh(true);
+            response.setShardInfo(new ReplicationResponse.ShardInfo(10, 5));
+            String output = Strings.toString(response);
+            assertEquals("{\"found\":true,\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":7,\"result\":\"deleted\"," +
+                "\"forced_refresh\":true,\"_shards\":{\"total\":10,\"successful\":5,\"failed\":0}}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        // Create a random DeleteResponse and converts it to XContent in bytes
+        DeleteResponse deleteResponse = randomDeleteResponse();
+        boolean humanReadable = randomBoolean();
+        BytesReference deleteResponseBytes = toXContent(deleteResponse, xContentType, humanReadable);
+
+        // Parse the XContent bytes to obtain a parsed
+        DeleteResponse parsedDeleteResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), deleteResponseBytes)) {
+            parsedDeleteResponse = DeleteResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        // We can't use equals() to compare the original and the parsed delete response
+        // because the random delete response can contain shard failures with exceptions,
+        // and those exceptions are not parsed back with the same types.
+
+        // Print the parsed object out and test that the output is the same as the original output
+        BytesReference parsedDeleteResponseBytes = toXContent(parsedDeleteResponse, xContentType, humanReadable);
+        try (XContentParser parser = createParser(xContentType.xContent(), parsedDeleteResponseBytes)) {
+            assertDeleteResponse(deleteResponse, parser.map());
+        }
+    }
+
+    private static void assertDeleteResponse(DeleteResponse expected, Map<String, Object> actual) {
+        assertDocWriteResponse(expected, actual);
+        if (expected.getResult() == DocWriteResponse.Result.DELETED) {
+            assertTrue((boolean) actual.get("found"));
+        } else {
+            assertFalse((boolean) actual.get("found"));
+        }
+    }
+
+    private static DeleteResponse randomDeleteResponse() {
+        ShardId shardId = new ShardId(randomAsciiOfLength(5), randomAsciiOfLength(5), randomIntBetween(0, 5));
+        String type = randomAsciiOfLength(5);
+        String id = randomAsciiOfLength(5);
+        long seqNo = randomFrom(SequenceNumbersService.UNASSIGNED_SEQ_NO, randomNonNegativeLong(), (long) randomIntBetween(0, 10000));
+        long version = randomBoolean() ? randomNonNegativeLong() : randomIntBetween(0, 10000);
+        boolean found = randomBoolean();
+
+        DeleteResponse response = new DeleteResponse(shardId, type, id, seqNo, version, found);
+        response.setForcedRefresh(randomBoolean());
+        response.setShardInfo(RandomObjects.randomShardInfo(random(), randomBoolean()));
+        return response;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
index 2592423e5a8..f1cbaa9df3b 100644
--- a/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/fieldstats/FieldStatsRequestTests.java
@@ -19,10 +19,19 @@
 
 package org.elasticsearch.action.fieldstats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.fieldstats.FieldStatsTests;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
+import org.elasticsearch.test.VersionUtils;
+
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.GT;
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Comparison.GTE;
@@ -85,4 +94,33 @@ public class FieldStatsRequestTests extends ESTestCase {
         assertThat(request.getIndexConstraints()[7].getOptionalFormat(), equalTo("date_optional_time"));
     }
 
+    public void testFieldStatsBWC() throws Exception {
+        int size = randomIntBetween(5, 20);
+        Map<String, FieldStats<?> > stats = new HashMap<> ();
+        for (int i = 0; i < size; i++) {
+            stats.put(Integer.toString(i), FieldStatsTests.randomFieldStats(true));
+        }
+
+        FieldStatsShardResponse response = new FieldStatsShardResponse(new ShardId("test", "test", 0), stats);
+        for (int i = 0; i < 10; i++) {
+            Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, Version.CURRENT);
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.setVersion(version);
+            response.writeTo(output);
+            output.flush();
+            StreamInput input = output.bytes().streamInput();
+            input.setVersion(version);
+            FieldStatsShardResponse deserialized = new FieldStatsShardResponse();
+            deserialized.readFrom(input);
+            final Map<String, FieldStats<?>> expected;
+            if (version.before(Version.V_5_2_0_UNRELEASED)) {
+                expected = deserialized.filterNullMinMax();
+            } else {
+                expected = deserialized.getFieldStats();
+            }
+            assertEquals(expected.size(), deserialized.getFieldStats().size());
+            assertThat(expected, equalTo(deserialized.getFieldStats()));
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java b/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java
new file mode 100644
index 00000000000..37824ef2b03
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/get/GetResponseTests.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.get.GetField;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.index.get.GetResultTests.copyGetResult;
+import static org.elasticsearch.index.get.GetResultTests.mutateGetResult;
+import static org.elasticsearch.index.get.GetResultTests.randomGetResult;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetResponseTests extends ESTestCase {
+
+    public void testToAndFromXContent() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResponse getResponse = new GetResponse(tuple.v1());
+        GetResponse expectedGetResponse = new GetResponse(tuple.v2());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(getResponse, xContentType, humanReadable);
+        //test that we can parse what we print out
+        GetResponse parsedGetResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedGetResponse = GetResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetResponse, parsedGetResponse);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContent(parsedGetResponse, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResponse.getSourceAsString(), parsedGetResponse.getSourceAsString());
+    }
+
+    public void testToXContent() throws IOException {
+        {
+            GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 1, true, new BytesArray("{ \"field1\" : " +
+                    "\"value1\", \"field2\":\"value2\"}"), Collections.singletonMap("field1", new GetField("field1",
+                    Collections.singletonList("value1")))));
+            String output = Strings.toString(getResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"found\":true,\"_source\":{ \"field1\" " +
+                    ": \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}", output);
+        }
+        {
+            GetResponse getResponse = new GetResponse(new GetResult("index", "type", "id", 1, false, null, null));
+            String output = Strings.toString(getResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"found\":false}", output);
+        }
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(new GetResponse(randomGetResult(XContentType.JSON).v1()), GetResponseTests::copyGetResponse,
+                GetResponseTests::mutateGetResponse);
+    }
+    private static GetResponse copyGetResponse(GetResponse getResponse) {
+        return new GetResponse(copyGetResult(getResponse.getResult));
+    }
+
+    private static GetResponse mutateGetResponse(GetResponse getResponse) {
+        return new GetResponse(mutateGetResult(getResponse.getResult));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java
new file mode 100644
index 00000000000..6c79b20774c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetRequestTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.get;
+
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.test.ESTestCase;
+
+public class MultiGetRequestTests extends ESTestCase {
+
+    public void testAddWithInvalidSourceValueIsRejected() throws Exception {
+        String sourceValue = randomFrom("on", "off", "0", "1");
+        XContentParser parser = createParser(XContentFactory.jsonBuilder()
+            .startObject()
+                .startArray("docs")
+                    .startObject()
+                        .field("_source", sourceValue)
+                    .endObject()
+                .endArray()
+            .endObject()
+        );
+
+        MultiGetRequest multiGetRequest = new MultiGetRequest();
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> multiGetRequest.add
+            (randomAsciiOfLength(5), randomAsciiOfLength(3), null, FetchSourceContext.FETCH_SOURCE, null, parser, true));
+        assertEquals("Failed to parse value [" + sourceValue + "] as only [true] or [false] are allowed.", ex.getMessage());
+    }
+
+    public void testAddWithValidSourceValueIsAccepted() throws Exception {
+        XContentParser parser = createParser(XContentFactory.jsonBuilder()
+            .startObject()
+                .startArray("docs")
+                    .startObject()
+                        .field("_source", randomFrom("false", "true"))
+                    .endObject()
+                    .startObject()
+                        .field("_source", randomBoolean())
+                    .endObject()
+                .endArray()
+            .endObject()
+        );
+
+        MultiGetRequest multiGetRequest = new MultiGetRequest();
+        multiGetRequest.add(
+            randomAsciiOfLength(5), randomAsciiOfLength(3), null, FetchSourceContext.FETCH_SOURCE, null, parser, true);
+
+        assertEquals(2, multiGetRequest.getItems().size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
index 06e9d586e36..5341588f722 100644
--- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestBuilderTests.java
@@ -74,8 +74,9 @@ public class IndexRequestBuilderTests extends ESTestCase {
         ByteArrayOutputStream docOut = new ByteArrayOutputStream();
         XContentBuilder doc = XContentFactory.jsonBuilder(docOut).startObject().field("SomeKey", "SomeValue").endObject();
         doc.close();
-        indexRequestBuilder.setSource(docOut.toByteArray());
-        assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true));
+        indexRequestBuilder.setSource(docOut.toByteArray(), XContentType.JSON);
+        assertEquals(EXPECTED_SOURCE, XContentHelper.convertToJson(indexRequestBuilder.request().source(), true,
+            indexRequestBuilder.request().getContentType()));
 
         doc = XContentFactory.jsonBuilder().startObject().field("SomeKey", "SomeValue").endObject();
         doc.close();
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
index 1d1532c4919..fb1991bcd91 100644
--- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java
@@ -18,17 +18,24 @@
  */
 package org.elasticsearch.action.index;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -83,19 +90,19 @@ public class IndexRequestTests extends ESTestCase {
     public void testIndexingRejectsLongIds() {
         String id = randomAsciiOfLength(511);
         IndexRequest request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         ActionRequestValidationException validate = request.validate();
         assertNull(validate);
 
         id = randomAsciiOfLength(512);
         request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         validate = request.validate();
         assertNull(validate);
 
         id = randomAsciiOfLength(513);
         request = new IndexRequest("index", "type", id);
-        request.source("{}");
+        request.source("{}", XContentType.JSON);
         validate = request.validate();
         assertThat(validate, notNullValue());
         assertThat(validate.getMessage(),
@@ -147,7 +154,52 @@ public class IndexRequestTests extends ESTestCase {
         assertEquals("IndexResponse[index=" + shardId.getIndexName() + ",type=" + type + ",id="+ id +
                 ",version=" + version + ",result=" + (created ? "created" : "updated") +
                 ",seqNo=" + SequenceNumbersService.UNASSIGNED_SEQ_NO +
-                ",shards={\"_shards\":{\"total\":" + total + ",\"successful\":" + successful + ",\"failed\":0}}]",
+                ",shards={\"total\":" + total + ",\"successful\":" + successful + ",\"failed\":0}]",
                 indexResponse.toString());
     }
+
+    public void testIndexRequestXContentSerialization() throws IOException {
+        IndexRequest indexRequest = new IndexRequest("foo", "bar", "1");
+        indexRequest.source("{}", XContentType.JSON);
+        assertEquals(XContentType.JSON, indexRequest.getContentType());
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        indexRequest.writeTo(out);
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        IndexRequest serialized = new IndexRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getContentType());
+        assertEquals(new BytesArray("{}"), serialized.source());
+    }
+
+    public void testIndexRequestXContentSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAD////+AgQDZm9vAAAAAQNiYXIBATEAAAAAAnt9AP/////////9AAAA//////////8AAAAAAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            IndexRequest serialized = new IndexRequest();
+            serialized.readFrom(in);
+            assertEquals(XContentType.JSON, serialized.getContentType());
+            assertEquals("{}", serialized.source().utf8ToString());
+            // don't test writing to earlier versions since output differs due to no timestamp
+        }
+    }
+
+    // reindex makes use of index requests without a source so this needs to be handled
+    public void testSerializationOfEmptyRequestWorks() throws IOException {
+        IndexRequest request = new IndexRequest("index", "type");
+        assertNull(request.getContentType());
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            request.writeTo(out);
+
+            try (StreamInput in = out.bytes().streamInput()) {
+                IndexRequest serialized = new IndexRequest();
+                serialized.readFrom(in);
+                assertNull(request.getContentType());
+                assertEquals("index", request.index());
+                assertEquals("type", request.type());
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
new file mode 100644
index 00000000000..696a836a37a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/index/IndexResponseTests.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.index;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+
+public class IndexResponseTests extends ESTestCase {
+
+    public void testToXContent() {
+        {
+            IndexResponse indexResponse = new IndexResponse(new ShardId("index", "index_uuid", 0), "type", "id", 3, 5, true);
+            String output = Strings.toString(indexResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":5,\"result\":\"created\",\"_shards\":null," +
+                    "\"_seq_no\":3,\"created\":true}", output);
+        }
+        {
+            IndexResponse indexResponse = new IndexResponse(new ShardId("index", "index_uuid", 0), "type", "id", -1, 7, true);
+            indexResponse.setForcedRefresh(true);
+            indexResponse.setShardInfo(new ReplicationResponse.ShardInfo(10, 5));
+            String output = Strings.toString(indexResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":7,\"result\":\"created\"," +
+                    "\"forced_refresh\":true,\"_shards\":{\"total\":10,\"successful\":5,\"failed\":0},\"created\":true}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        // Create a random IndexResponse and converts it to XContent in bytes
+        IndexResponse indexResponse = randomIndexResponse();
+        boolean humanReadable = randomBoolean();
+        BytesReference indexResponseBytes = toXContent(indexResponse, xContentType, humanReadable);
+
+        // Parse the XContent bytes to obtain a parsed
+        IndexResponse parsedIndexResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), indexResponseBytes)) {
+            parsedIndexResponse = IndexResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        // We can't use equals() to compare the original and the parsed index response
+        // because the random index response can contain shard failures with exceptions,
+        // and those exceptions are not parsed back with the same types.
+
+        // Print the parsed object out and test that the output is the same as the original output
+        BytesReference parsedIndexResponseBytes = toXContent(parsedIndexResponse, xContentType, humanReadable);
+        try (XContentParser parser = createParser(xContentType.xContent(), parsedIndexResponseBytes)) {
+            assertIndexResponse(indexResponse, parser.map());
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public static void assertDocWriteResponse(DocWriteResponse expected, Map<String, Object> actual) {
+        assertEquals(expected.getIndex(), actual.get("_index"));
+        assertEquals(expected.getType(), actual.get("_type"));
+        assertEquals(expected.getId(), actual.get("_id"));
+        assertEquals(expected.getVersion(), ((Number) actual.get("_version")).longValue());
+        assertEquals(expected.getResult().getLowercase(), actual.get("result"));
+        if (expected.forcedRefresh()) {
+            assertTrue((Boolean) actual.get("forced_refresh"));
+        } else {
+            assertFalse(actual.containsKey("forced_refresh"));
+        }
+        if (expected.getSeqNo() >= 0) {
+            assertEquals(expected.getSeqNo(), ((Number) actual.get("_seq_no")).longValue());
+        } else {
+            assertFalse(actual.containsKey("_seq_no"));
+        }
+
+        Map<String, Object> actualShards = (Map<String, Object>) actual.get("_shards");
+        assertNotNull(actualShards);
+        assertEquals(expected.getShardInfo().getTotal(), actualShards.get("total"));
+        assertEquals(expected.getShardInfo().getSuccessful(), actualShards.get("successful"));
+        assertEquals(expected.getShardInfo().getFailed(), actualShards.get("failed"));
+
+        List<Map<String, Object>> actualFailures = (List<Map<String, Object>>) actualShards.get("failures");
+        if (CollectionUtils.isEmpty(expected.getShardInfo().getFailures())) {
+            assertNull(actualFailures);
+        } else {
+            assertEquals(expected.getShardInfo().getFailures().length, actualFailures.size());
+            for (int i = 0; i < expected.getShardInfo().getFailures().length; i++) {
+                ReplicationResponse.ShardInfo.Failure failure = expected.getShardInfo().getFailures()[i];
+                Map<String, Object> actualFailure = actualFailures.get(i);
+
+                assertEquals(failure.index(), actualFailure.get("_index"));
+                assertEquals(failure.shardId(), actualFailure.get("_shard"));
+                assertEquals(failure.nodeId(), actualFailure.get("_node"));
+                assertEquals(failure.status(), RestStatus.valueOf((String) actualFailure.get("status")));
+                assertEquals(failure.primary(), actualFailure.get("primary"));
+
+                Throwable cause = failure.getCause();
+                Map<String, Object> actualClause = (Map<String, Object>) actualFailure.get("reason");
+                assertNotNull(actualClause);
+                while (cause != null) {
+                    // The expected IndexResponse has been converted in XContent, then the resulting bytes have been
+                    // parsed to create a new parsed IndexResponse. During this process, the type of the exceptions
+                    // have been lost.
+                    assertEquals("exception", actualClause.get("type"));
+                    String expectedMessage = "Elasticsearch exception [type=" + ElasticsearchException.getExceptionName(cause)
+                            + ", reason=" + cause.getMessage() + "]";
+                    assertEquals(expectedMessage, actualClause.get("reason"));
+
+                    if (cause instanceof ElasticsearchException) {
+                        ElasticsearchException ex = (ElasticsearchException) cause;
+                        Map<String, Object> actualHeaders = (Map<String, Object>) actualClause.get("header");
+
+                        // When a IndexResponse is converted to XContent, the exception headers that start with "es."
+                        // are added to the XContent as fields with the prefix removed. Other headers are added under
+                        // a "header" root object.
+                        // In the test, the "es." prefix is lost when the XContent is generating, so when the parsed
+                        // IndexResponse is converted back to XContent all exception headers are under the "header" object.
+                        for (String name : ex.getHeaderKeys()) {
+                            assertEquals(ex.getHeader(name).get(0), actualHeaders.get(name.replaceFirst("es.", "")));
+                        }
+                    }
+                    actualClause = (Map<String, Object>) actualClause.get("caused_by");
+                    cause = cause.getCause();
+                }
+            }
+        }
+    }
+
+    private static void assertIndexResponse(IndexResponse expected, Map<String, Object> actual) {
+        assertDocWriteResponse(expected, actual);
+        if (expected.getResult() == DocWriteResponse.Result.CREATED) {
+            assertTrue((boolean) actual.get("created"));
+        } else {
+            assertFalse((boolean) actual.get("created"));
+        }
+    }
+
+    private static IndexResponse randomIndexResponse() {
+        ShardId shardId = new ShardId(randomAsciiOfLength(5), randomAsciiOfLength(5), randomIntBetween(0, 5));
+        String type = randomAsciiOfLength(5);
+        String id = randomAsciiOfLength(5);
+        long seqNo = randomFrom(SequenceNumbersService.UNASSIGNED_SEQ_NO, randomNonNegativeLong(), (long) randomIntBetween(0, 10000));
+        long version = randomBoolean() ? randomNonNegativeLong() : randomIntBetween(0, 10000);
+        boolean created = randomBoolean();
+
+        IndexResponse indexResponse = new IndexResponse(shardId, type, id, seqNo, version, created);
+        indexResponse.setForcedRefresh(randomBoolean());
+        indexResponse.setShardInfo(RandomObjects.randomShardInfo(random(), randomBoolean()));
+        return indexResponse;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/action/index/TransportIndexActionIngestTests.java b/core/src/test/java/org/elasticsearch/action/index/TransportIndexActionIngestTests.java
deleted file mode 100644
index ebc765243c4..00000000000
--- a/core/src/test/java/org/elasticsearch/action/index/TransportIndexActionIngestTests.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.index;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateApplier;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.ingest.PipelineExecutionService;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-import org.junit.Before;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.MockitoAnnotations;
-
-import java.util.Collections;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Consumer;
-
-import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-
-public class TransportIndexActionIngestTests extends ESTestCase {
-
-    /** Services needed by index action */
-    TransportService transportService;
-    ClusterService clusterService;
-    IngestService ingestService;
-
-    /** The ingest execution service we can capture calls to */
-    PipelineExecutionService executionService;
-
-    /** Arguments to callbacks we want to capture, but which require generics, so we must use @Captor */
-    @Captor
-    ArgumentCaptor<Consumer<Exception>> exceptionHandler;
-    @Captor
-    ArgumentCaptor<Consumer<Boolean>> successHandler;
-    @Captor
-    ArgumentCaptor<TransportResponseHandler<IndexResponse>> remoteResponseHandler;
-
-    /** The actual action we want to test, with real indexing mocked */
-    TestTransportIndexAction action;
-
-    /** True if the next call to the index action should act as an ingest node */
-    boolean localIngest;
-
-    /** The nodes that forwarded index requests should be cycled through. */
-    DiscoveryNodes nodes;
-    DiscoveryNode remoteNode1;
-    DiscoveryNode remoteNode2;
-
-    /** A subclass of the real index action to allow skipping real indexing, and marking when it would have happened. */
-    class TestTransportIndexAction extends TransportIndexAction {
-        boolean isExecuted = false; // set when the "real" index execution happens
-        TestTransportIndexAction() {
-            super(Settings.EMPTY, transportService, clusterService, null, ingestService, null, null, null, null,
-                new ActionFilters(Collections.emptySet()), null, null);
-        }
-        @Override
-        protected boolean shouldAutoCreate(IndexRequest reqest, ClusterState state) {
-            return false;
-        }
-        @Override
-        protected void innerExecute(Task task, final IndexRequest request, final ActionListener<IndexResponse> listener) {
-            isExecuted = true;
-        }
-    }
-
-    @Before
-    public void setupAction() {
-        // initialize captors, which must be members to use @Capture because of generics
-        MockitoAnnotations.initMocks(this);
-        // setup services that will be called by action
-        transportService = mock(TransportService.class);
-        clusterService = mock(ClusterService.class);
-        localIngest = true;
-        // setup nodes for local and remote
-        DiscoveryNode localNode = mock(DiscoveryNode.class);
-        when(localNode.isIngestNode()).thenAnswer(stub -> localIngest);
-        when(clusterService.localNode()).thenReturn(localNode);
-        remoteNode1 = mock(DiscoveryNode.class);
-        remoteNode2 = mock(DiscoveryNode.class);
-        nodes = mock(DiscoveryNodes.class);
-        ImmutableOpenMap<String, DiscoveryNode> ingestNodes = ImmutableOpenMap.<String, DiscoveryNode>builder(2)
-            .fPut("node1", remoteNode1).fPut("node2", remoteNode2).build();
-        when(nodes.getIngestNodes()).thenReturn(ingestNodes);
-        ClusterState state = mock(ClusterState.class);
-        when(state.getNodes()).thenReturn(nodes);
-        when(clusterService.state()).thenReturn(state);
-        doAnswer(invocation -> {
-            ClusterChangedEvent event = mock(ClusterChangedEvent.class);
-            when(event.state()).thenReturn(state);
-            ((ClusterStateApplier)invocation.getArguments()[0]).applyClusterState(event);
-            return null;
-        }).when(clusterService).addStateApplier(any(ClusterStateApplier.class));
-        // setup the mocked ingest service for capturing calls
-        ingestService = mock(IngestService.class);
-        executionService = mock(PipelineExecutionService.class);
-        when(ingestService.getPipelineExecutionService()).thenReturn(executionService);
-        action = new TestTransportIndexAction();
-        reset(transportService); // call on construction of action
-    }
-
-    public void testIngestSkipped() throws Exception {
-        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
-        indexRequest.source(Collections.emptyMap());
-        action.execute(null, indexRequest, ActionListener.wrap(response -> {}, exception -> {
-            throw new AssertionError(exception);
-        }));
-        assertTrue(action.isExecuted);
-        verifyZeroInteractions(ingestService);
-    }
-
-    public void testIngestLocal() throws Exception {
-        Exception exception = new Exception("fake exception");
-        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
-        indexRequest.source(Collections.emptyMap());
-        indexRequest.setPipeline("testpipeline");
-        AtomicBoolean responseCalled = new AtomicBoolean(false);
-        AtomicBoolean failureCalled = new AtomicBoolean(false);
-        action.execute(null, indexRequest, ActionListener.wrap(
-            response -> {
-                responseCalled.set(true);
-            },
-            e -> {
-                assertThat(e, sameInstance(exception));
-                failureCalled.set(true);
-            }));
-
-        // check failure works, and passes through to the listener
-        assertFalse(action.isExecuted); // haven't executed yet
-        assertFalse(responseCalled.get());
-        assertFalse(failureCalled.get());
-        verify(executionService).executeIndexRequest(same(indexRequest), exceptionHandler.capture(), successHandler.capture());
-        exceptionHandler.getValue().accept(exception);
-        assertTrue(failureCalled.get());
-
-        // now check success
-        successHandler.getValue().accept(true);
-        assertTrue(action.isExecuted);
-        assertFalse(responseCalled.get()); // listener would only be called by real index action, not our mocked one
-        verifyZeroInteractions(transportService);
-    }
-
-    public void testIngestForward() throws Exception {
-        localIngest = false;
-        IndexRequest indexRequest = new IndexRequest("index", "type", "id");
-        indexRequest.source(Collections.emptyMap());
-        indexRequest.setPipeline("testpipeline");
-        IndexResponse indexResponse = mock(IndexResponse.class);
-        AtomicBoolean responseCalled = new AtomicBoolean(false);
-        ActionListener<IndexResponse> listener = ActionListener.wrap(
-            response -> {
-                responseCalled.set(true);
-                assertSame(indexResponse, response);
-            },
-            e -> {
-                throw new AssertionError(e);
-            });
-        action.execute(null, indexRequest, listener);
-
-        // should not have executed ingest locally
-        verify(executionService, never()).executeIndexRequest(any(), any(), any());
-        // but instead should have sent to a remote node with the transport service
-        ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
-        verify(transportService).sendRequest(node.capture(), eq(IndexAction.NAME), any(), remoteResponseHandler.capture());
-        boolean usedNode1 = node.getValue() == remoteNode1; // make sure we used one of the nodes
-        if (usedNode1 == false) {
-            assertSame(remoteNode2, node.getValue());
-        }
-        assertFalse(action.isExecuted); // no local index execution
-        assertFalse(responseCalled.get()); // listener not called yet
-
-        remoteResponseHandler.getValue().handleResponse(indexResponse); // call the listener for the remote node
-        assertTrue(responseCalled.get()); // now the listener we passed should have been delegated to by the remote listener
-        assertFalse(action.isExecuted); // still no local index execution
-
-        // now make sure ingest nodes are rotated through with a subsequent request
-        reset(transportService);
-        action.execute(null, indexRequest, listener);
-        verify(transportService).sendRequest(node.capture(), eq(IndexAction.NAME), any(), remoteResponseHandler.capture());
-        if (usedNode1) {
-            assertSame(remoteNode2, node.getValue());
-        } else {
-            assertSame(remoteNode1, node.getValue());
-        }
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java
new file mode 100644
index 00000000000..01aed87947a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ingest/PutPipelineRequestTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.ingest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class PutPipelineRequestTests extends ESTestCase {
+
+    public void testSerializationWithXContent() throws IOException {
+        PutPipelineRequest request = new PutPipelineRequest("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, request.getXContentType());
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        request.writeTo(output);
+        StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes);
+
+        PutPipelineRequest serialized = new PutPipelineRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getSource().utf8ToString());
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ADwDATECe30=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PutPipelineRequest request = new PutPipelineRequest();
+            request.readFrom(in);
+            assertEquals(XContentType.JSON, request.getXContentType());
+            assertEquals("{}", request.getSource().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
index 0966010a8f7..86dc56cdd04 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestTests.java
@@ -19,19 +19,23 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class SimulatePipelineRequestTests extends ESTestCase {
 
     public void testSerialization() throws IOException {
-        SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""));
+        SimulatePipelineRequest request = new SimulatePipelineRequest(new BytesArray(""), XContentType.JSON);
         // Sometimes we set an id
         if (randomBoolean()) {
             request.setId(randomAsciiOfLengthBetween(1, 10));
@@ -51,4 +55,38 @@ public class SimulatePipelineRequestTests extends ESTestCase {
         assertThat(otherRequest.getId(), equalTo(request.getId()));
         assertThat(otherRequest.isVerbose(), equalTo(request.isVerbose()));
     }
+
+    public void testSerializationWithXContent() throws IOException {
+        SimulatePipelineRequest request =
+            new SimulatePipelineRequest(new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, request.getXContentType());
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        request.writeTo(output);
+        StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes);
+
+        SimulatePipelineRequest serialized = new SimulatePipelineRequest();
+        serialized.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getSource().utf8ToString());
+    }
+
+    public void testSerializationWithXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAAAAnt9AAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            SimulatePipelineRequest request = new SimulatePipelineRequest();
+            request.readFrom(in);
+            assertEquals(XContentType.JSON, request.getXContentType());
+            assertEquals("{}", request.getSource().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index bc72094558e..d5417526c0c 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -127,7 +127,7 @@ public class WriteableIngestDocumentTests extends ESTestCase {
         builder.startObject();
         writeableIngestDocument.toXContent(builder, EMPTY_PARAMS);
         builder.endObject();
-        Map<String, Object> toXContentMap = XContentHelper.convertToMap(builder.bytes(), false).v2();
+        Map<String, Object> toXContentMap = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
 
         Map<String, Object> toXContentDoc = (Map<String, Object>) toXContentMap.get("doc");
         Map<String, Object> toXContentSource = (Map<String, Object>) toXContentDoc.get("_source");
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
index feb33772da0..f673af9d963 100644
--- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
@@ -121,7 +121,7 @@ public class MainActionTests extends ESTestCase {
         when(clusterService.state()).thenReturn(state);
 
         TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
-                null);
+            x -> null, null);
         TransportMainAction action = new TransportMainAction(settings, mock(ThreadPool.class), transportService, mock(ActionFilters.class),
                 mock(IndexNameExpressionResolver.class), clusterService);
         AtomicReference<MainResponse> responseRef = new AtomicReference<>();
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index d5b17861e0f..d1de2f7ccf7 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -20,24 +20,23 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
 
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
@@ -140,11 +139,6 @@ public class MultiSearchRequestTests extends ESTestCase {
                     new MultiSearchResponse.Item(null, new IllegalStateException("baaaaaazzzz"))
         });
 
-        XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
-        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
-
         assertEquals("{\"responses\":["
                         + "{"
                         + "\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"}],"
@@ -155,7 +149,7 @@ public class MultiSearchRequestTests extends ESTestCase {
                         + "\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"},\"status\":500"
                         + "}"
                         + "]}",
-                builder.string());
+                Strings.toString(response));
     }
 
     public void testMaxConcurrentSearchRequests() {
@@ -167,14 +161,14 @@ public class MultiSearchRequestTests extends ESTestCase {
 
     private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException {
         byte[] data = StreamsUtils.copyToBytesFromClasspath(sample);
-        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
-        return RestMultiSearchAction.parseRequest(restRequest, true, parsers(), ParseFieldMatcher.EMPTY);
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(data), XContentType.JSON).build();
+        return RestMultiSearchAction.parseRequest(restRequest, true);
     }
 
-    private SearchRequestParsers parsers() {
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        registry.register(parser, MatchAllQueryBuilder.NAME);
-        return new SearchRequestParsers(registry, null, null, null);
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(singletonList(new NamedXContentRegistry.Entry(QueryBuilder.class,
+                new ParseField(MatchAllQueryBuilder.NAME), (p, c) -> MatchAllQueryBuilder.fromXContent((QueryParseContext) c))));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/RemoteClusterConnectionTests.java b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterConnectionTests.java
new file mode 100644
index 00000000000..f08f810f33f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterConnectionTests.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsAction;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.CancellableThreads;
+import org.elasticsearch.mocksocket.MockServerSocket;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportConnectionListener;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.nio.channels.AlreadyConnectedException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+public class RemoteClusterConnectionTests extends ESTestCase {
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    private MockTransportService startTransport(String id, List<DiscoveryNode> knownNodes, Version version) {
+        return startTransport(id, knownNodes, version, threadPool);
+    }
+
+    public static MockTransportService startTransport(String id, List<DiscoveryNode> knownNodes, Version version, ThreadPool threadPool) {
+        boolean success = false;
+        MockTransportService newService = MockTransportService.createNewService(Settings.EMPTY, version, threadPool, null);
+        try {
+            newService.registerRequestHandler(ClusterSearchShardsAction.NAME, ClusterSearchShardsRequest::new, ThreadPool.Names.SAME,
+                (request, channel) -> {
+                    channel.sendResponse(new ClusterSearchShardsResponse(new ClusterSearchShardsGroup[0],
+                        knownNodes.toArray(new DiscoveryNode[0]), Collections.emptyMap()));
+                });
+            newService.registerRequestHandler(ClusterStateAction.NAME, ClusterStateRequest::new, ThreadPool.Names.SAME,
+                (request, channel) -> {
+                    DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+                    for (DiscoveryNode node : knownNodes) {
+                        builder.add(node);
+                    }
+                    ClusterState build = ClusterState.builder(ClusterName.DEFAULT).nodes(builder.build()).build();
+                    channel.sendResponse(new ClusterStateResponse(ClusterName.DEFAULT, build));
+                });
+            newService.start();
+            newService.acceptIncomingRequests();
+            success = true;
+            return newService;
+        } finally {
+            if (success == false) {
+                newService.close();
+            }
+        }
+    }
+
+    public void testDiscoverSingleNode() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testDiscoverSingleNodeWithIncompatibleSeed() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService incomaptibleTransport = startTransport("incompat_seed_node", knownNodes, Version.fromString("2.0.0"));
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode incompatibleSeedNode = incomaptibleTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(incomaptibleTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(incompatibleSeedNode, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, seedNodes);
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertFalse(service.nodeConnected(incompatibleSeedNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testNodeDisconnected() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT);
+             MockTransportService spareTransport = startTransport("spare_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode spareNode = spareTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertFalse(service.nodeConnected(spareNode));
+                    knownNodes.add(spareNode);
+                    CountDownLatch latchDisconnect = new CountDownLatch(1);
+                    CountDownLatch latchConnected = new CountDownLatch(1);
+                    service.addConnectionListener(new TransportConnectionListener() {
+                        @Override
+                        public void onNodeDisconnected(DiscoveryNode node) {
+                            if (node.equals(discoverableNode)) {
+                                latchDisconnect.countDown();
+                            }
+                        }
+
+                        @Override
+                        public void onNodeConnected(DiscoveryNode node) {
+                            if (node.equals(spareNode)) {
+                                latchConnected.countDown();
+                            }
+                        }
+                    });
+
+                    discoverableTransport.close();
+                    // now make sure we try to connect again to other nodes once we got disconnected
+                    assertTrue(latchDisconnect.await(10, TimeUnit.SECONDS));
+                    assertTrue(latchConnected.await(10, TimeUnit.SECONDS));
+                    assertTrue(service.nodeConnected(spareNode));
+                }
+            }
+        }
+    }
+
+    public void testFilterDiscoveredNodes() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode;
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false)) {
+                    updateSeedNodes(connection, Arrays.asList(seedNode));
+                    if (rejectedNode.equals(seedNode)) {
+                        assertFalse(service.nodeConnected(seedNode));
+                        assertTrue(service.nodeConnected(discoverableNode));
+                    } else {
+                        assertTrue(service.nodeConnected(seedNode));
+                        assertFalse(service.nodeConnected(discoverableNode));
+                    }
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    private void updateSeedNodes(RemoteClusterConnection connection, List<DiscoveryNode> seedNodes) throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<Exception> exceptionAtomicReference = new AtomicReference<>();
+        ActionListener<Void> listener = ActionListener.wrap(x -> latch.countDown(), x -> {
+            exceptionAtomicReference.set(x);
+            latch.countDown();
+        });
+        connection.updateSeedNodes(seedNodes, listener);
+        latch.await();
+        if (exceptionAtomicReference.get() != null) {
+            throw exceptionAtomicReference.get();
+        }
+    }
+
+    public void testConnectWithIncompatibleTransports() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    expectThrows(Exception.class, () -> updateSeedNodes(connection, Arrays.asList(seedNode)));
+                    assertFalse(service.nodeConnected(seedNode));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    @SuppressForbidden(reason = "calls getLocalHost here but it's fine in this case")
+    public void testSlowNodeCanBeCanceled() throws IOException, InterruptedException {
+        try (ServerSocket socket = new MockServerSocket()) {
+            socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
+            socket.setReuseAddress(true);
+            DiscoveryNode seedNode = new DiscoveryNode("TEST", new TransportAddress(socket.getInetAddress(),
+                socket.getLocalPort()), emptyMap(),
+                emptySet(), Version.CURRENT);
+            CountDownLatch acceptedLatch = new CountDownLatch(1);
+            CountDownLatch closeRemote = new CountDownLatch(1);
+            Thread t = new Thread() {
+                @Override
+                public void run() {
+                    try (Socket accept = socket.accept()) {
+                        acceptedLatch.countDown();
+                        closeRemote.await();
+                    } catch (IOException e) {
+                        // that's fine we might close
+                    } catch (InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                }
+            };
+            t.start();
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                CountDownLatch listenerCalled = new CountDownLatch(1);
+                AtomicReference<Exception> exceptionReference = new AtomicReference<>();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    ActionListener<Void> listener = ActionListener.wrap(x -> {
+                        listenerCalled.countDown();
+                        fail("expected exception");
+                    }, x -> {
+                        exceptionReference.set(x);
+                        listenerCalled.countDown();
+                    });
+                    connection.updateSeedNodes(Arrays.asList(seedNode), listener);
+                    acceptedLatch.await();
+                    connection.close(); // now close it, this should trigger an interrupt on the socket and we can move on
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+                closeRemote.countDown();
+                listenerCalled.await();
+                assertNotNull(exceptionReference.get());
+                expectThrows(CancellableThreads.ExecutionCancelledException.class, () -> {throw exceptionReference.get();});
+
+            }
+        }
+    }
+
+    public void testFetchShards() throws Exception {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
+                    if (randomBoolean()) {
+                        updateSeedNodes(connection, Arrays.asList(seedNode));
+                    }
+
+                    SearchRequest request = new SearchRequest("test-index");
+                    CountDownLatch responseLatch = new CountDownLatch(1);
+                    AtomicReference<ClusterSearchShardsResponse> reference = new AtomicReference<>();
+                    AtomicReference<Exception> failReference = new AtomicReference<>();
+                    ActionListener<ClusterSearchShardsResponse> shardsListener = ActionListener.wrap(
+                        x -> {
+                            reference.set(x);
+                            responseLatch.countDown();
+                        },
+                        x -> {
+                            failReference.set(x);
+                            responseLatch.countDown();
+                        });
+                    connection.fetchSearchShards(request, Arrays.asList("test-index"), shardsListener);
+                    responseLatch.await();
+                    assertNull(failReference.get());
+                    assertNotNull(reference.get());
+                    ClusterSearchShardsResponse clusterSearchShardsResponse = reference.get();
+                    assertEquals(knownNodes, Arrays.asList(clusterSearchShardsResponse.getNodes()));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testTriggerUpdatesConcurrently() throws IOException, InterruptedException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport1.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(seedNode1, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    int numThreads = randomIntBetween(4, 10);
+                    Thread[] threads = new Thread[numThreads];
+                    CyclicBarrier barrier = new CyclicBarrier(numThreads);
+                    for (int i = 0; i < threads.length; i++) {
+                        final int numConnectionAttempts = randomIntBetween(10, 200);
+                        threads[i] = new Thread() {
+                            @Override
+                            public void run() {
+                                try {
+                                    barrier.await();
+                                    CountDownLatch latch = new CountDownLatch(numConnectionAttempts);
+                                    for (int i = 0; i < numConnectionAttempts; i++) {
+                                        AtomicBoolean executed = new AtomicBoolean(false);
+                                        ActionListener<Void> listener = ActionListener.wrap(x -> {
+                                            assertTrue(executed.compareAndSet(false, true));
+                                            latch.countDown();}, x -> {
+                                            assertTrue(executed.compareAndSet(false, true));
+                                            latch.countDown();
+                                            if (x instanceof RejectedExecutionException) {
+                                                // that's fine
+                                            } else {
+                                                throw new AssertionError(x);
+                                            }
+                                        });
+                                        connection.updateSeedNodes(seedNodes, listener);
+                                    }
+                                    latch.await();
+                                } catch (Exception ex) {
+                                    throw new AssertionError(ex);
+                                }
+                            }
+                        };
+                        threads[i].start();
+                    }
+
+                    for (int i = 0; i < threads.length; i++) {
+                        threads[i].join();
+                    }
+                    assertTrue(service.nodeConnected(seedNode));
+                    assertTrue(service.nodeConnected(discoverableNode));
+                    assertTrue(service.nodeConnected(seedNode1));
+                    assertTrue(connection.assertNoRunningConnections());
+                }
+            }
+        }
+    }
+
+    public void testCloseWhileConcurrentlyConnecting() throws IOException, InterruptedException, BrokenBarrierException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
+             MockTransportService seedTransport1 = startTransport("seed_node_1", knownNodes, Version.CURRENT);
+             MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode seedNode1 = seedTransport1.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(discoverableTransport.getLocalDiscoNode());
+            knownNodes.add(seedTransport1.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+            List<DiscoveryNode> seedNodes = Arrays.asList(seedNode1, seedNode);
+            Collections.shuffle(seedNodes, random());
+
+            try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
+                service.start();
+                service.acceptIncomingRequests();
+                try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster",
+                    seedNodes, service, Integer.MAX_VALUE, n -> true)) {
+                    int numThreads = randomIntBetween(4, 10);
+                    Thread[] threads = new Thread[numThreads];
+                    CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
+                    for (int i = 0; i < threads.length; i++) {
+                        final int numConnectionAttempts = randomIntBetween(10, 100);
+                        threads[i] = new Thread() {
+                            @Override
+                            public void run() {
+                                try {
+                                    barrier.await();
+                                    CountDownLatch latch = new CountDownLatch(numConnectionAttempts);
+                                    for (int i = 0; i < numConnectionAttempts; i++) {
+                                        AtomicReference<RuntimeException> executed = new AtomicReference<>();
+                                        ActionListener<Void> listener = ActionListener.wrap(
+                                            x -> {
+                                                if (executed.compareAndSet(null, new RuntimeException())) {
+                                                    latch.countDown();
+                                                } else {
+                                                    throw new AssertionError("shit's been called twice", executed.get());
+                                                }
+                                            },
+                                            x -> {
+                                                if (executed.compareAndSet(null, new RuntimeException())) {
+                                                    latch.countDown();
+                                                } else {
+                                                    throw new AssertionError("shit's been called twice", executed.get());
+                                                }
+                                                if (x instanceof RejectedExecutionException || x instanceof AlreadyClosedException
+                                                    || x instanceof CancellableThreads.ExecutionCancelledException) {
+                                                    // that's fine
+                                                } else {
+                                                    throw new AssertionError(x);
+                                                }
+                                        });
+                                        connection.updateSeedNodes(seedNodes, listener);
+                                    }
+                                    latch.await();
+                                } catch (Exception ex) {
+                                    throw new AssertionError(ex);
+                                }
+                            }
+                        };
+                        threads[i].start();
+                    }
+                    barrier.await();
+                    connection.close();
+                }
+            }
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java
new file mode 100644
index 00000000000..1531d66e5da
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/RemoteClusterServiceTests.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
+import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardIterator;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.AliasFilter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+public class RemoteClusterServiceTests extends ESTestCase {
+
+    private final ThreadPool threadPool = new TestThreadPool(getClass().getName());
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
+
+    private MockTransportService startTransport(String id, List<DiscoveryNode> knownNodes, Version version) {
+        return RemoteClusterConnectionTests.startTransport(id, knownNodes, version, threadPool);
+    }
+
+    public void testSettingsAreRegistered() {
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_CLUSTERS_SEEDS));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_CONNECTIONS_PER_CLUSTER));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_INITIAL_CONNECTION_TIMEOUT_SETTING));
+        assertTrue(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.contains(RemoteClusterService.REMOTE_NODE_ATTRIBUTE));
+    }
+
+    public void testRemoteClusterSeedSetting() {
+        // simple validation
+        Settings settings = Settings.builder()
+            .put("search.remote.foo.seeds", "192.168.0.1:8080")
+            .put("search.remote.bar.seed", "[::1]:9090").build();
+        RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(settings).forEach(setting -> setting.get(settings));
+
+        Settings brokenSettings = Settings.builder()
+            .put("search.remote.foo.seeds", "192.168.0.1").build();
+        expectThrows(IllegalArgumentException.class, () ->
+        RemoteClusterService.REMOTE_CLUSTERS_SEEDS.getAllConcreteSettings(brokenSettings).forEach(setting -> setting.get(brokenSettings)));
+    }
+
+    public void testBuiltRemoteClustersSeeds() throws Exception {
+        Map<String, List<DiscoveryNode>> map = RemoteClusterService.buildRemoteClustersSeeds(
+            Settings.builder().put("search.remote.foo.seeds", "192.168.0.1:8080").put("search.remote.bar.seeds", "[::1]:9090").build());
+        assertEquals(2, map.size());
+        assertTrue(map.containsKey("foo"));
+        assertTrue(map.containsKey("bar"));
+        assertEquals(1, map.get("foo").size());
+        assertEquals(1, map.get("bar").size());
+
+        DiscoveryNode foo = map.get("foo").get(0);
+
+        assertEquals(foo.getAddress(), new TransportAddress(new InetSocketAddress(InetAddress.getByName("192.168.0.1"), 8080)));
+        assertEquals(foo.getId(), "foo#192.168.0.1:8080");
+        assertEquals(foo.getVersion(), Version.CURRENT.minimumCompatibilityVersion());
+
+        DiscoveryNode bar = map.get("bar").get(0);
+        assertEquals(bar.getAddress(), new TransportAddress(new InetSocketAddress(InetAddress.getByName("[::1]"), 9090)));
+        assertEquals(bar.getId(), "bar#[::1]:9090");
+        assertEquals(bar.getVersion(), Version.CURRENT.minimumCompatibilityVersion());
+    }
+
+
+    public void testGroupClusterIndices() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putArray("search.remote.cluster_1.seeds", seedNode.getAddress().toString());
+                builder.putArray("search.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                try (RemoteClusterService service = new RemoteClusterService(builder.build(), transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    assertTrue(service.isRemoteClusterRegistered("cluster_2"));
+                    assertFalse(service.isRemoteClusterRegistered("foo"));
+                    Map<String, List<String>> perClusterIndices = service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",
+                        "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo"}, i -> false);
+                    String[] localIndices = perClusterIndices.computeIfAbsent(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY,
+                        k -> Collections.emptyList()).toArray(new String[0]);
+                    assertNotNull(perClusterIndices.remove(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY));
+                    assertArrayEquals(new String[]{"foo:bar", "foo"}, localIndices);
+                    assertEquals(2, perClusterIndices.size());
+                    assertEquals(Arrays.asList("bar", "test"), perClusterIndices.get("cluster_1"));
+                    assertEquals(Arrays.asList("foo:bar", "foo*"), perClusterIndices.get("cluster_2"));
+
+                    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                        service.groupClusterIndices(new String[]{"foo:bar", "cluster_1:bar",
+                            "cluster_2:foo:bar", "cluster_1:test", "cluster_2:foo*", "foo"}, i -> "cluster_1:bar".equals(i)));
+
+                    assertEquals("Can not filter indices; index cluster_1:bar exists but there is also a remote cluster named:" +
+                            " cluster_1", iae.getMessage());
+                }
+            }
+        }
+    }
+
+    public void testIncrementallyAddClusters() throws IOException {
+        List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
+        try (MockTransportService seedTransport = startTransport("cluster_1_node", knownNodes, Version.CURRENT);
+             MockTransportService otherSeedTransport = startTransport("cluster_2_node", knownNodes, Version.CURRENT)) {
+            DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
+            DiscoveryNode otherSeedNode = otherSeedTransport.getLocalDiscoNode();
+            knownNodes.add(seedTransport.getLocalDiscoNode());
+            knownNodes.add(otherSeedTransport.getLocalDiscoNode());
+            Collections.shuffle(knownNodes, random());
+
+            try (MockTransportService transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool,
+                null)) {
+                transportService.start();
+                transportService.acceptIncomingRequests();
+                Settings.Builder builder = Settings.builder();
+                builder.putArray("search.remote.cluster_1.seeds", seedNode.getAddress().toString());
+                builder.putArray("search.remote.cluster_2.seeds", otherSeedNode.getAddress().toString());
+                try (RemoteClusterService service = new RemoteClusterService(Settings.EMPTY, transportService)) {
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.initializeRemoteClusters();
+                    assertFalse(service.isCrossClusterSearchEnabled());
+                    service.updateRemoteCluster("cluster_1", Collections.singletonList(seedNode.getAddress().address()));
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    service.updateRemoteCluster("cluster_2", Collections.singletonList(otherSeedNode.getAddress().address()));
+                    assertTrue(service.isCrossClusterSearchEnabled());
+                    assertTrue(service.isRemoteClusterRegistered("cluster_1"));
+                    assertTrue(service.isRemoteClusterRegistered("cluster_2"));
+                    service.updateRemoteCluster("cluster_2", Collections.emptyList());
+                    assertFalse(service.isRemoteClusterRegistered("cluster_2"));
+                    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+                        () -> service.updateRemoteCluster(RemoteClusterService.LOCAL_CLUSTER_GROUP_KEY, Collections.emptyList()));
+                    assertEquals("remote clusters must not have the empty string as its key", iae.getMessage());
+                }
+            }
+        }
+    }
+
+    public void testProcessRemoteShards() throws IOException {
+        try (RemoteClusterService service = new RemoteClusterService(Settings.EMPTY, null)) {
+            assertFalse(service.isCrossClusterSearchEnabled());
+            List<ShardIterator> iteratorList = new ArrayList<>();
+            Map<String, ClusterSearchShardsResponse> searchShardsResponseMap = new HashMap<>();
+            DiscoveryNode[] nodes = new DiscoveryNode[] {
+                new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT),
+                new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT)
+            };
+            Map<String, AliasFilter> indicesAndAliases = new HashMap<>();
+            indicesAndAliases.put("foo", new AliasFilter(new TermsQueryBuilder("foo", "bar"), Strings.EMPTY_ARRAY));
+            indicesAndAliases.put("bar", new AliasFilter(new MatchAllQueryBuilder(), Strings.EMPTY_ARRAY));
+            ClusterSearchShardsGroup[] groups = new ClusterSearchShardsGroup[] {
+                new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 0),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("foo", 0, "node2", false, ShardRoutingState.STARTED)}),
+                new ClusterSearchShardsGroup(new ShardId("foo", "foo_id", 1),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("foo", 0, "node1", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("foo", 1, "node2", false, ShardRoutingState.STARTED)}),
+                new ClusterSearchShardsGroup(new ShardId("bar", "bar_id", 0),
+                    new ShardRouting[] {TestShardRouting.newShardRouting("bar", 0, "node2", true, ShardRoutingState.STARTED),
+                        TestShardRouting.newShardRouting("bar", 0, "node1", false, ShardRoutingState.STARTED)})
+            };
+            searchShardsResponseMap.put("test_cluster_1", new ClusterSearchShardsResponse(groups, nodes, indicesAndAliases));
+            Map<String, AliasFilter> remoteAliases = new HashMap<>();
+            service.processRemoteShards(searchShardsResponseMap, iteratorList, remoteAliases);
+            assertEquals(3, iteratorList.size());
+            for (ShardIterator iterator : iteratorList) {
+                if (iterator.shardId().getIndexName().endsWith("foo")) {
+                    assertTrue(iterator.shardId().getId() == 0 || iterator.shardId().getId() == 1);
+                    assertEquals("test_cluster_1:foo", iterator.shardId().getIndexName());
+                    ShardRouting shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "foo");
+                    shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "foo");
+                    assertNull(iterator.nextOrNull());
+                } else {
+                    assertEquals(0, iterator.shardId().getId());
+                    assertEquals("test_cluster_1:bar", iterator.shardId().getIndexName());
+                    ShardRouting shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "bar");
+                    shardRouting = iterator.nextOrNull();
+                    assertNotNull(shardRouting);
+                    assertEquals(shardRouting.getIndexName(), "bar");
+                    assertNull(iterator.nextOrNull());
+                }
+            }
+            assertEquals(2, remoteAliases.size());
+            assertTrue(remoteAliases.toString(), remoteAliases.containsKey("foo_id"));
+            assertTrue(remoteAliases.toString(), remoteAliases.containsKey("bar_id"));
+            assertEquals(new TermsQueryBuilder("foo", "bar"), remoteAliases.get("foo_id").getQueryBuilder());
+            assertEquals(new MatchAllQueryBuilder(), remoteAliases.get("bar_id").getQueryBuilder());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
index 86a8a8fa7c3..2a3a5f7f934 100644
--- a/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java
@@ -27,10 +27,13 @@ import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.CheckedRunnable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.search.SearchPhaseResult;
@@ -38,6 +41,10 @@ import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.AliasFilter;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestOptions;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -76,26 +83,30 @@ public class SearchAsyncActionTests extends ESTestCase {
         AtomicInteger contextIdGenerator = new AtomicInteger(0);
         GroupShardsIterator shardsIter = getShardsIter("idx", randomIntBetween(1, 10), randomBoolean(), primaryNode, replicaNode);
         AtomicInteger numFreedContext = new AtomicInteger();
-        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, null) {
+        SearchTransportService transportService = new SearchTransportService(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+                Collections.singleton(RemoteClusterService.REMOTE_CLUSTERS_SEEDS)), null) {
             @Override
-            public void sendFreeContext(DiscoveryNode node, long contextId, SearchRequest request) {
+            public void sendFreeContext(Transport.Connection connection, long contextId, SearchRequest request) {
                 numFreedContext.incrementAndGet();
-                assertTrue(nodeToContextMap.containsKey(node));
-                assertTrue(nodeToContextMap.get(node).remove(contextId));
+                assertTrue(nodeToContextMap.containsKey(connection.getNode()));
+                assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId));
             }
         };
-        Map<String, DiscoveryNode> lookup = new HashMap<>();
-        lookup.put(primaryNode.getId(), primaryNode);
+        Map<String, Transport.Connection> lookup = new HashMap<>();
+        lookup.put(primaryNode.getId(), new MockConnection(primaryNode));
+        lookup.put(replicaNode.getId(), new MockConnection(replicaNode));
         Map<String, AliasFilter> aliasFilters = Collections.singletonMap("_na_", new AliasFilter(null, Strings.EMPTY_ARRAY));
         AbstractSearchAsyncAction asyncAction = new AbstractSearchAsyncAction<TestSearchPhaseResult>(logger, transportService, lookup::get,
-            aliasFilters, Collections.emptyMap(), null, request, responseListener, shardsIter, 0, 0, null) {
+            aliasFilters, Collections.emptyMap(), null, null, request, responseListener, shardsIter, 0, 0, null) {
             TestSearchResponse response = new TestSearchResponse();
 
             @Override
-            protected void sendExecuteFirstPhase(DiscoveryNode node, ShardSearchTransportRequest request, ActionListener listener) {
+            protected void sendExecuteFirstPhase(Transport.Connection connection, ShardSearchTransportRequest request,
+                                                 ActionListener listener) {
                 assertTrue("shard: " + request.shardId() + " has been queried twice", response.queried.add(request.shardId()));
-                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(), node);
-                Set<Long> ids = nodeToContextMap.computeIfAbsent(node, (n) -> new HashSet<>());
+                TestSearchPhaseResult testSearchPhaseResult = new TestSearchPhaseResult(contextIdGenerator.incrementAndGet(),
+                    connection.getNode());
+                Set<Long> ids = nodeToContextMap.computeIfAbsent(connection.getNode(), (n) -> new HashSet<>());
                 ids.add(testSearchPhaseResult.id);
                 if (randomBoolean()) {
                     listener.onResponse(testSearchPhaseResult);
@@ -105,18 +116,20 @@ public class SearchAsyncActionTests extends ESTestCase {
             }
 
             @Override
-            protected void moveToSecondPhase() throws Exception {
-                for (int i = 0; i < firstResults.length(); i++) {
-                    TestSearchPhaseResult result = firstResults.get(i);
-                    assertEquals(result.node.getId(), result.shardTarget().getNodeId());
-                    sendReleaseSearchContext(result.id(), result.node);
-                }
-                responseListener.onResponse(response);
-                latch.countDown();
+            protected CheckedRunnable<Exception> getNextPhase(AtomicArray<TestSearchPhaseResult> initialResults) {
+                return () -> {
+                    for (int i = 0; i < initialResults.length(); i++) {
+                        TestSearchPhaseResult result = initialResults.get(i);
+                        assertEquals(result.node.getId(), result.shardTarget().getNodeId());
+                        sendReleaseSearchContext(result.id(), new MockConnection(result.node));
+                    }
+                    responseListener.onResponse(response);
+                    latch.countDown();
+                };
             }
 
             @Override
-            protected String firstPhaseName() {
+            protected String initialPhaseName() {
                 return "test";
             }
 
@@ -130,10 +143,13 @@ public class SearchAsyncActionTests extends ESTestCase {
         latch.await();
         assertNotNull(response.get());
         assertFalse(nodeToContextMap.isEmpty());
-        assertTrue(nodeToContextMap.containsKey(primaryNode));
+        assertTrue(nodeToContextMap.toString(), nodeToContextMap.containsKey(primaryNode) || nodeToContextMap.containsKey(replicaNode));
         assertEquals(shardsIter.size(), numFreedContext.get());
-        assertTrue(nodeToContextMap.get(primaryNode).toString(), nodeToContextMap.get(primaryNode).isEmpty());
-
+        if (nodeToContextMap.containsKey(primaryNode)) {
+            assertTrue(nodeToContextMap.get(primaryNode).toString(), nodeToContextMap.get(primaryNode).isEmpty());
+        } else {
+            assertTrue(nodeToContextMap.get(replicaNode).toString(), nodeToContextMap.get(replicaNode).isEmpty());
+        }
     }
 
     private GroupShardsIterator getShardsIter(String index, int numShards, boolean doReplicas, DiscoveryNode primaryNode,
@@ -211,4 +227,29 @@ public class SearchAsyncActionTests extends ESTestCase {
 
         }
     }
+
+    public final class MockConnection implements Transport.Connection {
+
+        private final DiscoveryNode node;
+
+        public MockConnection(DiscoveryNode node) {
+            this.node = node;
+        }
+
+        @Override
+        public DiscoveryNode getNode() {
+            return node;
+        }
+
+        @Override
+        public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
+            throws IOException, TransportException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void close() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
new file mode 100644
index 00000000000..1c523d6503e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/SearchPhaseExecutionExceptionTests.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.TimestampParsingException;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.IndexShardClosedException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.InvalidIndexTemplateException;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+public class SearchPhaseExecutionExceptionTests extends ESTestCase {
+
+    public void testToXContent() throws IOException {
+        SearchPhaseExecutionException exception = new SearchPhaseExecutionException("test", "all shards failed",
+                new ShardSearchFailure[]{
+                        new ShardSearchFailure(new ParsingException(1, 2, "foobar", null),
+                                new SearchShardTarget("node_1", new Index("foo", "_na_"), 0)),
+                        new ShardSearchFailure(new IndexShardClosedException(new ShardId(new Index("foo", "_na_"), 1)),
+                                new SearchShardTarget("node_2", new Index("foo", "_na_"), 1)),
+                        new ShardSearchFailure(new ParsingException(5, 7, "foobar", null),
+                                new SearchShardTarget("node_3", new Index("foo", "_na_"), 2)),
+                });
+
+        // Failures are grouped (by default)
+        assertEquals("{" +
+                "\"type\":\"search_phase_execution_exception\"," +
+                "\"reason\":\"all shards failed\"," +
+                "\"phase\":\"test\"," +
+                "\"grouped\":true," +
+                "\"failed_shards\":[" +
+                        "{" +
+                            "\"shard\":0," +
+                            "\"index\":\"foo\"," +
+                            "\"node\":\"node_1\"," +
+                            "\"reason\":{" +
+                                        "\"type\":\"parsing_exception\"," +
+                                        "\"reason\":\"foobar\"," +
+                                        "\"line\":1," +
+                                        "\"col\":2" +
+                            "}" +
+                        "}," +
+                        "{" +
+                            "\"shard\":1," +
+                            "\"index\":\"foo\"," +
+                            "\"node\":\"node_2\"," +
+                            "\"reason\":{" +
+                                        "\"type\":\"index_shard_closed_exception\"," +
+                                        "\"reason\":\"CurrentState[CLOSED] Closed\"," +
+                                        "\"index_uuid\":\"_na_\"," +
+                                        "\"shard\":\"1\"," +
+                                        "\"index\":\"foo\"" +
+                            "}" +
+                        "}" +
+                "]}", Strings.toString(exception));
+
+        // Failures are NOT grouped
+        ToXContent.MapParams params = new ToXContent.MapParams(singletonMap("group_shard_failures", "false"));
+        try (XContentBuilder builder = jsonBuilder()) {
+            builder.startObject();
+            exception.toXContent(builder, params);
+            builder.endObject();
+
+            assertEquals("{" +
+                    "\"type\":\"search_phase_execution_exception\"," +
+                    "\"reason\":\"all shards failed\"," +
+                    "\"phase\":\"test\"," +
+                    "\"grouped\":false," +
+                    "\"failed_shards\":[" +
+                            "{" +
+                                "\"shard\":0," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_1\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"parsing_exception\"," +
+                                            "\"reason\":\"foobar\"," +
+                                            "\"line\":1," +
+                                            "\"col\":2" +
+                                "}" +
+                            "}," +
+                            "{" +
+                                "\"shard\":1," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_2\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"index_shard_closed_exception\"," +
+                                            "\"reason\":\"CurrentState[CLOSED] Closed\"," +
+                                            "\"index_uuid\":\"_na_\"," +
+                                            "\"shard\":\"1\"," +
+                                            "\"index\":\"foo\"" +
+                                "}" +
+                            "}," +
+                            "{" +
+                                "\"shard\":2," +
+                                "\"index\":\"foo\"," +
+                                "\"node\":\"node_3\"," +
+                                "\"reason\":{" +
+                                            "\"type\":\"parsing_exception\"," +
+                                            "\"reason\":\"foobar\"," +
+                                            "\"line\":5," +
+                                            "\"col\":7" +
+                                "}" +
+                            "}" +
+                    "]}", builder.string());
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContent xContent = randomFrom(XContentType.values()).xContent();
+
+        ShardSearchFailure[] shardSearchFailures = new ShardSearchFailure[randomIntBetween(1, 5)];
+        for (int i = 0; i < shardSearchFailures.length; i++) {
+            Exception cause = randomFrom(
+                    new ParsingException(1, 2, "foobar", null),
+                    new InvalidIndexTemplateException("foo", "bar"),
+                    new TimestampParsingException("foo", null),
+                    new NullPointerException()
+            );
+            shardSearchFailures[i] = new  ShardSearchFailure(cause, new SearchShardTarget("node_" + i, new Index("test", "_na_"), i));
+        }
+
+        final String phase = randomFrom("query", "search", "other");
+        SearchPhaseExecutionException actual = new SearchPhaseExecutionException(phase, "unexpected failures", shardSearchFailures);
+
+        BytesReference exceptionBytes = XContentHelper.toXContent(actual, xContent.type(), randomBoolean());
+
+        ElasticsearchException parsedException;
+        try (XContentParser parser = createParser(xContent, exceptionBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedException = ElasticsearchException.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+
+        assertNotNull(parsedException);
+        assertThat(parsedException.getHeaderKeys(), hasSize(0));
+        assertThat(parsedException.getMetadataKeys(), hasSize(1));
+        assertThat(parsedException.getMetadata("es.phase"), hasItem(phase));
+        // SearchPhaseExecutionException has no cause field
+        assertNull(parsedException.getCause());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
new file mode 100644
index 00000000000..94a7e1d44bb
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+
+public class ShardSearchFailureTests extends ESTestCase {
+
+    public static ShardSearchFailure createTestItem() {
+        String randomMessage = randomAsciiOfLengthBetween(3, 20);
+        Exception ex = new ParsingException(0, 0, randomMessage , new IllegalArgumentException("some bad argument"));
+        String nodeId = randomAsciiOfLengthBetween(5, 10);
+        String indexName = randomAsciiOfLengthBetween(5, 10);
+        String indexUuid = randomAsciiOfLengthBetween(5, 10);
+        int shardId = randomInt();
+        return new ShardSearchFailure(ex,
+                new SearchShardTarget(nodeId, new ShardId(new Index(indexName, indexUuid), shardId)));
+    }
+
+    public void testFromXContent() throws IOException {
+        ShardSearchFailure response = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(response, xContentType, humanReadable);
+
+        ShardSearchFailure parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsed = ShardSearchFailure.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(response.index(), parsed.index());
+        assertEquals(response.shard().getNodeId(), parsed.shard().getNodeId());
+        assertEquals(response.shardId(), parsed.shardId());
+
+        // we cannot compare the cause, because it will be wrapped in an outer ElasticSearchException
+        // best effort: try to check that the original message appears somewhere in the rendered xContent
+        String originalMsg = response.getCause().getMessage();
+        assertEquals(parsed.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" + originalMsg + "]");
+        String nestedMsg = response.getCause().getCause().getMessage();
+        assertEquals(parsed.getCause().getCause().getMessage(),
+                "Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]");
+    }
+
+    public void testToXContent() throws IOException {
+        ShardSearchFailure failure = new ShardSearchFailure(new ParsingException(0, 0, "some message", null),
+                new SearchShardTarget("nodeId", new ShardId(new Index("indexName", "indexUuid"), 123)));
+        BytesReference xContent = toXContent(failure, XContentType.JSON, randomBoolean());
+        assertEquals(
+                "{\"shard\":123,"
+                        + "\"index\":\"indexName\","
+                        + "\"node\":\"nodeId\","
+                        + "\"reason\":{"
+                            + "\"type\":\"parsing_exception\","
+                            + "\"reason\":\"some message\","
+                            + "\"line\":0,"
+                            + "\"col\":0"
+                        + "}"
+                + "}",
+                xContent.utf8ToString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
index 8c0e0025749..7773264f77f 100644
--- a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
@@ -48,7 +49,7 @@ import static org.mockito.Mockito.when;
 public class TransportMultiSearchActionTests extends ESTestCase {
 
     public void testBatchExecute() throws Exception {
-        // Initialize depedencies of TransportMultiSearchAction
+        // Initialize dependencies of TransportMultiSearchAction
         Settings settings = Settings.builder()
                 .put("node.name", TransportMultiSearchActionTests.class.getSimpleName())
                 .build();
@@ -57,7 +58,7 @@ public class TransportMultiSearchActionTests extends ESTestCase {
         ThreadPool threadPool = new ThreadPool(settings);
         TaskManager taskManager = mock(TaskManager.class);
         TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
-                null) {
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null) {
             @Override
             public TaskManager getTaskManager() {
                 return taskManager;
diff --git a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
index 19342e5e604..189fd1de59c 100644
--- a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -83,7 +84,9 @@ public class AutoCreateIndexTests extends ESTestCase {
     public void testAutoCreationDisabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(false));
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()));
+        assertEquals("no such index and [action.auto_create_index] is [false]", e.getMessage());
     }
 
     public void testAutoCreationEnabled() {
@@ -110,7 +113,9 @@ public class AutoCreateIndexTests extends ESTestCase {
                 randomAsciiOfLengthBetween(1, 10)))
             .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
-        assertThat(autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()), equalTo(false));
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(randomAsciiOfLengthBetween(1, 10), buildClusterState()));
+        assertEquals("no such index and [index.mapper.dynamic] is [false]", e.getMessage());
     }
 
     public void testAutoCreationPatternEnabled() {
@@ -119,16 +124,17 @@ public class AutoCreateIndexTests extends ESTestCase {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationPatternDisabled() {
         Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), "-index*").build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
-        //default is false when patterns are specified
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "index" + randomAsciiOfLengthBetween(1, 5), "-index*");
+        /* When patterns are specified, even if the are all negative, the default is can't create. So a pure negative pattern is the same
+         * as false, really. */
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultiplePatternsWithWildcards() {
@@ -136,9 +142,9 @@ public class AutoCreateIndexTests extends ESTestCase {
                 randomFrom("+test*,-index*", "test*,-index*")).build();
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "index" + randomAsciiOfLengthBetween(1, 5), "-index*");
         assertThat(autoCreateIndex.shouldAutoCreate("test" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultiplePatternsNoWildcards() {
@@ -146,9 +152,9 @@ public class AutoCreateIndexTests extends ESTestCase {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("index" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("test" + randomAsciiOfLengthBetween(2, 5), clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "index" + randomAsciiOfLengthBetween(1, 5));
+        expectNotMatch(clusterState, autoCreateIndex, "test" + randomAsciiOfLengthBetween(2, 5));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationMultipleIndexNames() {
@@ -157,7 +163,7 @@ public class AutoCreateIndexTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
         assertThat(autoCreateIndex.shouldAutoCreate("test2", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testAutoCreationConflictingPatternsFirstWins() {
@@ -166,8 +172,8 @@ public class AutoCreateIndexTests extends ESTestCase {
         AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings);
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()).build();
         assertThat(autoCreateIndex.shouldAutoCreate("test1", clusterState), equalTo(true));
-        assertThat(autoCreateIndex.shouldAutoCreate("test2", clusterState), equalTo(false));
-        assertThat(autoCreateIndex.shouldAutoCreate("does_not_match" + randomAsciiOfLengthBetween(1, 5), clusterState), equalTo(false));
+        expectForbidden(clusterState, autoCreateIndex, "test2", "-test2");
+        expectNotMatch(clusterState, autoCreateIndex, "does_not_match" + randomAsciiOfLengthBetween(1, 5));
     }
 
     public void testUpdate() {
@@ -208,4 +214,18 @@ public class AutoCreateIndexTests extends ESTestCase {
         return new AutoCreateIndex(settings, new ClusterSettings(settings,
                 ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver(settings));
     }
+
+    private void expectNotMatch(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index) {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(index, clusterState));
+        assertEquals("no such index and [action.auto_create_index] ([" + autoCreateIndex.getAutoCreate() + "]) doesn't match",
+                e.getMessage());
+    }
+
+    private void expectForbidden(ClusterState clusterState, AutoCreateIndex autoCreateIndex, String index, String forbiddingPattern) {
+        IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () ->
+            autoCreateIndex.shouldAutoCreate(index, clusterState));
+        assertEquals("no such index and [action.auto_create_index] contains [" + forbiddingPattern
+                + "] which forbids automatic creation of the index", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java b/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java
new file mode 100644
index 00000000000..37f12ca8225
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/ContextPreservingActionListenerTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class ContextPreservingActionListenerTests extends ESTestCase {
+
+    public void testOriginalContextIsPreservedAfterOnResponse() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true),
+                        new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void aVoid) {
+                        assertEquals("bar", threadContext.getHeader("foo"));
+                        assertNull(threadContext.getHeader("not empty"));
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        throw new RuntimeException("onFailure shouldn't be called", e);
+                    }
+                });
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            actionListener.onResponse(null);
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+
+    public void testOriginalContextIsPreservedAfterOnFailure() throws Exception {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true),
+                        new ActionListener<Void>() {
+                            @Override
+                            public void onResponse(Void aVoid) {
+                                throw new RuntimeException("onResponse shouldn't be called");
+                            }
+
+                            @Override
+                            public void onFailure(Exception e) {
+                                assertEquals("bar", threadContext.getHeader("foo"));
+                                assertNull(threadContext.getHeader("not empty"));
+                            }
+                        });
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            actionListener.onFailure(null);
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+
+    public void testOriginalContextIsWhenListenerThrows() throws Exception {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            final boolean nonEmptyContext = randomBoolean();
+            if (nonEmptyContext) {
+                threadContext.putHeader("not empty", "value");
+            }
+            ContextPreservingActionListener<Void> actionListener;
+            try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+                threadContext.putHeader("foo", "bar");
+                actionListener = new ContextPreservingActionListener<>(threadContext.newRestorableContext(true),
+                        new ActionListener<Void>() {
+                            @Override
+                            public void onResponse(Void aVoid) {
+                                assertEquals("bar", threadContext.getHeader("foo"));
+                                assertNull(threadContext.getHeader("not empty"));
+                                throw new RuntimeException("onResponse called");
+                            }
+
+                            @Override
+                            public void onFailure(Exception e) {
+                                assertEquals("bar", threadContext.getHeader("foo"));
+                                assertNull(threadContext.getHeader("not empty"));
+                                throw new RuntimeException("onFailure called");
+                            }
+                        });
+            }
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            RuntimeException e = expectThrows(RuntimeException.class, () -> actionListener.onResponse(null));
+            assertEquals("onResponse called", e.getMessage());
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+
+            e = expectThrows(RuntimeException.class, () -> actionListener.onFailure(null));
+            assertEquals("onFailure called", e.getMessage());
+
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals(nonEmptyContext ? "value" : null, threadContext.getHeader("not empty"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index d656e0f62a9..c687fc6cabc 100644
--- a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -31,7 +31,8 @@ public class IndicesOptionsTests extends ESTestCase {
     public void testSerialization() throws Exception {
         int iterations = randomIntBetween(5, 20);
         for (int i = 0; i < iterations; i++) {
-            IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
+            IndicesOptions indicesOptions = IndicesOptions.fromOptions(
+                randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean());
 
             BytesStreamOutput output = new BytesStreamOutput();
             Version outputVersion = randomVersion(random());
diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
index 6df7c39aea9..230812bec88 100644
--- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
@@ -221,7 +221,7 @@ public class TransportActionFilterChainTests extends ESTestCase {
         }
     }
 
-    private static enum RequestOperation implements RequestCallback {
+    private enum RequestOperation implements RequestCallback {
         CONTINUE_PROCESSING {
             @Override
             public <Request extends ActionRequest, Response extends ActionResponse> void execute(Task task, String action, Request request,
diff --git a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
index fbeac88dbe1..cab27d74c7d 100644
--- a/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
+++ b/core/src/test/java/org/elasticsearch/action/support/WaitActiveShardCountIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -44,15 +45,15 @@ public class WaitActiveShardCountIT extends ESIntegTestCase {
         assertAcked(createIndexResponse);
 
         // indexing, by default, will work (waiting for one shard copy only)
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test")).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON).execute().actionGet();
         try {
-            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                     .setWaitForActiveShards(2) // wait for 2 active shard copies
                     .setTimeout(timeValueMillis(100)).execute().actionGet();
             fail("can't index, does not enough active shard copies");
         } catch (UnavailableShardsException e) {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [2] (have 1, needed 2). Timeout: [100ms], request: [index {[test][type1][1], source[{ \"type1\" : { \"id\" : \"1\", \"name\" : \"test\" } }]}]"));
+            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [2] (have 1, needed 2). Timeout: [100ms], request: [BulkShardRequest to [test] containing [1] requests]"));
             // but really, all is well
         }
 
@@ -70,18 +71,18 @@ public class WaitActiveShardCountIT extends ESIntegTestCase {
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
 
         // this should work, since we now have two
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                 .setWaitForActiveShards(2)
                 .setTimeout(timeValueSeconds(1)).execute().actionGet();
 
         try {
-            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+            client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                     .setWaitForActiveShards(ActiveShardCount.ALL)
                     .setTimeout(timeValueMillis(100)).execute().actionGet();
             fail("can't index, not enough active shard copies");
         } catch (UnavailableShardsException e) {
             assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
-            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [" + ActiveShardCount.ALL + "] (have 2, needed 3). Timeout: [100ms], request: [index {[test][type1][1], source[{ \"type1\" : { \"id\" : \"1\", \"name\" : \"test\" } }]}]"));
+            assertThat(e.getMessage(), equalTo("[test][0] Not enough active copies to meet shard count of [" + ActiveShardCount.ALL + "] (have 2, needed 3). Timeout: [100ms], request: [BulkShardRequest to [test] containing [1] requests]"));
             // but really, all is well
         }
 
@@ -92,7 +93,7 @@ public class WaitActiveShardCountIT extends ESIntegTestCase {
         assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
 
         // this should work, since we now have all shards started
-        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"))
+        client().prepareIndex("test", "type1", "1").setSource(source("1", "test"), XContentType.JSON)
                 .setWaitForActiveShards(ActiveShardCount.ALL)
                 .setTimeout(timeValueSeconds(1)).execute().actionGet();
     }
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 2f743f9bf6b..ef025e3c37b 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -116,7 +116,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     class TestTransportBroadcastByNodeAction extends TransportBroadcastByNodeAction<Request, Response, TransportBroadcastByNodeAction.EmptyResult> {
         private final Map<ShardRouting, Object> shards = new HashMap<>();
 
-        public TestTransportBroadcastByNodeAction(Settings settings, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request, String executor) {
+        TestTransportBroadcastByNodeAction(Settings settings, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request, String executor) {
             super(settings, "indices:admin/test", THREAD_POOL, TransportBroadcastByNodeActionTests.this.clusterService, transportService, actionFilters, indexNameExpressionResolver, request, executor);
         }
 
@@ -170,7 +170,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
-        public MyResolver() {
+        MyResolver() {
             super(Settings.EMPTY);
         }
 
@@ -191,7 +191,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
         final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         setClusterState(clusterService, TEST_INDEX);
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 66fe95be0be..ae543aa14cd 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -87,7 +87,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(),
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 10e9d9d3f3d..de3af56a120 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -182,7 +182,7 @@ public class TransportNodesActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
         transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         int numNodes = randomIntBetween(3, 10);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 260f70e19ed..7fa900a921c 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -96,7 +96,7 @@ public class BroadcastReplicationTests extends ESTestCase {
             new NetworkService(Settings.EMPTY, Collections.emptyList()));
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService,
@@ -202,7 +202,7 @@ public class BroadcastReplicationTests extends ESTestCase {
     private class TestBroadcastReplicationAction extends TransportBroadcastReplicationAction<DummyBroadcastRequest, BroadcastResponse, BasicReplicationRequest, ReplicationResponse> {
         protected final Set<Tuple<ShardId, ActionListener<ReplicationResponse>>> capturedShardRequests = ConcurrentCollections.newConcurrentSet();
 
-        public TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+        TestBroadcastReplicationAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                               TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                               TransportReplicationAction replicatedBroadcastShardAction) {
             super("test-broadcast-replication-action", DummyBroadcastRequest::new, settings, threadPool, clusterService, transportService,
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index 47325bf5f98..a701193d220 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -33,8 +33,6 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.IndexShardNotStartedException;
@@ -42,7 +40,6 @@ import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -366,13 +363,8 @@ public class ReplicationOperationTests extends ESTestCase {
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
+        public String toString() {
+            return "Request{}";
         }
     }
 
@@ -411,7 +403,7 @@ public class ReplicationOperationTests extends ESTestCase {
             private final Request request;
             private ShardInfo shardInfo;
 
-            public Result(Request request) {
+            Result(Request request) {
                 this.request = request;
             }
 
@@ -525,12 +517,12 @@ public class ReplicationOperationTests extends ESTestCase {
     }
 
     class TestReplicationOperation extends ReplicationOperation<Request, Request, TestPrimary.Result> {
-        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+        TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
                 ActionListener<TestPrimary.Result> listener, Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier) {
             this(request, primary, listener, true, replicas, clusterStateSupplier, ReplicationOperationTests.this.logger, "test");
         }
 
-        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+        TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
                 ActionListener<TestPrimary.Result> listener, boolean executeOnReplicas,
                 Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier, Logger logger, String opType) {
             super(request, primary, listener, executeOnReplicas, replicas, clusterStateSupplier, logger, opType);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
index 7d82b744133..87eecc218f3 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationResponseTests.java
@@ -20,26 +20,29 @@
 package org.elasticsearch.action.support.replication;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.RoutingMissingException;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.IndexShardRecoveringException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.RandomObjects;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Set;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class ReplicationResponseTests extends ESTestCase {
 
@@ -62,12 +65,13 @@ public class ReplicationResponseTests extends ESTestCase {
                     new ReplicationResponse.ShardInfo(shardInfo.getTotal(), shardInfo.getSuccessful() + 1, shardInfo.getFailures()));
             mutations.add(() -> {
                 int nbFailures = randomIntBetween(1, 5);
-                return new ReplicationResponse.ShardInfo(shardInfo.getTotal(), shardInfo.getSuccessful(),  randomFailures(nbFailures));
+                ReplicationResponse.ShardInfo.Failure[] randomFailures = RandomObjects.randomShardInfoFailures(random(), nbFailures);
+                return new ReplicationResponse.ShardInfo(shardInfo.getTotal(), shardInfo.getSuccessful(), randomFailures);
             });
             return randomFrom(mutations).get();
         };
 
-        checkEqualsAndHashCode(randomShardInfo(), copy, mutate);
+        checkEqualsAndHashCode(RandomObjects.randomShardInfo(random(), randomBoolean()), copy, mutate);
     }
 
     public void testFailureEqualsAndHashcode() {
@@ -82,11 +86,16 @@ public class ReplicationResponseTests extends ESTestCase {
             List<Supplier<ReplicationResponse.ShardInfo.Failure>> mutations = new ArrayList<>();
 
             final Index index = failure.fullShardId().getIndex();
-            final ShardId randomIndex = new ShardId(randomUnicodeOfCodepointLength(5), index.getUUID(), failure.shardId());
+            final Set<String> indexNamePool = new HashSet<>(Arrays.asList(
+                    randomUnicodeOfCodepointLength(5), randomUnicodeOfCodepointLength(6)));
+            indexNamePool.remove(index.getName());
+            final ShardId randomIndex = new ShardId(randomFrom(indexNamePool), index.getUUID(), failure.shardId());
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(randomIndex, failure.nodeId(), (Exception) failure.getCause(),
                     failure.status(), failure.primary()));
 
-            final ShardId randomUUID = new ShardId(index.getName(), randomUnicodeOfCodepointLength(5), failure.shardId());
+            final Set<String> uuidPool = new HashSet<>(Arrays.asList(randomUnicodeOfCodepointLength(5), randomUnicodeOfCodepointLength(6)));
+            uuidPool.remove(index.getUUID());
+            final ShardId randomUUID = new ShardId(index.getName(), randomFrom(uuidPool), failure.shardId());
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(randomUUID, failure.nodeId(), (Exception) failure.getCause(),
                     failure.status(), failure.primary()));
 
@@ -94,15 +103,22 @@ public class ReplicationResponseTests extends ESTestCase {
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(randomShardId, failure.nodeId(), (Exception) failure.getCause(),
                     failure.status(), failure.primary()));
 
-            final String randomNode = randomUnicodeOfLength(3);
+            final Set<String> nodeIdPool = new HashSet<>(Arrays.asList(randomUnicodeOfLength(3), randomUnicodeOfLength(4)));
+            nodeIdPool.remove(failure.nodeId());
+            final String randomNode = randomFrom(nodeIdPool);
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(failure.fullShardId(), randomNode, (Exception) failure.getCause(),
                     failure.status(), failure.primary()));
 
-            final Exception randomException = randomFrom(new IllegalStateException("a"), new IllegalArgumentException("b"));
+            final Set<Exception> exceptionPool = new HashSet<>(Arrays.asList(
+                    new IllegalStateException("a"), new IllegalArgumentException("b")));
+            exceptionPool.remove(failure.getCause());
+            final Exception randomException = randomFrom(exceptionPool);
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(failure.fullShardId(), failure.nodeId(), randomException,
                     failure.status(), failure.primary()));
 
-            final RestStatus randomStatus = randomFrom(RestStatus.values());
+            final Set<RestStatus> otherStatuses = new HashSet<>(Arrays.asList(RestStatus.values()));
+            otherStatuses.remove(failure.status());
+            final RestStatus randomStatus = randomFrom(otherStatuses);
             mutations.add(() -> new ReplicationResponse.ShardInfo.Failure(failure.fullShardId(), failure.nodeId(),
                     (Exception) failure.getCause(), randomStatus, failure.primary()));
 
@@ -113,101 +129,171 @@ public class ReplicationResponseTests extends ESTestCase {
             return randomFrom(mutations).get();
         };
 
-        checkEqualsAndHashCode(randomFailure(), copy, mutate);
+        checkEqualsAndHashCode(RandomObjects.randomShardInfoFailure(random()), copy, mutate);
     }
 
     public void testShardInfoToXContent() throws IOException {
-        ReplicationResponse.ShardInfo shardInfo = new ReplicationResponse.ShardInfo(5, 3);
+        final XContentType xContentType = randomFrom(XContentType.values());
 
-        final XContent xContent = randomFrom(XContentType.values()).xContent();
-        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
-            builder.startObject();
-            shardInfo.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
+        final ReplicationResponse.ShardInfo shardInfo = new ReplicationResponse.ShardInfo(5, 3);
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfo, xContentType, randomBoolean());
 
-            // Expected JSON is {"_shards":{"total":5,"successful":3,"failed":0}}
-            try (XContentParser parser = xContent.createParser(builder.bytes())) {
-                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("_shards", parser.currentName());
-                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("total", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getTotal(), parser.intValue());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("successful", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getSuccessful(), parser.intValue());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("failed", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getFailed(), parser.intValue());
-                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
-                assertNull(parser.nextToken());
-            }
+        // Expected JSON is {"total":5,"successful":3,"failed":0}
+        assertThat(shardInfo, instanceOf(ToXContentObject.class));
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("total", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getTotal(), parser.intValue());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("successful", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getSuccessful(), parser.intValue());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("failed", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getFailed(), parser.intValue());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
         }
     }
 
-    public void testRandomShardInfoToXContent() throws IOException {
-        final ReplicationResponse.ShardInfo shardInfo = randomShardInfo();
+    public void testShardInfoToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
 
-        final XContent xContent = randomFrom(XContentType.values()).xContent();
-        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
-            builder.startObject();
-            shardInfo.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
+        final ReplicationResponse.ShardInfo shardInfo = new ReplicationResponse.ShardInfo(randomIntBetween(1, 5), randomIntBetween(1, 5));
+        boolean humanReadable = randomBoolean();
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfo, xContentType, humanReadable);
 
-            try (XContentParser parser = xContent.createParser(builder.bytes())) {
-                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("_shards", parser.currentName());
-                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("total", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getTotal(), parser.intValue());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("successful", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getSuccessful(), parser.intValue());
-                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals("failed", parser.currentName());
-                assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
-                assertEquals(shardInfo.getFailed(), parser.intValue());
+        ReplicationResponse.ShardInfo parsedShardInfo;
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            // Move to the first start object
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedShardInfo = ReplicationResponse.ShardInfo.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        // We can use assertEquals because the shardInfo doesn't have a failure (and exceptions)
+        assertEquals(shardInfo, parsedShardInfo);
 
-                if (shardInfo.getFailures() != null && shardInfo.getFailures().length > 0) {
-                    assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                    assertEquals("failures", parser.currentName());
-                    assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+        BytesReference parsedShardInfoBytes = XContentHelper.toXContent(parsedShardInfo, xContentType, humanReadable);
+        assertEquals(shardInfoBytes, parsedShardInfoBytes);
+    }
 
-                    for (int i = 0; i < shardInfo.getFailures().length; i++) {
-                        assertFailure(parser, shardInfo.getFailures()[i]);
-                    }
-                    assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken());
+    public void testShardInfoWithFailureToXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        final ReplicationResponse.ShardInfo shardInfo = RandomObjects.randomShardInfo(random(), true);
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfo, xContentType, randomBoolean());
+
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("total", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getTotal(), parser.intValue());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("successful", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getSuccessful(), parser.intValue());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            assertEquals("failed", parser.currentName());
+            assertEquals(XContentParser.Token.VALUE_NUMBER, parser.nextToken());
+            assertEquals(shardInfo.getFailed(), parser.intValue());
+
+            if (shardInfo.getFailures() != null && shardInfo.getFailures().length > 0) {
+                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+                assertEquals("failures", parser.currentName());
+                assertEquals(XContentParser.Token.START_ARRAY, parser.nextToken());
+
+                for (int i = 0; i < shardInfo.getFailures().length; i++) {
+                    assertFailure(parser, shardInfo.getFailures()[i]);
                 }
-
-                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
-                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
-                assertNull(parser.nextToken());
+                assertEquals(XContentParser.Token.END_ARRAY, parser.nextToken());
             }
+
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+    }
+
+    public void testRandomShardInfoFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        final ReplicationResponse.ShardInfo shardInfo = RandomObjects.randomShardInfo(random(), randomBoolean());
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfo, xContentType, randomBoolean());
+
+        ReplicationResponse.ShardInfo parsedShardInfo;
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            // Move to the first start object
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedShardInfo = ReplicationResponse.ShardInfo.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        // We can't use assertEquals to compare the original ShardInfo with the parsed ShardInfo
+        // because it may include random failures with exceptions, and exception types are not
+        // preserved during ToXContent->FromXContent process.
+        assertNotNull(parsedShardInfo);
+        assertEquals(shardInfo.getTotal(), parsedShardInfo.getTotal());
+        assertEquals(shardInfo.getSuccessful(), parsedShardInfo.getSuccessful());
+        assertEquals(shardInfo.getFailed(), parsedShardInfo.getFailed());
+        assertEquals(shardInfo.getFailures().length, parsedShardInfo.getFailures().length);
+
+        for (int i = 0; i < shardInfo.getFailures().length; i++) {
+            ReplicationResponse.ShardInfo.Failure parsedFailure = parsedShardInfo.getFailures()[i];
+            ReplicationResponse.ShardInfo.Failure failure = shardInfo.getFailures()[i];
+
+            assertEquals(failure.index(), parsedFailure.index());
+            assertEquals(failure.shardId(), parsedFailure.shardId());
+            assertEquals(failure.nodeId(), parsedFailure.nodeId());
+            assertEquals(failure.status(), parsedFailure.status());
+            assertEquals(failure.primary(), parsedFailure.primary());
+
+            Throwable cause = failure.getCause();
+            String expectedMessage = "Elasticsearch exception [type=" + ElasticsearchException.getExceptionName(cause)
+                    + ", reason=" + cause.getMessage() + "]";
+            assertEquals(expectedMessage, parsedFailure.getCause().getMessage());
         }
     }
 
     public void testRandomFailureToXContent() throws IOException {
-        ReplicationResponse.ShardInfo.Failure shardInfoFailure = randomFailure();
+        final XContentType xContentType = randomFrom(XContentType.values());
 
-        final XContent xContent = randomFrom(XContentType.values()).xContent();
-        try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
-            shardInfoFailure.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        final ReplicationResponse.ShardInfo.Failure shardInfoFailure = RandomObjects.randomShardInfoFailure(random());
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfoFailure, xContentType, randomBoolean());
 
-            try (XContentParser parser = xContent.createParser(builder.bytes())) {
-                assertFailure(parser, shardInfoFailure);
-            }
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            assertFailure(parser, shardInfoFailure);
         }
     }
 
+    public void testRandomFailureToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        final ReplicationResponse.ShardInfo.Failure shardInfoFailure = RandomObjects.randomShardInfoFailure(random());;
+        final BytesReference shardInfoBytes = XContentHelper.toXContent(shardInfoFailure, xContentType, randomBoolean());
+
+        ReplicationResponse.ShardInfo.Failure parsedFailure;
+        try (XContentParser parser = createParser(xContentType.xContent(), shardInfoBytes)) {
+            // Move to the first start object
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            parsedFailure = ReplicationResponse.ShardInfo.Failure.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(shardInfoFailure.index(), parsedFailure.index());
+        assertEquals(shardInfoFailure.shardId(), parsedFailure.shardId());
+        assertEquals(shardInfoFailure.nodeId(), parsedFailure.nodeId());
+        assertEquals(shardInfoFailure.status(), parsedFailure.status());
+        assertEquals(shardInfoFailure.primary(), parsedFailure.primary());
+
+        Throwable cause = shardInfoFailure.getCause();
+        String expectedMessage = "Elasticsearch exception [type=" + ElasticsearchException.getExceptionName(cause)
+                + ", reason=" + cause.getMessage() + "]";
+        assertEquals(expectedMessage, parsedFailure.getCause().getMessage());
+    }
+
     private static void assertFailure(XContentParser parser, ReplicationResponse.ShardInfo.Failure failure) throws IOException {
         assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
         assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
@@ -254,10 +340,16 @@ public class ReplicationResponseTests extends ESTestCase {
             ElasticsearchException ex = (ElasticsearchException) cause;
             for (String name : ex.getHeaderKeys()) {
                 assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
-                assertEquals(name.replaceFirst("es.", ""), parser.currentName());
+                assertEquals(name, parser.currentName());
                 assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
                 assertEquals(ex.getHeader(name).get(0), parser.text());
             }
+            for (String name : ex.getMetadataKeys()) {
+                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+                assertEquals(name.replaceFirst("es.", ""), parser.currentName());
+                assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
+                assertEquals(ex.getMetadata(name).get(0), parser.text());
+            }
             if (ex.getCause() != null) {
                 assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
                 assertEquals("caused_by", parser.currentName());
@@ -266,32 +358,4 @@ public class ReplicationResponseTests extends ESTestCase {
         }
         assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
     }
-
-    private static ReplicationResponse.ShardInfo randomShardInfo() {
-        int total = randomIntBetween(1, 10);
-        int successful = randomIntBetween(0, total);
-        return new ReplicationResponse.ShardInfo(total, successful, randomFailures(Math.max(0, (total - successful))));
-    }
-
-    private static ReplicationResponse.ShardInfo.Failure[] randomFailures(int nbFailures) {
-        List<ReplicationResponse.ShardInfo.Failure> randomFailures = new ArrayList<>(nbFailures);
-        for (int i = 0; i < nbFailures; i++) {
-            randomFailures.add(randomFailure());
-        }
-        return randomFailures.toArray(new ReplicationResponse.ShardInfo.Failure[nbFailures]);
-    }
-
-    private static ReplicationResponse.ShardInfo.Failure randomFailure() {
-        return new ReplicationResponse.ShardInfo.Failure(
-                new ShardId(randomAsciiOfLength(5), randomAsciiOfLength(5), randomIntBetween(0, 5)),
-                randomAsciiOfLength(3),
-                randomFrom(
-                        new IndexShardRecoveringException(new ShardId("_test", "_0", 5)),
-                        new ElasticsearchException(new IllegalArgumentException("argument is wrong")),
-                        new RoutingMissingException("_test", "_type", "_id")
-                ),
-                randomFrom(RestStatus.values()),
-                randomBoolean()
-        );
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 655244e286f..c2c484fe3e6 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.action.support.replication;
 
+import org.apache.lucene.store.AlreadyClosedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -54,13 +56,14 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.EngineClosedException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.ClusterStateChanges;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -152,7 +155,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
@@ -184,6 +187,12 @@ public class TransportReplicationActionTests extends ESTestCase {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
+        Action action = new Action(Settings.EMPTY, "testActionWithBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel globalBlockLevel() {
+                return ClusterBlockLevel.WRITE;
+            }
+        };
 
         ClusterBlocks.Builder block = ClusterBlocks.builder()
             .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
@@ -216,6 +225,17 @@ public class TransportReplicationActionTests extends ESTestCase {
         assertListenerThrows("primary phase should fail operation when moving from a retryable block to a non-retryable one", listener,
             ClusterBlockException.class);
         assertIndexShardUninitialized();
+
+        action = new Action(Settings.EMPTY, "testActionWithNoBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel globalBlockLevel() {
+                return null;
+            }
+        };
+        listener = new PlainActionFuture<>();
+        reroutePhase = action.new ReroutePhase(task, new Request().timeout("5ms"), listener);
+        reroutePhase.run();
+        assertListenerThrows("should fail with an IndexNotFoundException when no blocks checked", listener, IndexNotFoundException.class);
     }
 
     public void assertIndexShardUninitialized() {
@@ -337,6 +357,34 @@ public class TransportReplicationActionTests extends ESTestCase {
 
     }
 
+    public void testClosedIndexOnReroute() throws InterruptedException {
+        final String index = "test";
+        // no replicas in oder to skip the replication part
+        setState(clusterService, new ClusterStateChanges(xContentRegistry(), threadPool).closeIndices(state(index, true,
+            ShardRoutingState.UNASSIGNED), new CloseIndexRequest(index)));
+        logger.debug("--> using initial state:\n{}", clusterService.state());
+        Request request = new Request(new ShardId("test", "_na_", 0)).timeout("1ms");
+        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        ReplicationTask task = maybeTask();
+
+        ClusterBlockLevel indexBlockLevel = randomBoolean() ? ClusterBlockLevel.WRITE : null;
+        Action action = new Action(Settings.EMPTY, "testActionWithBlocks", transportService, clusterService, shardStateAction, threadPool) {
+            @Override
+            protected ClusterBlockLevel indexBlockLevel() {
+                return indexBlockLevel;
+            }
+        };
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        reroutePhase.run();
+        if (indexBlockLevel == ClusterBlockLevel.WRITE) {
+            assertListenerThrows("must throw block exception", listener, ClusterBlockException.class);
+        } else {
+            assertListenerThrows("must throw index closed exception", listener, IndexClosedException.class);
+        }
+        assertPhase(task, "failed");
+        assertFalse(request.isRetrySet.get());
+    }
+
     public void testStalePrimaryShardOnReroute() throws InterruptedException {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
@@ -383,12 +431,12 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
     }
 
-    private ElasticsearchException randomRetryPrimaryException(ShardId shardId) {
+    private Exception randomRetryPrimaryException(ShardId shardId) {
         return randomFrom(
             new ShardNotFoundException(shardId),
             new IndexNotFoundException(shardId.getIndex()),
             new IndexShardClosedException(shardId),
-            new EngineClosedException(shardId),
+            new AlreadyClosedException(shardId + " primary is closed"),
             new ReplicationOperation.RetryOnPrimaryException(shardId, "hello")
         );
     }
@@ -444,9 +492,11 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
         action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -494,9 +544,11 @@ public class TransportReplicationActionTests extends ESTestCase {
         AtomicBoolean executed = new AtomicBoolean();
         action.new AsyncPrimaryAction(request, primaryShard.allocationId().getRelocationId(), createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -529,7 +581,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         };
         Action.PrimaryShardReference primary = action.new PrimaryShardReference(shard, releasable);
         final Request request = new Request();
-        Request replicaRequest = primary.perform(request).replicaRequest;
+        Request replicaRequest = (Request) primary.perform(request).replicaRequest;
 
         assertThat(replicaRequest.primaryTerm(), equalTo(primaryTerm));
 
@@ -639,13 +691,15 @@ public class TransportReplicationActionTests extends ESTestCase {
         action.new AsyncPrimaryAction(new Request(shardId), primaryShard.allocationId().getId(),
             createTransportChannel(new PlainActionFuture<>()), null) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult<Request, Response>> createReplicatedOperation(
+                    Request request, ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                    TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
                     boolean executeOnReplicas) {
                 assertFalse(executeOnReplicas);
                 assertFalse(executed.getAndSet(true));
                 return new NoopReplicationOperation(request, actionListener);
             }
+
         }.run();
         assertThat(executed.get(), equalTo(true));
     }
@@ -705,9 +759,11 @@ public class TransportReplicationActionTests extends ESTestCase {
         final boolean respondWithError = i == 3;
         action.new AsyncPrimaryAction(request, primaryShard.allocationId().getId(), createTransportChannel(listener), task) {
             @Override
-            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
-                    boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, TransportReplicationAction.PrimaryResult<Request, Response>>
+            createReplicatedOperation(Request request,
+                                      ActionListener<TransportReplicationAction.PrimaryResult<Request, Response>> actionListener,
+                                      TransportReplicationAction<Request, Request, Response>.PrimaryShardReference primaryShardReference,
+                                      boolean executeOnReplicas) {
                 assertIndexShardCounter(1);
                 if (throwExceptionOnCreation) {
                     throw new ElasticsearchException("simulated exception, during createReplicatedOperation");
@@ -984,6 +1040,11 @@ public class TransportReplicationActionTests extends ESTestCase {
             super.onRetry();
             isRetrySet.set(true);
         }
+
+        @Override
+        public String toString() {
+            return "Request{}";
+        }
     }
 
     static class Response extends ReplicationResponse {
@@ -1095,14 +1156,15 @@ public class TransportReplicationActionTests extends ESTestCase {
         return indexShard;
     }
 
-    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Action.PrimaryResult> {
-        public NoopReplicationOperation(Request request, ActionListener<Action.PrimaryResult> listener) {
+    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Action.PrimaryResult<Request, Response>> {
+        NoopReplicationOperation(Request request, ActionListener<Action.PrimaryResult<Request, Response>> listener) {
             super(request, null, listener, true, null, null, TransportReplicationActionTests.this.logger, "noop");
         }
 
         @Override
         public void execute() throws Exception {
-            this.resultListener.onResponse(action.new PrimaryResult(null, new Response()));
+            // Using the diamond operator (<>) prevents Eclipse from being able to compile this code
+            this.resultListener.onResponse(new TransportReplicationAction.PrimaryResult<Request, Response>(null, new Response()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index 571bbfa72e0..82cd013c195 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -36,7 +36,6 @@ import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 
 import java.util.HashSet;
-import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 import static org.mockito.Matchers.any;
@@ -55,21 +54,27 @@ public class TransportWriteActionTests extends ESTestCase {
     }
 
     public void testPrimaryNoRefreshCall() throws Exception {
-        noRefreshCall(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond);
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testReplicaNoRefreshCall() throws Exception {
-        noRefreshCall(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond);
-    }
-
-    private <Result, Response> void noRefreshCall(ThrowingTriFunction<TestAction, TestRequest, IndexShard, Result> action,
-                                        BiConsumer<Result, CapturingActionListener<Response>> responder)
-            throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit
-        Result result = action.apply(new TestAction(), request, indexShard);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result =
+                testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
         assertNotNull(listener.response);
         assertNull(listener.failure);
         verify(indexShard, never()).refresh(any());
@@ -77,45 +82,43 @@ public class TransportWriteActionTests extends ESTestCase {
     }
 
     public void testPrimaryImmediateRefresh() throws Exception {
-        immediateRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond, r -> assertTrue(r.forcedRefresh));
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
+        assertTrue(listener.response.forcedRefresh);
+        verify(indexShard).refresh("refresh_flag_index");
+        verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testReplicaImmediateRefresh() throws Exception {
-        immediateRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, r -> {});
-    }
-
-    private <Result, Response> void immediateRefresh(ThrowingTriFunction<TestAction, TestRequest, IndexShard, Result> action,
-                                                     BiConsumer<Result, CapturingActionListener<Response>> responder,
-                                                     Consumer<Response> responseChecker) throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
-        Result result = action.apply(new TestAction(), request, indexShard);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result =
+                testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
         assertNotNull(listener.response);
         assertNull(listener.failure);
-        responseChecker.accept(listener.response);
         verify(indexShard).refresh("refresh_flag_index");
         verify(indexShard, never()).addRefreshListener(any(), any());
     }
 
     public void testPrimaryWaitForRefresh() throws Exception {
-        waitForRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond,
-            (r, forcedRefresh) -> assertEquals(forcedRefresh, r.forcedRefresh));
-    }
-
-    public void testReplicaWaitForRefresh() throws Exception {
-        waitForRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, (r, forcedRefresh) -> {});
-    }
-
-    private <Result, Response> void waitForRefresh(ThrowingTriFunction<TestAction, TestRequest, IndexShard, Result> action,
-                                                   BiConsumer<Result, CapturingActionListener<Response>> responder,
-                                         BiConsumer<Response, Boolean> resultChecker) throws Exception {
         TestRequest request = new TestRequest();
         request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
-        Result result = action.apply(new TestAction(), request, indexShard);
-        CapturingActionListener<Response> listener = new CapturingActionListener<>();
-        responder.accept(result, listener);
+
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> result =
+                testAction.shardOperationOnPrimary(request, indexShard);
+        CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
+        result.respond(listener);
         assertNull(listener.response); // Haven't reallresponded yet
 
         @SuppressWarnings({ "unchecked", "rawtypes" })
@@ -128,13 +131,33 @@ public class TransportWriteActionTests extends ESTestCase {
         refreshListener.getValue().accept(forcedRefresh);
         assertNotNull(listener.response);
         assertNull(listener.failure);
-        resultChecker.accept(listener.response, forcedRefresh);
+        assertEquals(forcedRefresh, listener.response.forcedRefresh);
+    }
+
+    public void testReplicaWaitForRefresh() throws Exception {
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        TestAction testAction = new TestAction();
+        TransportWriteAction.WriteReplicaResult<TestRequest> result = testAction.shardOperationOnReplica(request, indexShard);
+        CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
+        result.respond(listener);
+        assertNull(listener.response); // Haven't responded yet
+        @SuppressWarnings({ "unchecked", "rawtypes" })
+        ArgumentCaptor<Consumer<Boolean>> refreshListener = ArgumentCaptor.forClass((Class) Consumer.class);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard).addRefreshListener(any(), refreshListener.capture());
+
+        // Now we can fire the listener manually and we'll get a response
+        boolean forcedRefresh = randomBoolean();
+        refreshListener.getValue().accept(forcedRefresh);
+        assertNotNull(listener.response);
+        assertNull(listener.failure);
     }
 
     public void testDocumentFailureInShardOperationOnPrimary() throws Exception {
         TestRequest request = new TestRequest();
         TestAction testAction = new TestAction(true, true);
-        TransportWriteAction<TestRequest, TestRequest, TestResponse>.WritePrimaryResult writePrimaryResult =
+        TransportWriteAction.WritePrimaryResult<TestRequest, TestResponse> writePrimaryResult =
                 testAction.shardOperationOnPrimary(request, indexShard);
         CapturingActionListener<TestResponse> listener = new CapturingActionListener<>();
         writePrimaryResult.respond(listener);
@@ -145,7 +168,7 @@ public class TransportWriteActionTests extends ESTestCase {
     public void testDocumentFailureInShardOperationOnReplica() throws Exception {
         TestRequest request = new TestRequest();
         TestAction testAction = new TestAction(randomBoolean(), true);
-        TransportWriteAction<TestRequest, TestRequest, TestResponse>.WriteReplicaResult writeReplicaResult =
+        TransportWriteAction.WriteReplicaResult<TestRequest> writeReplicaResult =
                 testAction.shardOperationOnReplica(request, indexShard);
         CapturingActionListener<TransportResponse.Empty> listener = new CapturingActionListener<>();
         writeReplicaResult.respond(listener);
@@ -163,8 +186,8 @@ public class TransportWriteActionTests extends ESTestCase {
         }
         protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentFailureOnReplica) {
             super(Settings.EMPTY, "test",
-                    new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null), null, null, null,
-                    null, new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY), TestRequest::new,
+                    new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null), null,
+                    null, null, null, new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY), TestRequest::new,
                     TestRequest::new, ThreadPool.Names.SAME);
             this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary;
             this.withDocumentFailureOnReplica = withDocumentFailureOnReplica;
@@ -176,32 +199,38 @@ public class TransportWriteActionTests extends ESTestCase {
         }
 
         @Override
-        protected WritePrimaryResult shardOperationOnPrimary(TestRequest request, IndexShard primary) throws Exception {
-            final WritePrimaryResult primaryResult;
+        protected WritePrimaryResult<TestRequest, TestResponse> shardOperationOnPrimary(
+                TestRequest request, IndexShard primary) throws Exception {
+            final WritePrimaryResult<TestRequest, TestResponse> primaryResult;
             if (withDocumentFailureOnPrimary) {
-                primaryResult = new WritePrimaryResult(request, null, null, new RuntimeException("simulated"), primary);
+                primaryResult = new WritePrimaryResult<>(request, null, null, new RuntimeException("simulated"), primary, logger);
             } else {
-                primaryResult = new WritePrimaryResult(request, new TestResponse(), location, null, primary);
+                primaryResult = new WritePrimaryResult<>(request, new TestResponse(), location, null, primary, logger);
             }
             return primaryResult;
         }
 
         @Override
-        protected WriteReplicaResult shardOperationOnReplica(TestRequest request, IndexShard replica) throws Exception {
-            final WriteReplicaResult replicaResult;
+        protected WriteReplicaResult<TestRequest> shardOperationOnReplica(TestRequest request, IndexShard replica) throws Exception {
+            final WriteReplicaResult<TestRequest> replicaResult;
             if (withDocumentFailureOnReplica) {
-                replicaResult = new WriteReplicaResult(request, null, new RuntimeException("simulated"), replica);
+                replicaResult = new WriteReplicaResult<>(request, null, new RuntimeException("simulated"), replica, logger);
             } else {
-                replicaResult = new WriteReplicaResult(request, location, null, replica);
+                replicaResult = new WriteReplicaResult<>(request, location, null, replica, logger);
             }
             return replicaResult;
         }
     }
 
     private static class TestRequest extends ReplicatedWriteRequest<TestRequest> {
-        public TestRequest() {
+        TestRequest() {
             setShardId(new ShardId("test", "test", 1));
         }
+
+        @Override
+        public String toString() {
+            return "TestRequest{}";
+        }
     }
 
     private static class TestResponse extends ReplicationResponse implements WriteResponse {
@@ -227,8 +256,4 @@ public class TransportWriteActionTests extends ESTestCase {
             this.failure = failure;
         }
     }
-
-    private interface ThrowingTriFunction<A, B, C, R> {
-        R apply(A a, B b, C c) throws Exception;
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 96e90cedf7f..c83a76ddc1e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -88,7 +88,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
     class TestTransportInstanceSingleOperationAction extends TransportInstanceSingleOperationAction<Request, Response> {
         private final Map<ShardId, Object> shards = new HashMap<>();
 
-        public TestTransportInstanceSingleOperationAction(Settings settings, String actionName, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
+        TestTransportInstanceSingleOperationAction(Settings settings, String actionName, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
             super(settings, actionName, THREAD_POOL, TransportInstanceSingleOperationActionTests.this.clusterService, transportService, actionFilters, indexNameExpressionResolver, request);
         }
 
@@ -122,7 +122,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
     }
 
     class MyResolver extends IndexNameExpressionResolver {
-        public MyResolver() {
+        MyResolver() {
             super(Settings.EMPTY);
         }
 
@@ -144,7 +144,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
         transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new TestTransportInstanceSingleOperationAction(
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
index 1611c63d2ba..294a0ffde8f 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
@@ -137,9 +137,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, null);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
@@ -193,9 +191,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, null);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
@@ -252,9 +248,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         assertThat(iterator.next(), Matchers.nullValue());
 
         XContentBuilder xBuilder = XContentFactory.jsonBuilder();
-        xBuilder.startObject();
         response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS);
-        xBuilder.endObject();
         String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 7ed1b18953a..ba2f5de24ba 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -82,7 +82,7 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
             assertThat(actionGet.getIndex(), equalTo("test"));
             assertThat(actionGet.isExists(), equalTo(false));
             // check response is nevertheless serializable to json
-            actionGet.toXContent(jsonBuilder().startObject(), ToXContent.EMPTY_PARAMS);
+            actionGet.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
index c255ef6fc88..e034cff3f1d 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/TermVectorsUnitTests.java
@@ -36,12 +36,16 @@ import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.TermVectorsRequest.Flag;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.mapper.AllFieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -56,6 +60,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
@@ -236,6 +241,7 @@ public class TermVectorsUnitTests extends ESTestCase {
             request.parent(parent);
             String pref = random().nextBoolean() ? "somePreference" : null;
             request.preference(pref);
+            request.doc(new BytesArray("{}"), randomBoolean(), XContentType.JSON);
 
             // write
             ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
@@ -255,7 +261,37 @@ public class TermVectorsUnitTests extends ESTestCase {
             assertThat(request.termStatistics(), equalTo(req2.termStatistics()));
             assertThat(request.preference(), equalTo(pref));
             assertThat(request.routing(), equalTo(null));
+            assertEquals(new BytesArray("{}"), request.doc());
+            assertEquals(XContentType.JSON, request.xContentType());
+        }
+    }
 
+    public void testStreamRequestWithXContentBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AAABBWluZGV4BHR5cGUCaWQBAnt9AAABDnNvbWVQcmVmZXJlbmNlFgAAAAEA//////////0AAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            TermVectorsRequest request = new TermVectorsRequest();
+            request.readFrom(in);
+            assertEquals("index", request.index());
+            assertEquals("type", request.type());
+            assertEquals("id", request.id());
+            assertTrue(request.offsets());
+            assertFalse(request.fieldStatistics());
+            assertTrue(request.payloads());
+            assertFalse(request.positions());
+            assertTrue(request.termStatistics());
+            assertNull(request.parent());
+            assertEquals("somePreference", request.preference());
+            assertEquals("{}", request.doc().utf8ToString());
+            assertEquals(XContentType.JSON, request.xContentType());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                request.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
index 8c342265a28..e1133542e41 100644
--- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.shard.ShardId;
@@ -56,10 +57,10 @@ public class UpdateRequestTests extends ESTestCase {
     public void testUpdateRequest() throws Exception {
         UpdateRequest request = new UpdateRequest("test", "type", "1");
         // simple script
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .field("script", "script1")
-                .endObject().bytes()));
+                .endObject()));
         Script script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -69,9 +70,9 @@ public class UpdateRequestTests extends ESTestCase {
         assertThat(params, equalTo(Collections.emptyMap()));
 
         // simple verbose script
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder().startObject()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
                     .startObject("script").field("inline", "script1").endObject()
-                .endObject().bytes()));
+                .endObject()));
         script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -82,13 +83,13 @@ public class UpdateRequestTests extends ESTestCase {
 
         // script with params
         request = new UpdateRequest("test", "type", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder().startObject()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
             .startObject("script")
                 .field("inline", "script1")
                 .startObject("params")
                     .field("param1", "value1")
                 .endObject()
-            .endObject().endObject().bytes()));
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -100,7 +101,7 @@ public class UpdateRequestTests extends ESTestCase {
         assertThat(params.get("param1").toString(), equalTo("value1"));
 
         request = new UpdateRequest("test", "type", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .startObject("script")
                         .startObject("params")
@@ -108,7 +109,7 @@ public class UpdateRequestTests extends ESTestCase {
                         .endObject()
                         .field("inline", "script1")
                     .endObject()
-                .endObject().bytes()));
+                .endObject()));
         script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -121,7 +122,7 @@ public class UpdateRequestTests extends ESTestCase {
 
         // script with params and upsert
         request = new UpdateRequest("test", "type", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder().startObject()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
             .startObject("script")
                 .startObject("params")
                     .field("param1", "value1")
@@ -133,7 +134,7 @@ public class UpdateRequestTests extends ESTestCase {
                 .startObject("compound")
                     .field("field2", "value2")
                 .endObject()
-            .endObject().endObject().bytes()));
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -143,12 +144,13 @@ public class UpdateRequestTests extends ESTestCase {
         assertThat(params, notNullValue());
         assertThat(params.size(), equalTo(1));
         assertThat(params.get("param1").toString(), equalTo("value1"));
-        Map<String, Object> upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2();
+        Map<String, Object> upsertDoc =
+            XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2();
         assertThat(upsertDoc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2"));
 
         request = new UpdateRequest("test", "type", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder().startObject()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder().startObject()
             .startObject("upsert")
                 .field("field1", "value1")
                 .startObject("compound")
@@ -160,7 +162,7 @@ public class UpdateRequestTests extends ESTestCase {
                     .field("param1", "value1")
                 .endObject()
                 .field("inline", "script1")
-            .endObject().endObject().bytes()));
+            .endObject().endObject()));
         script = request.script();
         assertThat(script, notNullValue());
         assertThat(script.getIdOrCode(), equalTo("script1"));
@@ -170,13 +172,13 @@ public class UpdateRequestTests extends ESTestCase {
         assertThat(params, notNullValue());
         assertThat(params.size(), equalTo(1));
         assertThat(params.get("param1").toString(), equalTo("value1"));
-        upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true).v2();
+        upsertDoc = XContentHelper.convertToMap(request.upsertRequest().source(), true, request.upsertRequest().getContentType()).v2();
         assertThat(upsertDoc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) upsertDoc.get("compound")).get("field2").toString(), equalTo("value2"));
 
         // script with doc
         request = new UpdateRequest("test", "type", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .startObject("doc")
                         .field("field1", "value1")
@@ -184,7 +186,7 @@ public class UpdateRequestTests extends ESTestCase {
                             .field("field2", "value2")
                         .endObject()
                     .endObject()
-                .endObject().bytes()));
+                .endObject()));
         Map<String, Object> doc = request.doc().sourceAsMap();
         assertThat(doc.get("field1").toString(), equalTo("value1"));
         assertThat(((Map) doc.get("compound")).get("field2").toString(), equalTo("value2"));
@@ -192,54 +194,54 @@ public class UpdateRequestTests extends ESTestCase {
 
     // Related to issue 15338
     public void testFieldsParsing() throws Exception {
-        UpdateRequest request = new UpdateRequest("test", "type1", "1")
-                .fromXContent(XContentHelper.createParser(new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}")));
+        UpdateRequest request = new UpdateRequest("test", "type1", "1").fromXContent(
+                createParser(JsonXContent.jsonXContent, new BytesArray("{\"doc\": {\"field1\": \"value1\"}, \"fields\": \"_source\"}")));
         assertThat(request.doc().sourceAsMap().get("field1").toString(), equalTo("value1"));
         assertThat(request.fields(), arrayContaining("_source"));
 
-        request = new UpdateRequest("test", "type2", "2").fromXContent(
-                XContentHelper.createParser(new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}")));
+        request = new UpdateRequest("test", "type2", "2").fromXContent(createParser(JsonXContent.jsonXContent,
+                new BytesArray("{\"doc\": {\"field2\": \"value2\"}, \"fields\": [\"field1\", \"field2\"]}")));
         assertThat(request.doc().sourceAsMap().get("field2").toString(), equalTo("value2"));
         assertThat(request.fields(), arrayContaining("field1", "field2"));
     }
 
     public void testFetchSourceParsing() throws Exception {
         UpdateRequest request = new UpdateRequest("test", "type1", "1");
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .field("_source", true)
-                .endObject().bytes()));
+                .endObject()));
         assertThat(request.fetchSource(), notNullValue());
         assertThat(request.fetchSource().includes().length, equalTo(0));
         assertThat(request.fetchSource().excludes().length, equalTo(0));
         assertThat(request.fetchSource().fetchSource(), equalTo(true));
 
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .field("_source", false)
-                .endObject().bytes()));
+                .endObject()));
         assertThat(request.fetchSource(), notNullValue());
         assertThat(request.fetchSource().includes().length, equalTo(0));
         assertThat(request.fetchSource().excludes().length, equalTo(0));
         assertThat(request.fetchSource().fetchSource(), equalTo(false));
 
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .field("_source", "path.inner.*")
-                .endObject().bytes()));
+                .endObject()));
         assertThat(request.fetchSource(), notNullValue());
         assertThat(request.fetchSource().fetchSource(), equalTo(true));
         assertThat(request.fetchSource().includes().length, equalTo(1));
         assertThat(request.fetchSource().excludes().length, equalTo(0));
         assertThat(request.fetchSource().includes()[0], equalTo("path.inner.*"));
 
-        request.fromXContent(XContentHelper.createParser(XContentFactory.jsonBuilder()
+        request.fromXContent(createParser(XContentFactory.jsonBuilder()
                 .startObject()
                     .startObject("_source")
                         .field("includes", "path.inner.*")
                         .field("excludes", "another.inner.*")
                     .endObject()
-                .endObject().bytes()));
+                .endObject()));
         assertThat(request.fetchSource(), notNullValue());
         assertThat(request.fetchSource().fetchSource(), equalTo(true));
         assertThat(request.fetchSource().includes().length, equalTo(1));
@@ -291,7 +293,7 @@ public class UpdateRequestTests extends ESTestCase {
                 .upsert(indexRequest)
                 .script(new Script(ScriptType.INLINE, "mock", "ctx._source.update_timestamp = ctx._now", Collections.emptyMap()))
                 .scriptedUpsert(true);
-            long nowInMillis = randomPositiveLong();
+            long nowInMillis = randomNonNegativeLong();
             // We simulate that the document is not existing yet
             GetResult getResult = new GetResult("test", "type1", "2", 0, false, null, null);
             UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult, () -> nowInMillis);
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
new file mode 100644
index 00000000000..35d9eb8edf1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateResponseTests.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.update;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.index.IndexResponseTests;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.get.GetField;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.get.GetResultTests;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.action.DocWriteResponse.Result.DELETED;
+import static org.elasticsearch.action.DocWriteResponse.Result.NOT_FOUND;
+import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+
+public class UpdateResponseTests extends ESTestCase {
+
+    public void testToXContent() throws IOException {
+        {
+            UpdateResponse updateResponse = new UpdateResponse(new ShardId("index", "index_uuid", 0), "type", "id", 0, NOT_FOUND);
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":0,\"result\":\"not_found\"," +
+                    "\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0}}", output);
+        }
+        {
+            UpdateResponse updateResponse = new UpdateResponse(new ReplicationResponse.ShardInfo(10, 6),
+                    new ShardId("index", "index_uuid", 1), "type", "id", 3, 1, DELETED);
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"result\":\"deleted\"," +
+                    "\"_shards\":{\"total\":10,\"successful\":6,\"failed\":0},\"_seq_no\":3}", output);
+        }
+        {
+            BytesReference source = new BytesArray("{\"title\":\"Book title\",\"isbn\":\"ABC-123\"}");
+            Map<String, GetField> fields = new HashMap<>();
+            fields.put("title", new GetField("title", Collections.singletonList("Book title")));
+            fields.put("isbn", new GetField("isbn", Collections.singletonList("ABC-123")));
+
+            UpdateResponse updateResponse = new UpdateResponse(new ReplicationResponse.ShardInfo(3, 2),
+                    new ShardId("books", "books_uuid", 2), "book", "1", 7, 2, UPDATED);
+            updateResponse.setGetResult(new GetResult("books", "book", "1", 2, true, source, fields));
+
+            String output = Strings.toString(updateResponse);
+            assertEquals("{\"_index\":\"books\",\"_type\":\"book\",\"_id\":\"1\",\"_version\":2,\"result\":\"updated\"," +
+                    "\"_shards\":{\"total\":3,\"successful\":2,\"failed\":0},\"_seq_no\":7,\"get\":{\"found\":true," +
+                    "\"_source\":{\"title\":\"Book title\",\"isbn\":\"ABC-123\"},\"fields\":{\"isbn\":[\"ABC-123\"],\"title\":[\"Book " +
+                    "title\"]}}}", output);
+        }
+    }
+
+    public void testToAndFromXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final Tuple<UpdateResponse, UpdateResponse> tuple = randomUpdateResponse(xContentType);
+        boolean humanReadable = randomBoolean();
+
+        // Parse the XContent bytes to obtain a parsed UpdateResponse
+        UpdateResponse parsedUpdateResponse;
+        try (XContentParser parser = createParser(xContentType.xContent(), toXContent(tuple.v1(), xContentType, humanReadable))) {
+            parsedUpdateResponse = UpdateResponse.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        final UpdateResponse expectedUpdateResponse = tuple.v2();
+        try (XContentParser parser = createParser(xContentType.xContent(), toXContent(parsedUpdateResponse, xContentType, humanReadable))) {
+            IndexResponseTests.assertDocWriteResponse(expectedUpdateResponse, parser.map());
+        }
+        assertEquals(expectedUpdateResponse.getGetResult(), parsedUpdateResponse.getGetResult());
+    }
+
+    private static Tuple<UpdateResponse, UpdateResponse> randomUpdateResponse(XContentType xContentType) {
+        Tuple<GetResult, GetResult> getResults = GetResultTests.randomGetResult(xContentType);
+        GetResult actualGetResult = getResults.v1();
+        GetResult expectedGetResult = getResults.v2();
+
+        ShardId shardId = new ShardId(actualGetResult.getIndex(), randomAsciiOfLength(5), randomIntBetween(0, 5));
+        String type = actualGetResult.getType();
+        String id = actualGetResult.getId();
+        long version = actualGetResult.getVersion();
+        DocWriteResponse.Result result = actualGetResult.isExists() ? DocWriteResponse.Result.UPDATED : DocWriteResponse.Result.NOT_FOUND;
+
+        // We also want small number values (randomNonNegativeLong() tend to generate high numbers)
+        // in order to catch some conversion error that happen between int/long after parsing.
+        Long seqNo = randomFrom(randomNonNegativeLong(), (long) randomIntBetween(0, 10_000), null);
+
+        UpdateResponse actual, expected;
+        if (seqNo != null) {
+            ReplicationResponse.ShardInfo shardInfo = RandomObjects.randomShardInfo(random(), true);
+            actual = new UpdateResponse(shardInfo, shardId, type, id, seqNo, version, result);
+            expected = new UpdateResponse(shardInfo, shardId, type, id, seqNo, version, result);
+
+        } else {
+            actual = new UpdateResponse(shardId, type, id, version, result);
+            expected = new UpdateResponse(shardId, type, id, version, result);
+        }
+
+        if (actualGetResult.isExists()) {
+            actual.setGetResult(actualGetResult);
+        }
+
+        if (expectedGetResult.isExists()) {
+            expected.setGetResult(new GetResult(shardId.getIndexName(), type, id, version,
+                    expectedGetResult.isExists(), expectedGetResult.internalSourceRef(), expectedGetResult.getFields()));
+        }
+
+        boolean forcedRefresh = randomBoolean();
+        actual.setForcedRefresh(forcedRefresh);
+        expected.setForcedRefresh(forcedRefresh);
+
+        return Tuple.tuple(actual, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 9f04f74cd24..497b134ebe3 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
@@ -90,7 +91,8 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test", "alias1"));
 
         logger.info("--> indexing against [alias1], should work now");
-        IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet();
+        IndexResponse indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getIndex(), equalTo("test"));
 
         logger.info("--> creating index [test_x]");
@@ -102,7 +104,8 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().removeAlias("test", "alias1").addAlias("test_x", "alias1"));
 
         logger.info("--> indexing against [alias1], should work against [test_x]");
-        indexResponse = client().index(indexRequest("alias1").type("type1").id("1").source(source("1", "test"))).actionGet();
+        indexResponse = client().index(indexRequest("alias1").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getIndex(), equalTo("test_x"));
     }
 
@@ -164,15 +167,14 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test")));
 
         logger.info("--> indexing against [test]");
-        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
-        client().index(
-                indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
-                .actionGet();
+        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
+        client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)
+                    .setRefreshPolicy(RefreshPolicy.IMMEDIATE)).actionGet();
 
         logger.info("--> checking single filtering alias search");
         SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get();
@@ -254,16 +256,16 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test2", "foos", termQuery("name", "foo")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get();
-        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get();
-        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get();
-        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get();
+        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get();
 
         logger.info("--> indexing against [test2]");
-        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get();
-        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get();
-        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get();
-        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get();
+        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get();
 
         refresh();
 
@@ -322,17 +324,17 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test3", "filter13", termQuery("name", "baz")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"))).get();
-        client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"))).get();
-        client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"))).get();
+        client().index(indexRequest("test1").type("type1").id("11").source(source("11", "foo test1"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("12").source(source("12", "bar test1"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("13").source(source("13", "baz test1"), XContentType.JSON)).get();
 
-        client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"))).get();
-        client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"))).get();
-        client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"))).get();
+        client().index(indexRequest("test2").type("type1").id("21").source(source("21", "foo test2"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("22").source(source("22", "bar test2"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("23").source(source("23", "baz test2"), XContentType.JSON)).get();
 
-        client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"))).get();
-        client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"))).get();
-        client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"))).get();
+        client().index(indexRequest("test3").type("type1").id("31").source(source("31", "foo test3"), XContentType.JSON)).get();
+        client().index(indexRequest("test3").type("type1").id("32").source(source("32", "bar test3"), XContentType.JSON)).get();
+        client().index(indexRequest("test3").type("type1").id("33").source(source("33", "baz test3"), XContentType.JSON)).get();
 
         refresh();
 
@@ -382,16 +384,16 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test2", "tests", termQuery("name", "test")));
 
         logger.info("--> indexing against [test1]");
-        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"))).get();
-        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"))).get();
-        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"))).get();
-        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"))).get();
+        client().index(indexRequest("test1").type("type1").id("1").source(source("1", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("2").source(source("2", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("3").source(source("3", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test1").type("type1").id("4").source(source("4", "something else"), XContentType.JSON)).get();
 
         logger.info("--> indexing against [test2]");
-        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"))).get();
-        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"))).get();
-        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"))).get();
-        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"))).get();
+        client().index(indexRequest("test2").type("type1").id("5").source(source("5", "foo test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("6").source(source("6", "bar test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("7").source(source("7", "baz test"), XContentType.JSON)).get();
+        client().index(indexRequest("test2").type("type1").id("8").source(source("8", "something else"), XContentType.JSON)).get();
 
         refresh();
 
@@ -435,7 +437,7 @@ public class IndexAliasesIT extends ESIntegTestCase {
 
         for (int i = 0; i < 10; i++) {
             assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i));
-            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get();
+            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get();
         }
     }
 
@@ -447,7 +449,7 @@ public class IndexAliasesIT extends ESIntegTestCase {
 
         for (int i = 0; i < 10; i++) {
             assertAcked(admin().indices().prepareAliases().addAlias("test", "alias" + i));
-            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"))).get();
+            client().index(indexRequest("alias" + i).type("type1").id("1").source(source("1", "test"), XContentType.JSON)).get();
         }
     }
 
@@ -466,7 +468,8 @@ public class IndexAliasesIT extends ESIntegTestCase {
                 @Override
                 public void run() {
                     assertAcked(admin().indices().prepareAliases().addAlias("test", aliasName));
-                    client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"))).actionGet();
+                    client().index(indexRequest(aliasName).type("type1").id("1").source(source("1", "test"), XContentType.JSON))
+                        .actionGet();
                 }
             });
         }
@@ -745,7 +748,7 @@ public class IndexAliasesIT extends ESIntegTestCase {
                 "        \"alias2\" : {\"filter\" : {\"match_all\": {}}},\n" +
                 "        \"alias3\" : { \"index_routing\" : \"index\", \"search_routing\" : \"search\"}\n" +
                 "    }\n" +
-                "}"));
+                "}", XContentType.JSON));
 
         checkAliases();
     }
@@ -826,8 +829,8 @@ public class IndexAliasesIT extends ESIntegTestCase {
                         .addMapping("parent")
                         .addMapping("child", "_parent", "type=parent")
         );
-        client().prepareIndex("my-index", "parent", "1").setSource("{}").get();
-        client().prepareIndex("my-index", "child", "2").setSource("{}").setParent("1").get();
+        client().prepareIndex("my-index", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("my-index", "child", "2").setSource("{}", XContentType.JSON).setParent("1").get();
         refresh();
 
         assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", hasChildQuery("child", matchAllQuery(), ScoreMode.None)));
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index bc810254920..881d03b3dc6 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -246,7 +246,7 @@ public class BootstrapCheckTests extends ESTestCase {
             private final boolean isMemoryLocked;
             private final boolean shouldFail;
 
-            public MlockallCheckTestCase(final boolean mlockallSet, final boolean isMemoryLocked, final boolean shouldFail) {
+            MlockallCheckTestCase(final boolean mlockallSet, final boolean isMemoryLocked, final boolean shouldFail) {
                 this.mlockallSet = mlockallSet;
                 this.isMemoryLocked = isMemoryLocked;
                 this.shouldFail = shouldFail;
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
index 9a1417bdfa6..8a00a430dbc 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.nio.file.Path;
+import java.util.Map;
 import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -151,10 +152,10 @@ public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 ExitCodes.OK,
                 true,
                 output -> {},
-                (foreground, pidFile, quiet, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(2));
-                    assertThat(esSettings, hasEntry("foo", "bar"));
-                    assertThat(esSettings, hasEntry("baz", "qux"));
+                (foreground, pidFile, quiet, env) -> {
+                    Map<String, String> settings = env.settings().getAsMap();
+                    assertThat(settings, hasEntry("foo", "bar"));
+                    assertThat(settings, hasEntry("baz", "qux"));
                 },
                 "-Efoo=bar", "-E", "baz=qux"
         );
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
index 6657a307908..76e9a4eb105 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
@@ -35,6 +35,7 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.RecoverySource;
@@ -388,7 +389,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
         SearchHit hit = searchReq.get().getHits().getAt(0);
         String docId = hit.getId();
         // foo is new, it is not a field in the generated index
-        client().prepareUpdate(indexName, "doc", docId).setDoc("foo", "bar").get();
+        client().prepareUpdate(indexName, "doc", docId).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "bar").get();
         GetResponse getRsp = client().prepareGet(indexName, "doc", docId).get();
         Map<String, Object> source = getRsp.getSourceAsMap();
         assertThat(source, Matchers.hasKey("foo"));
@@ -544,7 +545,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
             String indexName = indexFile.replace(".zip", "").toLowerCase(Locale.ROOT).replace("unsupported-", "index-");
             Path nodeDir = getNodeDir(indexFile);
             logger.info("Parsing cluster state files from index [{}]", indexName);
-            final MetaData metaData = globalFormat.loadLatestState(logger, nodeDir);
+            final MetaData metaData = globalFormat.loadLatestState(logger, xContentRegistry(), nodeDir);
             assertNotNull(metaData);
 
             final Version version = Version.fromString(indexName.substring("index-".length()));
@@ -555,7 +556,7 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
                 dataDir = nodeDir.getParent();
             }
             final Path indexDir = getIndexDir(logger, indexName, indexFile, dataDir);
-            assertNotNull(indexFormat.loadLatestState(logger, indexDir));
+            assertNotNull(indexFormat.loadLatestState(logger, xContentRegistry(), indexDir));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
index ec8c12cb525..02ba7f5ea4f 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
@@ -154,7 +154,7 @@ public class RepositoryUpgradabilityIT extends AbstractSnapshotIntegTestCase {
         final String prefix = "repo";
         final List<String> repoVersions = new ArrayList<>();
         final Path repoFiles = getBwcIndicesPath();
-        try (final DirectoryStream<Path> dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) {
+        try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) {
             for (final Path entry : dirStream) {
                 final String fileName = entry.getFileName().toString();
                 String version = fileName.substring(prefix.length() + 1);
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
index 6d13d3cf418..9dd9e9dd4a6 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
@@ -26,21 +26,22 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.common.io.FileTestUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.repositories.uri.URLRepository;
+import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.VersionUtils;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -52,6 +53,7 @@ import java.util.SortedSet;
 import java.util.TreeSet;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -60,27 +62,19 @@ import static org.hamcrest.Matchers.notNullValue;
 @ClusterScope(scope = Scope.TEST)
 public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
 
+    private static Path repoPath;
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        if (randomBoolean()) {
-            // Configure using path.repo
-            return Settings.builder()
-                    .put(super.nodeSettings(nodeOrdinal))
-                    .put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath())
-                    .build();
-        } else {
-            // Configure using url white list
-            try {
-                URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/");
-                return Settings.builder()
-                        .put(super.nodeSettings(nodeOrdinal))
-                        .putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString())
-                        .build();
-            } catch (URISyntaxException ex) {
-                throw new IllegalArgumentException(ex);
-            }
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(Environment.PATH_REPO_SETTING.getKey(), repoPath)
+            .build();
+    }
 
-        }
+    @BeforeClass
+    public static void repoSetup() throws IOException {
+        repoPath = createTempDir("repositories");
     }
 
     public void testRestoreOldSnapshots() throws Exception {
@@ -150,13 +144,14 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
     }
 
     private void createRepo(String prefix, String version, String repo) throws Exception {
-        Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
-        URI repoFileUri = repoFile.toUri();
-        URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
+        Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
+        String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0];
+        Path fsRepoPath = repoPath.resolve(repoFileName);
+        FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null);
         logger.info("-->  creating repository [{}] for version [{}]", repo, version);
         assertAcked(client().admin().cluster().preparePutRepository(repo)
-                .setType("url").setSettings(Settings.builder()
-                        .put("url", repoJarUri.toString())));
+            .setType(MockRepository.TYPE).setSettings(Settings.builder()
+                .put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString())));
     }
 
     private void testOldSnapshot(String version, String repo, String snapshot) throws IOException {
@@ -189,7 +184,15 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
         assertThat(template.patterns(), equalTo(Collections.singletonList("te*")));
         assertThat(template.settings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), equalTo(1));
         assertThat(template.mappings().size(), equalTo(1));
-        assertThat(template.mappings().get("type1").string(), equalTo("{\"type1\":{\"_source\":{\"enabled\":false}}}"));
+        assertThat(template.mappings().get("type1").string(),
+            anyOf(
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":false}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"false\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"0\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"),
+                equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}")
+            ));
         assertThat(template.aliases().size(), equalTo(3));
         assertThat(template.aliases().get("alias1"), notNullValue());
         assertThat(template.aliases().get("alias2").filter().string(), containsString(version));
diff --git a/core/src/test/java/org/elasticsearch/cli/TerminalTests.java b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
index 6673bdbc858..795780b4890 100644
--- a/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/TerminalTests.java
@@ -46,6 +46,33 @@ public class TerminalTests extends ESTestCase {
         assertPrinted(terminal, Terminal.Verbosity.NORMAL, "This message contains percent like %20n");
     }
 
+    public void testPromptYesNoDefault() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("");
+        assertTrue(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("");
+        assertFalse(terminal.promptYesNo("Answer?", false));
+    }
+
+    public void testPromptYesNoReprompt() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("blah");
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer? [Y/n]\nDid not understand answer 'blah'\nAnswer? [Y/n]", true));
+    }
+
+    public void testPromptYesNoCase() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.addTextInput("Y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("y");
+        assertTrue(terminal.promptYesNo("Answer?", false));
+        terminal.addTextInput("N");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+        terminal.addTextInput("n");
+        assertFalse(terminal.promptYesNo("Answer?", true));
+    }
+
     private void assertPrinted(MockTerminal logTerminal, Terminal.Verbosity verbosity, String text) throws Exception {
         logTerminal.println(verbosity, text);
         String output = logTerminal.getOutput();
diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
index ccc72db9d7a..d3a0e0b3e4e 100644
--- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
+++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
@@ -36,6 +36,7 @@ import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAc
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -106,7 +107,7 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase {
         client.prepareSearch().execute().addListener(new AssertingActionListener<>(SearchAction.NAME, client.threadPool()));
         client.prepareDelete("idx", "type", "id").execute().addListener(new AssertingActionListener<>(DeleteAction.NAME, client.threadPool()));
         client.admin().cluster().prepareDeleteStoredScript("lang", "id").execute().addListener(new AssertingActionListener<>(DeleteStoredScriptAction.NAME, client.threadPool()));
-        client.prepareIndex("idx", "type", "id").setSource("source").execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool()));
+        client.prepareIndex("idx", "type", "id").setSource("source", XContentType.JSON).execute().addListener(new AssertingActionListener<>(IndexAction.NAME, client.threadPool()));
 
         // choosing arbitrary cluster admin actions to test
         client.admin().cluster().prepareClusterStats().execute().addListener(new AssertingActionListener<>(ClusterStatsAction.NAME, client.threadPool()));
diff --git a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
index fff3b3cc3af..160c14c243c 100644
--- a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
@@ -43,7 +43,7 @@ public class NodeClientHeadersTests extends AbstractClientHeadersTestCase {
         Settings settings = HEADER_SETTINGS;
         Actions actions = new Actions(settings, threadPool, testedActions);
         NodeClient client = new NodeClient(settings, threadPool);
-        client.initialize(actions);
+        client.initialize(actions, () -> "test");
         return client;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index d5ae66c6bd6..c7c98aaa99d 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -113,8 +114,8 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
                 fail("takes way too long to get the cluster state");
             }
 
-            assertThat(client.connectedNodes().size(), is(1));
-            assertThat(client.connectedNodes().get(0).getAddress(), is(transportService.boundAddress().publishAddress()));
+            assertEquals(1, client.connectedNodes().size());
+            assertEquals(client.connectedNodes().get(0).getAddress(), transportService.boundAddress().publishAddress());
         }
     }
 
@@ -128,12 +129,14 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
             private InternalTransportServiceInterceptor instance = new InternalTransportServiceInterceptor();
 
             @Override
-            public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+            public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                       ThreadContext threadContext) {
                 return Collections.singletonList(new TransportInterceptor() {
                     @Override
                     public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                                boolean forceExecution,
                                                                                                 TransportRequestHandler<T> actualHandler) {
-                        return instance.interceptHandler(action, executor, actualHandler);
+                        return instance.interceptHandler(action, executor, forceExecution, actualHandler);
                     }
 
                     @Override
@@ -165,7 +168,8 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
                         ClusterName cluster1 = new ClusterName("cluster1");
                         ClusterState.Builder builder = ClusterState.builder(cluster1);
                         //the sniffer detects only data nodes
-                        builder.nodes(DiscoveryNodes.builder().add(new DiscoveryNode("node_id", address, Collections.emptyMap(),
+                        builder.nodes(DiscoveryNodes.builder().add(new DiscoveryNode("node_id", "someId", "some_ephemeralId_id",
+                            address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(),
                                 Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT)));
                         ((TransportResponseHandler<ClusterStateResponse>) handler)
                                 .handleResponse(new ClusterStateResponse(cluster1, builder.build()));
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index eaaafe1e1ec..a6e2431a15f 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
@@ -143,6 +144,9 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                         }
                     };
                 }
+            }, (addr) -> {
+                assert addr == null : "boundAddress: " + addr;
+                return DiscoveryNode.createLocal(settings, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID());
             }, null);
             transportService.start();
             transportService.acceptIncomingRequests();
@@ -199,7 +203,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
     public void testListenerFailures() throws InterruptedException {
         int iters = iterations(10, 100);
         for (int i = 0; i <iters; i++) {
-            try(final TestIteration iteration = new TestIteration()) {
+            try(TestIteration iteration = new TestIteration()) {
                 iteration.transport.endConnectMode(); // stop transport from responding early
                 final CountDownLatch latch = new CountDownLatch(1);
                 final AtomicInteger finalFailures = new AtomicInteger();
@@ -281,7 +285,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
     public void testConnectedNodes() {
         int iters = iterations(10, 100);
         for (int i = 0; i <iters; i++) {
-            try(final TestIteration iteration = new TestIteration()) {
+            try(TestIteration iteration = new TestIteration()) {
                 assertThat(iteration.transportClientNodesService.connectedNodes().size(), lessThanOrEqualTo(iteration.listNodesCount));
                 for (DiscoveryNode discoveryNode : iteration.transportClientNodesService.connectedNodes()) {
                     assertThat(discoveryNode.getHostName(), startsWith("liveness-"));
@@ -303,7 +307,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
 
     private void checkRemoveAddress(boolean sniff) {
         Object[] extraSettings = {TransportClient.CLIENT_TRANSPORT_SNIFF.getKey(), sniff};
-        try(final TestIteration iteration = new TestIteration(extraSettings)) {
+        try(TestIteration iteration = new TestIteration(extraSettings)) {
             final TransportClientNodesService service = iteration.transportClientNodesService;
             assertEquals(iteration.listNodesCount + iteration.sniffNodesCount, service.connectedNodes().size());
             final TransportAddress addressToRemove = randomFrom(iteration.listNodeAddresses);
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 63d34f683de..240add84464 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -244,7 +244,7 @@ public class ClusterChangedEventTests extends ESTestCase {
         event = new ClusterChangedEvent("_na_", originalState, nextState);
         Set<String> changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 1);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
 
         // next state has same custom metadata
         nextState = nextState(originalState, Collections.singletonList(customMetaData1));
@@ -263,14 +263,14 @@ public class ClusterChangedEventTests extends ESTestCase {
         event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
         changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 1);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
 
         // next state updates custom metadata
         nextState = nextState(stateWithCustomMetaData, Collections.singletonList(new CustomMetaData1("data1")));
         event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
         changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 1);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
 
         // next state adds new custom metadata type
         CustomMetaData2 customMetaData2 = new CustomMetaData2("data2");
@@ -278,15 +278,15 @@ public class ClusterChangedEventTests extends ESTestCase {
         event = new ClusterChangedEvent("_na_", stateWithCustomMetaData, nextState);
         changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 1);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
 
         // next state adds two custom metadata type
         nextState = nextState(originalState, Arrays.asList(customMetaData1, customMetaData2));
         event = new ClusterChangedEvent("_na_", originalState, nextState);
         changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 2);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.type()));
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
 
         // next state removes two custom metadata type
         nextState = originalState;
@@ -294,25 +294,17 @@ public class ClusterChangedEventTests extends ESTestCase {
                 nextState(originalState, Arrays.asList(customMetaData1, customMetaData2)), nextState);
         changedCustomMetaDataTypeSet = event.changedCustomMetaDataSet();
         assertTrue(changedCustomMetaDataTypeSet.size() == 2);
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.type()));
-        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.type()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData2.getWriteableName()));
+        assertTrue(changedCustomMetaDataTypeSet.contains(customMetaData1.getWriteableName()));
     }
 
     private static class CustomMetaData2 extends TestCustomMetaData {
-        static {
-            MetaData.registerPrototype("2", new CustomMetaData2(""));
-        }
         protected CustomMetaData2(String data) {
             super(data);
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData2(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return "2";
         }
 
@@ -323,20 +315,12 @@ public class ClusterChangedEventTests extends ESTestCase {
     }
 
     private static class CustomMetaData1 extends TestCustomMetaData {
-        static {
-            MetaData.registerPrototype("1", new CustomMetaData1(""));
-        }
         protected CustomMetaData1(String data) {
             super(data);
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData1(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return "1";
         }
 
@@ -378,7 +362,7 @@ public class ClusterChangedEventTests extends ESTestCase {
             }
         }
         for (TestCustomMetaData testCustomMetaData : customMetaDataList) {
-            metaDataBuilder.putCustom(testCustomMetaData.type(), testCustomMetaData);
+            metaDataBuilder.putCustom(testCustomMetaData.getWriteableName(), testCustomMetaData);
         }
         builder.metaData(metaDataBuilder);
         return builder.build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 637b6733e22..b5e7a1201f9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -172,7 +172,6 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
             IndexShard indexShard = indexService.getShardOrNull(shard.id());
             assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath);
         }
-
     }
 
     public void testClusterInfoServiceInformationClearOnError() throws InterruptedException, ExecutionException {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 3811e37389c..c4241a2a989 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.cluster;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
@@ -39,6 +41,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -51,7 +54,6 @@ import org.elasticsearch.plugins.ClusterPlugin;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -59,7 +61,8 @@ import java.util.function.Supplier;
 
 public class ClusterModuleTests extends ModuleTestCase {
     private ClusterService clusterService = new ClusterService(Settings.EMPTY,
-        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
+        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, () ->
+        new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), Version.CURRENT));
     static class FakeAllocationDecider extends AllocationDecider {
         protected FakeAllocationDecider(Settings settings) {
             super(settings);
@@ -71,10 +74,9 @@ public class ClusterModuleTests extends ModuleTestCase {
         public void allocate(RoutingAllocation allocation) {
             // noop
         }
-
         @Override
-        public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-            return new HashMap<>();
+        public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+            throw new UnsupportedOperationException("explain API not supported on FakeShardsAllocator");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index 9d44dbbca38..722f9590b56 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -31,6 +31,8 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -75,13 +77,15 @@ import static org.hamcrest.Matchers.is;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0)
 public class ClusterStateDiffIT extends ESIntegTestCase {
     public void testClusterStateDiffSerialization() throws Exception {
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         DiscoveryNode masterNode = new DiscoveryNode("master", buildNewFakeTransportAddress(),
                 emptyMap(), emptySet(), Version.CURRENT);
         DiscoveryNode otherNode = new DiscoveryNode("other", buildNewFakeTransportAddress(),
                 emptyMap(), emptySet(), Version.CURRENT);
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(masterNode).add(otherNode).localNodeId(masterNode.getId()).build();
         ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(discoveryNodes).build();
-        ClusterState clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), otherNode);
+        ClusterState clusterStateFromDiffs =
+            ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), otherNode, namedWriteableRegistry);
 
         int iterationCount = randomIntBetween(10, 300);
         for (int iteration = 0; iteration < iterationCount; iteration++) {
@@ -117,7 +121,8 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
 
             if (randomIntBetween(0, 10) < 1) {
                 // Update cluster state via full serialization from time to time
-                clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), previousClusterStateFromDiffs.nodes().getLocalNode());
+                clusterStateFromDiffs = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState),
+                    previousClusterStateFromDiffs.nodes().getLocalNode(), namedWriteableRegistry);
             } else {
                 // Update cluster states using diffs
                 Diff<ClusterState> diffBeforeSerialization = clusterState.diff(previousClusterState);
@@ -126,7 +131,8 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                 byte[] diffBytes = BytesReference.toBytes(os.bytes());
                 Diff<ClusterState> diff;
                 try (StreamInput input = StreamInput.wrap(diffBytes)) {
-                    diff = previousClusterStateFromDiffs.readDiffFrom(input);
+                    StreamInput namedInput = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
+                    diff = ClusterState.readDiffFrom(namedInput, previousClusterStateFromDiffs.nodes().getLocalNode());
                     clusterStateFromDiffs = diff.apply(previousClusterStateFromDiffs);
                 }
             }
@@ -618,7 +624,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
 
             @Override
             public MetaData.Builder put(MetaData.Builder builder, MetaData.Custom part) {
-                return builder.putCustom(part.type(), part);
+                return builder.putCustom(part.getWriteableName(), part);
             }
 
             @Override
@@ -660,7 +666,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
 
             @Override
             public ClusterState.Builder put(ClusterState.Builder builder, ClusterState.Custom part) {
-                return builder.putCustom(part.type(), part);
+                return builder.putCustom(part.getWriteableName(), part);
             }
 
             @Override
@@ -679,6 +685,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                                 SnapshotsInProgress.State.fromValue((byte) randomIntBetween(0, 6)),
                                 Collections.emptyList(),
                                 Math.abs(randomLong()),
+                                (long) randomIntBetween(0, 1000),
                                 ImmutableOpenMap.of()));
                     case 1:
                         return new RestoreInProgress(new RestoreInProgress.Entry(
diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
index ee6d719f55b..d63203eda25 100644
--- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
@@ -375,7 +375,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
         otherNodes.remove(master);
         NetworkDisruption partition = new NetworkDisruption(
             new TwoPartitions(Collections.singleton(master), otherNodes),
-            new NetworkDelay(TimeValue.timeValueMinutes(1)));
+            new NetworkDisruption.NetworkDisconnect());
         internalCluster().setDisruptionScheme(partition);
 
         final CountDownLatch latch = new CountDownLatch(1);
diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
index 0be4f7c4e51..c799969a3c8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -42,6 +43,7 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.util.Collections;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
@@ -59,13 +61,9 @@ public class NoMasterNodeIT extends ESIntegTestCase {
     }
 
     public void testNoMasterActions() throws Exception {
-        // note, sometimes, we want to check with the fact that an index gets created, sometimes not...
-        boolean autoCreateIndex = randomBoolean();
-        logger.info("auto_create_index set to {}", autoCreateIndex);
-
         Settings settings = Settings.builder()
                 .put("discovery.type", "zen")
-                .put("action.auto_create_index", autoCreateIndex)
+                .put("action.auto_create_index", true)
                 .put("discovery.zen.minimum_master_nodes", 2)
                 .put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms")
                 .put("discovery.initial_state_timeout", "500ms")
@@ -121,14 +119,12 @@ public class NoMasterNodeIT extends ESIntegTestCase {
                 ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-        checkWriteAction(
-                false, timeout,
+        checkUpdateAction(false, timeout,
                 client().prepareUpdate("test", "type1", "1")
                         .setScript(new Script(
                             ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
 
-        checkWriteAction(
-                autoCreateIndex, timeout,
+        checkUpdateAction(true, timeout,
                 client().prepareUpdate("no_index", "type1", "1")
                         .setScript(new Script(
                             ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
@@ -137,24 +133,28 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         checkWriteAction(false, timeout,
                 client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
-        checkWriteAction(autoCreateIndex, timeout,
+        checkWriteAction(true, timeout,
                 client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.add(client().prepareIndex("test", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-        checkBulkAction(false, bulkRequestBuilder);
+        // the request should fail very quickly - use a large timeout and make sure it didn't pass...
+        timeout = new TimeValue(5000);
+        bulkRequestBuilder.setTimeout(timeout);
+        checkWriteAction(false, timeout, bulkRequestBuilder);
 
         bulkRequestBuilder = client().prepareBulk();
         bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
         bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
-        checkBulkAction(autoCreateIndex, bulkRequestBuilder);
+        bulkRequestBuilder.setTimeout(timeout);
+        checkWriteAction(true, timeValueSeconds(5), bulkRequestBuilder);
 
         internalCluster().startNode(settings);
         client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet();
     }
 
-    void checkWriteAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?, ?> builder) {
+    void checkUpdateAction(boolean autoCreateIndex, TimeValue timeout, ActionRequestBuilder<?, ?, ?> builder) {
         // we clean the metadata when loosing a master, therefore all operations on indices will auto create it, if allowed
         long now = System.currentTimeMillis();
         try {
@@ -172,18 +172,7 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         }
     }
 
-    void checkBulkAction(boolean indexShouldBeAutoCreated, BulkRequestBuilder builder) {
-        // bulk operation do not throw MasterNotDiscoveredException exceptions. The only test that auto create kicked in and failed is
-        // via the timeout, as bulk operation do not wait on blocks.
-        TimeValue timeout;
-        if (indexShouldBeAutoCreated) {
-            // we expect the bulk to fail because it will try to go to the master. Use small timeout and detect it has passed
-            timeout = new TimeValue(200);
-        } else {
-            // the request should fail very quickly - use a large timeout and make sure it didn't pass...
-            timeout = new TimeValue(5000);
-        }
-        builder.setTimeout(timeout);
+    void checkWriteAction(boolean indexShouldBeAutoCreated, TimeValue timeout, ActionRequestBuilder<?, ?, ?> builder) {
         long now = System.currentTimeMillis();
         try {
             builder.get();
@@ -195,7 +184,7 @@ public class NoMasterNodeIT extends ESIntegTestCase {
                 assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
             } else {
                 // timeout is 5000
-                assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() - 50));
+                assertThat(System.currentTimeMillis() - now, lessThan(timeout.millis() + 50));
             }
         }
     }
@@ -248,7 +237,7 @@ public class NoMasterNodeIT extends ESIntegTestCase {
         TimeValue timeout = TimeValue.timeValueMillis(200);
         long now = System.currentTimeMillis();
         try {
-            client().prepareUpdate("test1", "type1", "1").setDoc("field", "value2").setTimeout(timeout).get();
+            client().prepareUpdate("test1", "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").setTimeout(timeout).get();
             fail("Expected ClusterBlockException");
         } catch (ClusterBlockException e) {
             assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 2b8333700a3..a7cf75e6eb6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
@@ -87,19 +88,19 @@ public class NodeConnectionsServiceTests extends ESTestCase {
         ClusterState current = clusterStateFromNodes(Collections.emptyList());
         ClusterChangedEvent event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
 
-        service.connectToNodes(event.nodesDelta().addedNodes());
-        assertConnected(event.nodesDelta().addedNodes());
+        service.connectToNodes(event.state().nodes());
+        assertConnected(event.state().nodes());
 
-        service.disconnectFromNodes(event.nodesDelta().removedNodes());
+        service.disconnectFromNodesExcept(event.state().nodes());
         assertConnectedExactlyToNodes(event.state());
 
         current = event.state();
         event = new ClusterChangedEvent("test", clusterStateFromNodes(randomSubsetOf(nodes)), current);
 
-        service.connectToNodes(event.nodesDelta().addedNodes());
-        assertConnected(event.nodesDelta().addedNodes());
+        service.connectToNodes(event.state().nodes());
+        assertConnected(event.state().nodes());
 
-        service.disconnectFromNodes(event.nodesDelta().removedNodes());
+        service.disconnectFromNodesExcept(event.state().nodes());
         assertConnectedExactlyToNodes(event.state());
     }
 
@@ -151,7 +152,8 @@ public class NodeConnectionsServiceTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         this.transport = new MockTransport();
-        transportService = new TransportService(Settings.EMPTY, transport, THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+        transportService = new TransportService(Settings.EMPTY, transport, THREAD_POOL, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), UUIDs.randomBase64UUID()), null);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
index 1944ed6e3bc..89d33043e57 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java
@@ -96,18 +96,37 @@ public class SimpleClusterStateIT extends ESIntegTestCase {
         client().admin().indices().preparePutTemplate("foo_template")
                 .setPatterns(Collections.singletonList("te*"))
                 .setOrder(0)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field1").field("type", "text").field("store", true).endObject()
-                        .startObject("field2").field("type", "keyword").field("store", true).endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "keyword")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .get();
 
         client().admin().indices().preparePutTemplate("fuu_template")
                 .setPatterns(Collections.singletonList("test*"))
                 .setOrder(1)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field2").field("type", "text").field("store", "no").endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                            .startObject("type1")
+                                .startObject("properties")
+                                    .startObject("field2")
+                                        .field("type", "text")
+                                        .field("store", false)
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject())
                 .get();
 
         ClusterStateResponse clusterStateResponseUnfiltered = client().admin().cluster().prepareState().get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
index a0a00256817..c2f6c3b64fa 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SimpleDataNodesIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -40,7 +41,8 @@ public class SimpleDataNodesIT extends ESIntegTestCase {
         internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build());
         client().admin().indices().create(createIndexRequest("test").waitForActiveShards(ActiveShardCount.NONE)).actionGet();
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet();
+            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+                .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
             // all is well
@@ -51,7 +53,8 @@ public class SimpleDataNodesIT extends ESIntegTestCase {
 
         // still no shard should be allocated
         try {
-            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")).timeout(timeValueSeconds(1))).actionGet();
+            client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)
+                .timeout(timeValueSeconds(1))).actionGet();
             fail("no allocation should happen");
         } catch (UnavailableShardsException e) {
             // all is well
@@ -61,7 +64,8 @@ public class SimpleDataNodesIT extends ESIntegTestCase {
         internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).build());
         assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("3").setLocal(true).execute().actionGet().isTimedOut(), equalTo(false));
 
-        IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet();
+        IndexResponse indexResponse = client().index(Requests.indexRequest("test").type("type1").id("1")
+            .source(source("1", "test"), XContentType.JSON)).actionGet();
         assertThat(indexResponse.getId(), equalTo("1"));
         assertThat(indexResponse.getType(), equalTo("type1"));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index 00edd536258..91420fa227a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -88,7 +88,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
 
     public void testEmptyTaskListProducesSameClusterState() throws Exception {
         List<ShardStateAction.ShardEntry> tasks = Collections.emptyList();
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result =
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result =
             executor.execute(clusterState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, false);
     }
@@ -97,7 +97,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
         String reason = "test duplicate failures are okay";
         ClusterState currentState = createClusterStateWithStartedShards(reason);
         List<ShardStateAction.ShardEntry> tasks = createExistingShards(currentState, reason);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, true);
     }
 
@@ -105,7 +105,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
         String reason = "test non existent shards are marked as successful";
         ClusterState currentState = createClusterStateWithStartedShards(reason);
         List<ShardStateAction.ShardEntry> tasks = createNonExistentShards(currentState, reason);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(clusterState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(clusterState, tasks);
         assertTasksSuccessful(tasks, result, clusterState, false);
     }
 
@@ -123,7 +123,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
         List<ShardStateAction.ShardEntry> tasks = new ArrayList<>();
         tasks.addAll(failingTasks);
         tasks.addAll(nonExistentTasks);
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = failingExecutor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = failingExecutor.execute(currentState, tasks);
         Map<ShardStateAction.ShardEntry, ClusterStateTaskExecutor.TaskResult> taskResultMap =
             failingTasks.stream().collect(Collectors.toMap(Function.identity(), task -> ClusterStateTaskExecutor.TaskResult.failure(new RuntimeException("simulated applyFailedShards failure"))));
         taskResultMap.putAll(nonExistentTasks.stream().collect(Collectors.toMap(Function.identity(), task -> ClusterStateTaskExecutor.TaskResult.success())));
@@ -146,7 +146,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
                 task -> ClusterStateTaskExecutor.TaskResult.failure(new ShardStateAction.NoLongerPrimaryShardException(task.shardId,
                     "primary term [" + task.primaryTerm + "] did not match current primary term [" +
                         currentState.metaData().index(task.shardId.getIndex()).primaryTerm(task.shardId.id()) + "]"))));
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result = executor.execute(currentState, tasks);
         assertTaskResults(taskResultMap, result, currentState, false);
     }
 
@@ -214,7 +214,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
 
     private static void assertTasksSuccessful(
         List<ShardStateAction.ShardEntry> tasks,
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result,
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result,
         ClusterState clusterState,
         boolean clusterStateChanged
     ) {
@@ -225,7 +225,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
 
     private static void assertTaskResults(
         Map<ShardStateAction.ShardEntry, ClusterStateTaskExecutor.TaskResult> taskResultMap,
-        ClusterStateTaskExecutor.BatchResult<ShardStateAction.ShardEntry> result,
+        ClusterStateTaskExecutor.ClusterTasksResult<ShardStateAction.ShardEntry> result,
         ClusterState clusterState,
         boolean clusterStateChanged
     ) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index ce6704cef82..58c9651a82a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -54,6 +54,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongConsumer;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -71,7 +72,7 @@ public class ShardStateActionTests extends ESTestCase {
     private ClusterService clusterService;
 
     private static class TestShardStateAction extends ShardStateAction {
-        public TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) {
+        TestShardStateAction(Settings settings, ClusterService clusterService, TransportService transportService, AllocationService allocationService, RoutingService routingService) {
             super(settings, clusterService, transportService, allocationService, routingService, THREAD_POOL);
         }
 
@@ -88,9 +89,9 @@ public class ShardStateActionTests extends ESTestCase {
         }
 
         @Override
-        protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener) {
+        protected void waitForNewMasterAndRetry(String actionName, ClusterStateObserver observer, ShardEntry shardEntry, Listener listener, Predicate<ClusterState> changePredicate) {
             onBeforeWaitForNewMasterAndRetry.run();
-            super.waitForNewMasterAndRetry(actionName, observer, shardEntry, listener);
+            super.waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
             onAfterWaitForNewMasterAndRetry.run();
         }
     }
@@ -107,7 +108,7 @@ public class ShardStateActionTests extends ESTestCase {
         this.transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
         transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
index 03866269cae..93771019ef4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/FilteringAllocationIT.java
@@ -24,8 +24,10 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -144,5 +146,15 @@ public class FilteringAllocationIT extends ESIntegTestCase {
         clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(clusterState.routingTable().index("test").numberOfNodesShardsAreAllocatedOn(), equalTo(2));
     }
+
+    public void testInvalidIPFilterClusterSettings() {
+        String ipKey = randomFrom("_ip", "_host_ip", "_publish_ip");
+        Setting<Settings> filterSetting = randomFrom(FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP_SETTING,
+            FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(filterSetting.getKey() + ipKey, "192.168.1.1."))
+            .execute().actionGet());
+        assertEquals("invalid IP address [192.168.1.1.] for [" + ipKey + "]", e.getMessage());
+    }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 937658736d6..229862afa9b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -95,7 +96,7 @@ public class ClusterStateHealthTests extends ESTestCase {
         super.setUp();
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
@@ -130,22 +131,17 @@ public class ClusterStateHealthTests extends ESTestCase {
         });
 
         logger.info("--> submit task to restore master");
-        clusterService.submitStateUpdateTask("restore master", new ClusterStateUpdateTask() {
+        clusterService.submitStateUpdateTask("restore master", new LocalClusterUpdateTask() {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                final DiscoveryNodes nodes = currentState.nodes();
-                return ClusterState.builder(currentState).nodes(DiscoveryNodes.builder(nodes).masterNodeId(nodes.getLocalNodeId())).build();
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
+                return newState(ClusterState.builder(currentState).nodes(
+                    DiscoveryNodes.builder(currentState.nodes()).masterNodeId(currentState.nodes().getLocalNodeId())).build());
             }
 
             @Override
             public void onFailure(String source, Exception e) {
                 logger.warn("unexpected failure", e);
             }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
         });
 
         logger.info("--> waiting for listener to be called and cluster state being blocked");
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index 0ecde102e80..ea5c55d9a0d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -57,7 +57,7 @@ public class IndexGraveyardTests extends ESTestCase {
         final IndexGraveyard graveyard = createRandom();
         final BytesStreamOutput out = new BytesStreamOutput();
         graveyard.writeTo(out);
-        assertThat(IndexGraveyard.fromStream(out.bytes().streamInput()), equalTo(graveyard));
+        assertThat(new IndexGraveyard(out.bytes().streamInput()), equalTo(graveyard));
     }
 
     public void testXContent() throws IOException {
@@ -68,7 +68,7 @@ public class IndexGraveyardTests extends ESTestCase {
         builder.endObject();
         XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
         parser.nextToken(); // the beginning of the parser
-        assertThat(IndexGraveyard.PROTO.fromXContent(parser), equalTo(graveyard));
+        assertThat(IndexGraveyard.fromXContent(parser), equalTo(graveyard));
     }
 
     public void testAddTombstones() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
index caeb889800c..7b11f96ac4d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
@@ -53,7 +53,7 @@ public class IndexMetaDataTests extends ESTestCase {
         metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
-        final IndexMetaData fromXContentMeta = IndexMetaData.PROTO.fromXContent(parser, null);
+        final IndexMetaData fromXContentMeta = IndexMetaData.fromXContent(parser);
         assertEquals(metaData, fromXContentMeta);
         assertEquals(metaData.hashCode(), fromXContentMeta.hashCode());
 
@@ -67,7 +67,7 @@ public class IndexMetaDataTests extends ESTestCase {
 
         final BytesStreamOutput out = new BytesStreamOutput();
         metaData.writeTo(out);
-        IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(out.bytes().streamInput());
+        IndexMetaData deserialized = IndexMetaData.readFrom(out.bytes().streamInput());
         assertEquals(metaData, deserialized);
         assertEquals(metaData.hashCode(), deserialized.hashCode());
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
index 5265a7548a6..bfc6f5d78d2 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
@@ -58,7 +58,7 @@ public class IndexTemplateMetaDataTests extends ESTestCase {
 
         try (StreamInput in = bytes.streamInput()) {
             in.setVersion(Version.V_5_0_0);
-            IndexTemplateMetaData readMetaData = IndexTemplateMetaData.Builder.readFrom(in);
+            IndexTemplateMetaData readMetaData = IndexTemplateMetaData.readFrom(in);
             assertEquals(0, in.available());
             assertEquals(metaData.getName(), readMetaData.getName());
             assertEquals(metaData.getPatterns(), readMetaData.getPatterns());
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index d44e533fb54..50fe3c88b65 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -38,13 +38,16 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 
 import static java.util.Collections.emptyMap;
+import static java.util.Collections.min;
 import static org.hamcrest.Matchers.endsWith;
 
 public class MetaDataCreateIndexServiceTests extends ESTestCase {
@@ -150,11 +153,20 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
 
     public void testShrinkIndexSettings() {
         String indexName = randomAsciiOfLength(10);
+        List<Version> versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()),
+            VersionUtils.randomVersion(random()));
+        versions.sort((l, r) -> Long.compare(l.id, r.id));
+        Version version = versions.get(0);
+        Version minCompat = versions.get(1);
+        Version upgraded = versions.get(2);
         // create one that won't fail
         ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0,
             Settings.builder()
                 .put("index.blocks.write", true)
                 .put("index.similarity.default.type", "BM25")
+                .put("index.version.created", version)
+                .put("index.version.upgraded", upgraded)
+                .put("index.version.minimum_compatible", minCompat.luceneVersion)
                 .put("index.analysis.analyzer.my_analyzer.tokenizer", "keyword")
                 .build())).nodes(DiscoveryNodes.builder().add(newNode("node1")))
             .build();
@@ -177,6 +189,10 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
             "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer"));
         assertEquals("node1", builder.build().get("index.routing.allocation.initial_recovery._id"));
         assertEquals("1", builder.build().get("index.allocation.max_retries"));
+        assertEquals(version, builder.build().getAsVersion("index.version.created", null));
+        assertEquals(upgraded, builder.build().getAsVersion("index.version.upgraded", null));
+        assertEquals(minCompat.luceneVersion.toString(), builder.build().get("index.version.minimum_compatible", null));
+
     }
 
     private DiscoveryNode newNode(String nodeId) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
index a48f3ae3e10..0ad36713810 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexServiceTests.java
@@ -59,7 +59,7 @@ public class MetaDataDeleteIndexServiceTests extends ESTestCase {
         Snapshot snapshot = new Snapshot("doesn't matter", new SnapshotId("snapshot name", "snapshot uuid"));
         SnapshotsInProgress snaps = new SnapshotsInProgress(new SnapshotsInProgress.Entry(snapshot, true, false,
                 SnapshotsInProgress.State.INIT, singletonList(new IndexId(index, "doesn't matter")),
-                System.currentTimeMillis(), ImmutableOpenMap.of()));
+                System.currentTimeMillis(), (long) randomIntBetween(0, 1000), ImmutableOpenMap.of()));
         ClusterState state = ClusterState.builder(clusterState(index))
                 .putCustom(SnapshotsInProgress.TYPE, snaps)
                 .build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
index 8a342057dab..3b894eb8d7c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexAliasesServiceTests.java
@@ -41,7 +41,7 @@ public class MetaDataIndexAliasesServiceTests extends ESTestCase {
     private final AliasValidator aliasValidator = new AliasValidator(Settings.EMPTY);
     private final MetaDataDeleteIndexService deleteIndexService = mock(MetaDataDeleteIndexService.class);
     private final MetaDataIndexAliasesService service = new MetaDataIndexAliasesService(Settings.EMPTY, null, null, aliasValidator,
-            deleteIndexService);
+            deleteIndexService, xContentRegistry());
 
     public MetaDataIndexAliasesServiceTests() {
         // Mock any deletes so we don't need to worry about how MetaDataDeleteIndexService does its job
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
index a1dfdbc74f9..88e2835b5ab 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
@@ -30,7 +30,8 @@ import java.util.Collections;
 public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testArchiveBrokenIndexSettings() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
         IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src);
         assertSame(indexMetaData, src);
@@ -56,8 +57,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
     }
 
     public void testUpgrade() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(),
-            Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
@@ -68,8 +69,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
     }
 
     public void testIsUpgraded() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(),
-            Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion());
@@ -80,8 +81,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
     }
 
     public void testFailUpgrade() {
-        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, new MapperRegistry(Collections.emptyMap(),
-            Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
         final IndexMetaData metaData = newIndexMeta("foo", Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_5_0_0_beta1)
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("2.4.0"))
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index 4d248adf1e6..6cc6a1cb54c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -20,9 +20,13 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -173,7 +177,7 @@ public class MetaDataTests extends ESTestCase {
         originalMeta.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
         XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
-        final MetaData fromXContentMeta = MetaData.PROTO.fromXContent(parser, null);
+        final MetaData fromXContentMeta = MetaData.fromXContent(parser);
         assertThat(fromXContentMeta.indexGraveyard(), equalTo(originalMeta.indexGraveyard()));
     }
 
@@ -182,7 +186,10 @@ public class MetaDataTests extends ESTestCase {
         final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build();
         final BytesStreamOutput out = new BytesStreamOutput();
         originalMeta.writeTo(out);
-        final MetaData fromStreamMeta = MetaData.PROTO.readFrom(out.bytes().streamInput());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+        final MetaData fromStreamMeta = MetaData.readFrom(
+            new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry)
+        );
         assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
index 7a69df8fcfc..3b80d1f6e2c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java
@@ -155,7 +155,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.getSettings().size(), equalTo(3));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
 
         indexMetaData = parsedMetaData.index("test2");
@@ -164,7 +164,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.primaryTerm(0), equalTo(2L));
         assertThat(indexMetaData.primaryTerm(1), equalTo(2L));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
@@ -173,7 +173,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(3));
+        assertThat(indexMetaData.getSettings().size(), equalTo(3));
         assertThat(indexMetaData.getMappings().size(), equalTo(1));
         assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
 
@@ -181,14 +181,14 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.getSettings().size(), equalTo(4));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
 
         indexMetaData = parsedMetaData.index("test5");
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -199,7 +199,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(0));
@@ -208,7 +208,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(4));
+        assertThat(indexMetaData.getSettings().size(), equalTo(4));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
         assertThat(indexMetaData.getMappings().get("mapping1").source().string(), equalTo(MAPPING_SOURCE1));
         assertThat(indexMetaData.getMappings().get("mapping2").source().string(), equalTo(MAPPING_SOURCE2));
@@ -217,7 +217,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -231,7 +231,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -245,7 +245,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -259,7 +259,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(-1L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5));
+        assertThat(indexMetaData.getSettings().size(), equalTo(5));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
@@ -277,7 +277,7 @@ public class ToAndFromJsonMetaDataTests extends ESTestCase {
         assertThat(indexMetaData.getNumberOfShards(), equalTo(1));
         assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2));
         assertThat(indexMetaData.getCreationDate(), equalTo(2L));
-        assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(6));
+        assertThat(indexMetaData.getSettings().size(), equalTo(6));
         assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1"));
         assertThat(indexMetaData.getSettings().get("setting2"), equalTo("value2"));
         assertThat(indexMetaData.getMappings().size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
index 1f226427237..c78de758d96 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
@@ -245,6 +245,17 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
         assertThat(filters.match(node), equalTo(true));
     }
 
+    public void testCommaSeparatedValuesTrimmed() {
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "B"), emptySet(), null);
+
+        Settings settings = shuffleSettings(Settings.builder()
+            .put("xxx." + randomFrom("_ip", "_host_ip", "_publish_ip"), "192.1.1.1, 192.1.1.54")
+            .put("xxx.tag", "A, B")
+            .build());
+        DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
+        assertTrue(filters.match(node));
+    }
+
     private Settings shuffleSettings(Settings source) {
         Settings.Builder settings = Settings.builder();
         List<String> keys = new ArrayList<>(source.getAsMap().keySet());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 994510ec56c..f4ce40e16db 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -464,7 +464,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
     private static class TestDelayAllocationService extends DelayedAllocationService {
         private volatile long nanoTimeOverride = -1L;
 
-        public TestDelayAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+        TestDelayAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                           AllocationService allocationService) {
             super(settings, threadPool, clusterService, allocationService);
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index 3118692b341..c24a86176dc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -22,7 +22,6 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -35,6 +34,7 @@ import org.elasticsearch.threadpool.TestThreadPool;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -43,8 +43,8 @@ import java.util.TreeMap;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.object.HasToString.hasToString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.object.HasToString.hasToString;
 
 public class OperationRoutingTests extends ESTestCase{
 
@@ -75,6 +75,142 @@ public class OperationRoutingTests extends ESTestCase{
         }
     }
 
+    public void testPartitionedIndex() {
+        // make sure the same routing value always has each _id fall within the configured partition size
+        for (int shards = 1; shards < 5; shards++) {
+            for (int partitionSize = 1; partitionSize == 1 || partitionSize < shards; partitionSize++) {
+                IndexMetaData metaData = IndexMetaData.builder("test")
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(shards)
+                    .routingPartitionSize(partitionSize)
+                    .numberOfReplicas(1)
+                    .build();
+
+                for (int i = 0; i < 20; i++) {
+                    String routing = randomUnicodeOfLengthBetween(1, 50);
+                    Set<Integer> shardSet = new HashSet<>();
+
+                    for (int k = 0; k < 150; k++) {
+                        String id = randomUnicodeOfLengthBetween(1, 50);
+
+                        shardSet.add(OperationRouting.generateShardId(metaData, id, routing));
+                    }
+
+                    assertEquals(partitionSize, shardSet.size());
+                }
+            }
+        }
+    }
+
+    public void testPartitionedIndexShrunk() {
+        Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
+
+        Map<String, Integer> routingA = new HashMap<>();
+        routingA.put("a_0", 1);
+        routingA.put("a_1", 2);
+        routingA.put("a_2", 2);
+        routingA.put("a_3", 2);
+        routingA.put("a_4", 1);
+        routingA.put("a_5", 2);
+        routingIdToShard.put("a", routingA);
+
+        Map<String, Integer> routingB = new HashMap<>();
+        routingB.put("b_0", 0);
+        routingB.put("b_1", 0);
+        routingB.put("b_2", 0);
+        routingB.put("b_3", 0);
+        routingB.put("b_4", 3);
+        routingB.put("b_5", 3);
+        routingIdToShard.put("b", routingB);
+
+        Map<String, Integer> routingC = new HashMap<>();
+        routingC.put("c_0", 1);
+        routingC.put("c_1", 1);
+        routingC.put("c_2", 0);
+        routingC.put("c_3", 0);
+        routingC.put("c_4", 0);
+        routingC.put("c_5", 1);
+        routingIdToShard.put("c", routingC);
+
+        Map<String, Integer> routingD = new HashMap<>();
+        routingD.put("d_0", 2);
+        routingD.put("d_1", 2);
+        routingD.put("d_2", 3);
+        routingD.put("d_3", 3);
+        routingD.put("d_4", 3);
+        routingD.put("d_5", 3);
+        routingIdToShard.put("d", routingD);
+
+        IndexMetaData metaData = IndexMetaData.builder("test")
+                .settings(settings(Version.CURRENT))
+                .setRoutingNumShards(8)
+                .numberOfShards(4)
+                .routingPartitionSize(3)
+                .numberOfReplicas(1)
+                .build();
+
+        for (Map.Entry<String, Map<String, Integer>> routingIdEntry : routingIdToShard.entrySet()) {
+            String routing = routingIdEntry.getKey();
+
+            for (Map.Entry<String, Integer> idEntry : routingIdEntry.getValue().entrySet()) {
+                String id = idEntry.getKey();
+                int shard = idEntry.getValue();
+
+                assertEquals(shard, OperationRouting.generateShardId(metaData, id, routing));
+            }
+        }
+    }
+
+    public void testPartitionedIndexBWC() {
+        Map<String, Map<String, Integer>> routingIdToShard = new HashMap<>();
+
+        Map<String, Integer> routingA = new HashMap<>();
+        routingA.put("a_0", 3);
+        routingA.put("a_1", 2);
+        routingA.put("a_2", 2);
+        routingA.put("a_3", 3);
+        routingIdToShard.put("a", routingA);
+
+        Map<String, Integer> routingB = new HashMap<>();
+        routingB.put("b_0", 5);
+        routingB.put("b_1", 0);
+        routingB.put("b_2", 0);
+        routingB.put("b_3", 0);
+        routingIdToShard.put("b", routingB);
+
+        Map<String, Integer> routingC = new HashMap<>();
+        routingC.put("c_0", 4);
+        routingC.put("c_1", 4);
+        routingC.put("c_2", 3);
+        routingC.put("c_3", 4);
+        routingIdToShard.put("c", routingC);
+
+        Map<String, Integer> routingD = new HashMap<>();
+        routingD.put("d_0", 3);
+        routingD.put("d_1", 4);
+        routingD.put("d_2", 4);
+        routingD.put("d_3", 4);
+        routingIdToShard.put("d", routingD);
+
+        IndexMetaData metaData = IndexMetaData.builder("test")
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(6)
+                .routingPartitionSize(2)
+                .numberOfReplicas(1)
+                .build();
+
+        for (Map.Entry<String, Map<String, Integer>> routingIdEntry : routingIdToShard.entrySet()) {
+            String routing = routingIdEntry.getKey();
+
+            for (Map.Entry<String, Integer> idEntry : routingIdEntry.getValue().entrySet()) {
+                String id = idEntry.getKey();
+                int shard = idEntry.getValue();
+
+                assertEquals(shard, OperationRouting.generateShardId(metaData, id, routing));
+            }
+        }
+    }
+
     /**
      * Ensures that all changes to the hash-function / shard selection are BWC
      */
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
index 211071c9555..09b957069a6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
@@ -46,7 +46,7 @@ public class RoutingServiceTests extends ESAllocationTestCase {
 
         private AtomicBoolean rerouted = new AtomicBoolean();
 
-        public TestRoutingService() {
+        TestRoutingService() {
             super(Settings.EMPTY, null, null);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
index 79f8f4f03bf..0f194a0f912 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecisionTests.java
@@ -71,7 +71,7 @@ public class AllocateUnassignedDecisionTests extends ESTestCase {
             assertThat(noDecision.getExplanation(), startsWith("cannot allocate because the cluster is still waiting"));
         } else {
             assertThat(noDecision.getExplanation(),
-                startsWith("cannot allocate because a previous copy of the shard existed"));
+                startsWith("cannot allocate because a previous copy of the primary shard existed"));
         }
         assertNull(noDecision.getNodeDecisions());
         assertNull(noDecision.getTargetNode());
@@ -107,7 +107,7 @@ public class AllocateUnassignedDecisionTests extends ESTestCase {
         nodeDecisions.add(new NodeAllocationResult(node2, Decision.THROTTLE, 2));
         AllocateUnassignedDecision throttleDecision = AllocateUnassignedDecision.throttle(nodeDecisions);
         assertTrue(throttleDecision.isDecisionTaken());
-        assertEquals(AllocationDecision.THROTTLE, throttleDecision.getAllocationDecision());
+        assertEquals(AllocationDecision.THROTTLED, throttleDecision.getAllocationDecision());
         assertEquals(AllocationStatus.DECIDERS_THROTTLED, throttleDecision.getAllocationStatus());
         assertThat(throttleDecision.getExplanation(), startsWith("allocation temporarily throttled"));
         assertEquals(nodeDecisions.stream().sorted().collect(Collectors.toList()), throttleDecision.getNodeDecisions());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index f0a50811d38..8e9cddcbea4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -33,12 +33,10 @@ import org.elasticsearch.cluster.routing.allocation.command.AbstractAllocateAllo
 import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -46,6 +44,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -489,8 +488,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         // move two tokens, parser expected to be "on" `commands` field
         parser.nextToken();
         parser.nextToken();
-        AllocationCommandRegistry registry = NetworkModule.getAllocationCommandRegistry();
-        AllocationCommands sCommands = AllocationCommands.fromXContent(parser, ParseFieldMatcher.STRICT, registry);
+        AllocationCommands sCommands = AllocationCommands.fromXContent(parser);
 
         assertThat(sCommands.commands().size(), equalTo(5));
         assertThat(((AllocateEmptyPrimaryAllocationCommand) (sCommands.commands().get(0))).shardId(), equalTo(1));
@@ -517,4 +515,9 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).node(), equalTo("node5"));
         assertThat(((CancelAllocationCommand) (sCommands.commands().get(4))).allowPrimary(), equalTo(true));
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(NetworkModule.getNamedXContents());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java
index 7f3c3466cd4..0c76338cefc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationDecisionTests.java
@@ -48,21 +48,21 @@ public class AllocationDecisionTests extends ESTestCase {
      */
     public void testValuesOrder() {
         assertEquals(0, AllocationDecision.YES.ordinal());
-        assertEquals(1, AllocationDecision.THROTTLE.ordinal());
+        assertEquals(1, AllocationDecision.THROTTLED.ordinal());
         assertEquals(2, AllocationDecision.NO.ordinal());
         assertEquals(3, AllocationDecision.WORSE_BALANCE.ordinal());
-        assertEquals(4, AllocationDecision.FETCH_PENDING.ordinal());
-        assertEquals(5, AllocationDecision.DELAYED_ALLOCATION.ordinal());
+        assertEquals(4, AllocationDecision.AWAITING_INFO.ordinal());
+        assertEquals(5, AllocationDecision.ALLOCATION_DELAYED.ordinal());
         assertEquals(6, AllocationDecision.NO_VALID_SHARD_COPY.ordinal());
         assertEquals(7, AllocationDecision.NO_ATTEMPT.ordinal());
         AllocationDecision[] decisions = AllocationDecision.values();
         Arrays.sort(decisions);
         assertEquals(AllocationDecision.YES, decisions[0]);
-        assertEquals(AllocationDecision.THROTTLE, decisions[1]);
+        assertEquals(AllocationDecision.THROTTLED, decisions[1]);
         assertEquals(AllocationDecision.NO, decisions[2]);
         assertEquals(AllocationDecision.WORSE_BALANCE, decisions[3]);
-        assertEquals(AllocationDecision.FETCH_PENDING, decisions[4]);
-        assertEquals(AllocationDecision.DELAYED_ALLOCATION, decisions[5]);
+        assertEquals(AllocationDecision.AWAITING_INFO, decisions[4]);
+        assertEquals(AllocationDecision.ALLOCATION_DELAYED, decisions[5]);
         assertEquals(AllocationDecision.NO_VALID_SHARD_COPY, decisions[6]);
         assertEquals(AllocationDecision.NO_ATTEMPT, decisions[7]);
     }
@@ -74,7 +74,7 @@ public class AllocationDecisionTests extends ESTestCase {
         Type type = randomFrom(Type.values());
         AllocationDecision allocationDecision = AllocationDecision.fromDecisionType(type);
         AllocationDecision expected = type == Type.NO ? AllocationDecision.NO :
-                                          type == Type.THROTTLE ? AllocationDecision.THROTTLE : AllocationDecision.YES;
+                                          type == Type.THROTTLE ? AllocationDecision.THROTTLED : AllocationDecision.YES;
         assertEquals(expected, allocationDecision);
     }
 
@@ -88,11 +88,11 @@ public class AllocationDecisionTests extends ESTestCase {
         if (allocationStatus == null) {
             expected = AllocationDecision.YES;
         } else if (allocationStatus == AllocationStatus.DECIDERS_THROTTLED) {
-            expected = AllocationDecision.THROTTLE;
+            expected = AllocationDecision.THROTTLED;
         } else if (allocationStatus == AllocationStatus.FETCHING_SHARD_DATA) {
-            expected = AllocationDecision.FETCH_PENDING;
+            expected = AllocationDecision.AWAITING_INFO;
         } else if (allocationStatus == AllocationStatus.DELAYED_ALLOCATION) {
-            expected = AllocationDecision.DELAYED_ALLOCATION;
+            expected = AllocationDecision.ALLOCATION_DELAYED;
         } else if (allocationStatus == AllocationStatus.NO_VALID_SHARD_COPY) {
             expected = AllocationDecision.NO_VALID_SHARD_COPY;
         } else if (allocationStatus == AllocationStatus.NO_ATTEMPT) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index fec0a33b917..2c1ec07c7fa 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -32,10 +32,14 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
@@ -803,4 +807,50 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(4)); // +1 for relocating shard.
         assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1)); // Still 1 unassigned.
     }
+
+    public void testMultipleAwarenessAttributes() {
+        AllocationService strategy = createAllocationService(Settings.builder()
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack")
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b")
+            .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d")
+            .put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always")
+            .build());
+
+        logger.info("Building initial routing table for 'testUnbalancedZones'");
+
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
+
+        ClusterState clusterState = ClusterState.builder(
+            org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)
+        ).metaData(metaData).routingTable(initialRoutingTable).build();
+
+        logger.info("--> adding two nodes in different zones and do rerouting");
+        Map<String, String> nodeAAttributes = new HashMap<>();
+        nodeAAttributes.put("zone", "a");
+        nodeAAttributes.put("rack", "c");
+        Map<String, String> nodeBAttributes = new HashMap<>();
+        nodeBAttributes.put("zone", "b");
+        nodeBAttributes.put("rack", "d");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
+            .add(newNode("A-0", nodeAAttributes))
+            .add(newNode("B-0", nodeBAttributes))
+        ).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+
+        logger.info("--> start the shards (primaries)");
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+
+        clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        logger.info("--> all replicas are allocated and started since we have one node in each zone and rack");
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index e2a360b93fe..cfca80b78ca 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -45,9 +45,7 @@ import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.test.gateway.TestGatewayAllocator;
 import org.hamcrest.Matchers;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -257,10 +255,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
         AllocationService strategy = new AllocationService(settings.build(), randomAllocationDeciders(settings.build(),
                 new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), random()),
                 new TestGatewayAllocator(), new ShardsAllocator() {
-
-                    public Map<DiscoveryNode, Float> weighShard(RoutingAllocation allocation, ShardRouting shard) {
-                        return new HashMap<DiscoveryNode, Float>();
-                    }
             /*
              *  // this allocator tries to rebuild this scenario where a rebalance is
              *  // triggered solely by the primary overload on node [1] where a shard
@@ -327,6 +321,11 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
 
                 }
             }
+
+            @Override
+            public ShardAllocationDecision decideShardAllocation(ShardRouting shard, RoutingAllocation allocation) {
+                throw new UnsupportedOperationException("explain not supported");
+            }
         }, EmptyClusterInfoService.INSTANCE);
         MetaData.Builder metaDataBuilder = MetaData.builder();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
@@ -391,7 +390,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
 
     private class NoopGatewayAllocator extends GatewayAllocator {
 
-        public NoopGatewayAllocator() {
+        NoopGatewayAllocator() {
             super(Settings.EMPTY, null, null);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
index b3089e57db6..cb69a5de87f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java
@@ -60,8 +60,8 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         ClusterState clusterState = ClusterStateCreationUtils.state("idx", randomBoolean(),
             randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.UNASSIGNED, ShardRoutingState.RELOCATING));
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
-        MoveDecision rebalanceDecision = allocator.decideRebalance(shard, newRoutingAllocation(
-            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState));
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, newRoutingAllocation(
+            new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState)).getMoveDecision();
         assertSame(MoveDecision.NOT_TAKEN, rebalanceDecision);
     }
 
@@ -72,9 +72,9 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         RoutingAllocation routingAllocation = newRoutingAllocation(
             new AllocationDeciders(Settings.EMPTY, Collections.emptyList()), clusterState);
         routingAllocation.setHasPendingAsyncFetch();
-        MoveDecision rebalanceDecision = allocator.decideRebalance(shard, routingAllocation);
-        assertNotNull(rebalanceDecision.getCanRebalanceDecision());
-        assertEquals(AllocationDecision.FETCH_PENDING, rebalanceDecision.getAllocationDecision());
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+        assertNotNull(rebalanceDecision.getClusterRebalanceDecision());
+        assertEquals(AllocationDecision.AWAITING_INFO, rebalanceDecision.getAllocationDecision());
         assertThat(rebalanceDecision.getExplanation(),
             startsWith("cannot rebalance as information about existing copies of this shard in the cluster is still being gathered"));
         assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
@@ -96,15 +96,15 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         RoutingAllocation routingAllocation =  newRoutingAllocation(
             new AllocationDeciders(Settings.EMPTY, Collections.singleton(noRebalanceDecider)), clusterState);
-        MoveDecision rebalanceDecision = allocator.decideRebalance(shard, routingAllocation);
-        assertEquals(canRebalanceDecision.type(), rebalanceDecision.getCanRebalanceDecision().type());
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
+        assertEquals(canRebalanceDecision.type(), rebalanceDecision.getClusterRebalanceDecision().type());
         assertEquals(AllocationDecision.fromDecisionType(canRebalanceDecision.type()), rebalanceDecision.getAllocationDecision());
         assertThat(rebalanceDecision.getExplanation(), containsString(canRebalanceDecision.type() == Type.THROTTLE ?
             "rebalancing is throttled" : "rebalancing is not allowed"));
         assertNotNull(rebalanceDecision.getNodeDecisions());
         assertNull(rebalanceDecision.getTargetNode());
-        assertEquals(1, rebalanceDecision.getCanRebalanceDecision().getDecisions().size());
-        for (Decision subDecision : rebalanceDecision.getCanRebalanceDecision().getDecisions()) {
+        assertEquals(1, rebalanceDecision.getClusterRebalanceDecision().getDecisions().size());
+        for (Decision subDecision : rebalanceDecision.getClusterRebalanceDecision().getDecisions()) {
             assertEquals("foobar", ((Decision.Single) subDecision).getExplanation());
         }
 
@@ -121,7 +121,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, Settings.EMPTY, true);
         ClusterState clusterState = rebalance.v1();
         MoveDecision rebalanceDecision = rebalance.v2();
-        assertEquals(Type.YES, rebalanceDecision.getCanRebalanceDecision().type());
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
         assertNotNull(rebalanceDecision.getExplanation());
         assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
     }
@@ -136,7 +136,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, Settings.EMPTY, false);
         ClusterState clusterState = rebalance.v1();
         MoveDecision rebalanceDecision = rebalance.v2();
-        assertEquals(Type.YES, rebalanceDecision.getCanRebalanceDecision().type());
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
         assertEquals(AllocationDecision.NO, rebalanceDecision.getAllocationDecision());
         assertThat(rebalanceDecision.getExplanation(), startsWith(
             "cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance"));
@@ -161,7 +161,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         Tuple<ClusterState, MoveDecision> rebalance = setupStateAndRebalance(canAllocateDecider, balancerSettings, false);
         ClusterState clusterState = rebalance.v1();
         MoveDecision rebalanceDecision = rebalance.v2();
-        assertEquals(Type.YES, rebalanceDecision.getCanRebalanceDecision().type());
+        assertEquals(Type.YES, rebalanceDecision.getClusterRebalanceDecision().type());
         assertEquals(AllocationDecision.NO, rebalanceDecision.getAllocationDecision());
         assertNotNull(rebalanceDecision.getExplanation());
         assertEquals(clusterState.nodes().getSize() - 1, rebalanceDecision.getNodeDecisions().size());
@@ -232,7 +232,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         routingAllocation = newRoutingAllocation(new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
         routingAllocation.debugDecision(true);
         ShardRouting shard = clusterState.getRoutingNodes().activePrimary(shardToRebalance.shardId());
-        MoveDecision rebalanceDecision = allocator.decideRebalance(shard, routingAllocation);
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
         assertEquals(shardToRebalance.relocatingNodeId(), rebalanceDecision.getTargetNode().getId());
         // make sure all excluded nodes returned a NO decision
         for (NodeAllocationResult nodeResult : rebalanceDecision.getNodeDecisions()) {
@@ -325,7 +325,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         BalancedShardsAllocator allocator = new BalancedShardsAllocator(settings);
         RoutingAllocation routingAllocation = newRoutingAllocation(
             new AllocationDeciders(Settings.EMPTY, Arrays.asList(allocationDecider, rebalanceDecider)), clusterState);
-        return allocator.decideRebalance(shardRouting, routingAllocation);
+        return allocator.decideShardAllocation(shardRouting, routingAllocation).getMoveDecision();
     }
 
     private ClusterState addNodesToClusterState(ClusterState clusterState, int numNodesToAdd) {
@@ -357,7 +357,7 @@ public class BalancedSingleShardTests extends ESAllocationTestCase {
         ShardRouting shard = clusterState.routingTable().index("idx").shard(0).primaryShard();
         RoutingAllocation routingAllocation = newRoutingAllocation(
             new AllocationDeciders(Settings.EMPTY, allocationDeciders), clusterState);
-        MoveDecision rebalanceDecision = allocator.decideRebalance(shard, routingAllocation);
+        MoveDecision rebalanceDecision = allocator.decideShardAllocation(shard, routingAllocation).getMoveDecision();
 
         if (rebalanceExpected == false) {
             assertAssignedNodeRemainsSame(allocator, routingAllocation, shard);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
index 5ec162eb719..d4ec30f6e51 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
@@ -37,6 +37,8 @@ import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDeci
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -191,4 +193,16 @@ public class FilterAllocationDeciderTests extends ESAllocationTestCase {
             .build();
         return service.reroute(clusterState, "reroute", false);
     }
+
+    public void testInvalidIPFilter() {
+        String ipKey = randomFrom("_ip", "_host_ip", "_publish_ip");
+        Setting<Settings> filterSetting = randomFrom(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING,
+            IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING, IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+            indexScopedSettings.updateDynamicSettings(Settings.builder().put(filterSetting.getKey() + ipKey, "192..168.1.1").build(),
+                Settings.builder().put(Settings.EMPTY), Settings.builder(), "test ip validation");
+        });
+        assertEquals("invalid IP address [192..168.1.1] for [" + ipKey + "]", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java
index f066b655269..c5474803908 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java
@@ -72,7 +72,6 @@ public class MoveDecisionTests extends ESTestCase {
         assertFalse(stay.forceMove());
         assertTrue(stay.isDecisionTaken());
         assertNull(stay.getNodeDecisions());
-        assertNotNull(stay.getExplanation());
         assertEquals(AllocationDecision.NO_ATTEMPT, stay.getAllocationDecision());
 
         stay = MoveDecision.stay(Decision.YES);
@@ -80,7 +79,6 @@ public class MoveDecisionTests extends ESTestCase {
         assertFalse(stay.forceMove());
         assertTrue(stay.isDecisionTaken());
         assertNull(stay.getNodeDecisions());
-        assertEquals("shard can remain on its current node", stay.getExplanation());
         assertEquals(AllocationDecision.NO_ATTEMPT, stay.getAllocationDecision());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 3aba43f366a..d005b05737d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -786,8 +786,9 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         } catch (IllegalArgumentException e) {
             assertThat("can't be allocated because there isn't enough room: " + e.getMessage(),
                     e.getMessage(),
-                    containsString("the node is above the low watermark [cluster.routing.allocation.disk.watermark.low=0.7], using " +
-                                   "more disk space than the maximum allowed [70.0%], actual free: [26.0%]"));
+                    containsString("the node is above the low watermark cluster setting " +
+                                   "[cluster.routing.allocation.disk.watermark.low=0.7], using more disk space than the maximum " +
+                                   "allowed [70.0%], actual free: [26.0%]"));
         }
 
     }
@@ -858,7 +859,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
         assertThat(((Decision.Single) decision).getExplanation(), containsString(
-            "the shard cannot remain on this node because it is above the high watermark " +
+            "the shard cannot remain on this node because it is above the high watermark cluster setting " +
             "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " +
             "actual free: [20.0%]"));
 
@@ -890,12 +891,12 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         assertThat(decision.type(), equalTo(Decision.Type.NO));
         if (fooRouting.recoverySource().getType() == RecoverySource.Type.EMPTY_STORE) {
             assertThat(((Decision.Single) decision).getExplanation(), containsString(
-                "the node is above the high watermark [cluster.routing.allocation.disk.watermark.high=70%], using more disk space than " +
-                "the maximum allowed [70.0%], actual free: [20.0%]"));
+                "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=70%], using " +
+                "more disk space than the maximum allowed [70.0%], actual free: [20.0%]"));
         } else {
             assertThat(((Decision.Single) decision).getExplanation(), containsString(
-                "the node is above the low watermark [cluster.routing.allocation.disk.watermark.low=60%], using more disk space than " +
-                "the maximum allowed [60.0%], actual free: [20.0%]"));
+                "the node is above the low watermark cluster setting [cluster.routing.allocation.disk.watermark.low=60%], using more " +
+                "disk space than the maximum allowed [60.0%], actual free: [20.0%]"));
         }
 
         // Creating AllocationService instance and the services it depends on...
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 659c3b25833..7379ee78d03 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -106,8 +106,8 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         decision = decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation);
         assertEquals(mostAvailableUsage.toString(), Decision.Type.NO, decision.type());
         assertThat(((Decision.Single) decision).getExplanation(), containsString(
-            "the node is above the high watermark [cluster.routing.allocation.disk.watermark.high=90%], using more disk space than " +
-            "the maximum allowed [90.0%]"));
+            "the node is above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%], using more " +
+            "disk space than the maximum allowed [90.0%]"));
     }
 
     public void testCanRemainUsesLeastAvailableSpace() {
@@ -181,8 +181,8 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         decision = decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation);
         assertEquals(Decision.Type.NO, decision.type());
         assertThat(((Decision.Single) decision).getExplanation(), containsString("the shard cannot remain on this node because it is " +
-            "above the high watermark [cluster.routing.allocation.disk.watermark.high=90%] and there is less than the required [10.0%] " +
-            "free disk on node, actual free: [9.0%]"));
+            "above the high watermark cluster setting [cluster.routing.allocation.disk.watermark.high=90%] and there is less than " +
+            "the required [10.0%] free disk on node, actual free: [9.0%]"));
         try {
             decider.canRemain(test_0, new RoutingNode("node_1", node_1), allocation);
             fail("not allocated on this node");
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index 00d9a8ff096..4e77741694e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -20,19 +20,34 @@
 package org.elasticsearch.cluster.serialization;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.Diff;
 import org.elasticsearch.cluster.ESAllocationTestCase;
+import org.elasticsearch.cluster.RestoreInProgress;
+import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 
+import java.util.Collections;
+
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ClusterSerializationTests extends ESAllocationTestCase {
 
@@ -52,7 +67,8 @@ public class ClusterSerializationTests extends ESAllocationTestCase {
         AllocationService strategy = createAllocationService();
         clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build();
 
-        ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1"));
+        ClusterState serializedClusterState = ClusterState.Builder.fromBytes(ClusterState.Builder.toBytes(clusterState), newNode("node1"),
+            new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
 
         assertThat(serializedClusterState.getClusterName().value(), equalTo(clusterState.getClusterName().value()));
 
@@ -79,9 +95,69 @@ public class ClusterSerializationTests extends ESAllocationTestCase {
         BytesStreamOutput outStream = new BytesStreamOutput();
         source.writeTo(outStream);
         StreamInput inStream = outStream.bytes().streamInput();
-        RoutingTable target = RoutingTable.Builder.readFrom(inStream);
+        RoutingTable target = RoutingTable.readFrom(inStream);
 
         assertThat(target.toString(), equalTo(source.toString()));
     }
 
+    public void testSnapshotDeletionsInProgressSerialization() throws Exception {
+
+        boolean includeRestore = randomBoolean();
+
+        ClusterState.Builder builder = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .putCustom(SnapshotDeletionsInProgress.TYPE,
+                SnapshotDeletionsInProgress.newInstance(
+                    new SnapshotDeletionsInProgress.Entry(
+                        new Snapshot("repo1", new SnapshotId("snap1", UUIDs.randomBase64UUID())),
+                        randomNonNegativeLong(), randomNonNegativeLong())
+                ));
+        if (includeRestore) {
+            builder.putCustom(RestoreInProgress.TYPE,
+                new RestoreInProgress(
+                    new RestoreInProgress.Entry(
+                        new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())),
+                        RestoreInProgress.State.STARTED,
+                        Collections.singletonList("index_name"),
+                        ImmutableOpenMap.of()
+                    )
+                ));
+        }
+
+        ClusterState clusterState = builder.incrementVersion().build();
+
+        Diff<ClusterState> diffs = clusterState.diff(ClusterState.EMPTY_STATE);
+
+        // serialize with current version
+        BytesStreamOutput outStream = new BytesStreamOutput();
+        diffs.writeTo(outStream);
+        StreamInput inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        Diff<ClusterState> serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        ClusterState stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), notNullValue());
+
+        // serialize with old version
+        outStream = new BytesStreamOutput();
+        outStream.setVersion(Version.CURRENT.minimumCompatibilityVersion());
+        diffs.writeTo(outStream);
+        inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        stateAfterDiffs = serializedDiffs.apply(ClusterState.EMPTY_STATE);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
+
+        // remove the custom and try serializing again with old version
+        clusterState = ClusterState.builder(clusterState).removeCustom(SnapshotDeletionsInProgress.TYPE).incrementVersion().build();
+        outStream = new BytesStreamOutput();
+        diffs.writeTo(outStream);
+        inStream = outStream.bytes().streamInput();
+        inStream = new NamedWriteableAwareStreamInput(inStream, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()));
+        serializedDiffs = ClusterState.readDiffFrom(inStream, clusterState.nodes().getLocalNode());
+        stateAfterDiffs = serializedDiffs.apply(stateAfterDiffs);
+        assertThat(stateAfterDiffs.custom(RestoreInProgress.TYPE), includeRestore ? notNullValue() : nullValue());
+        assertThat(stateAfterDiffs.custom(SnapshotDeletionsInProgress.TYPE), nullValue());
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index 9ce3d1fcee8..65d780f3fd9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -56,7 +56,7 @@ public class ClusterStateToStringTests extends ESAllocationTestCase {
         AllocationService strategy = createAllocationService();
         clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build();
 
-        String clusterStateString = Strings.toString(clusterState, true);
+        String clusterStateString = Strings.toString(clusterState);
         assertNotNull(clusterStateString);
 
         assertThat(clusterStateString, containsString("test_idx"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
index 611c261e334..4475aed7dc3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
@@ -65,7 +65,7 @@ public class DiffableTests extends ESTestCase {
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readJdkMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readJdkMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readJdkMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -113,7 +113,7 @@ public class DiffableTests extends ESTestCase {
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readImmutableOpenMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -161,7 +161,7 @@ public class DiffableTests extends ESTestCase {
             @Override
             protected MapDiff readDiff(StreamInput in) throws IOException {
                 return useProtoForDiffableSerialization
-                        ? DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, TestDiffable.PROTO)
+                        ? DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, TestDiffable::readFrom, TestDiffable::readDiffFrom)
                         : DiffableUtils.readImmutableOpenIntMapDiff(in, keySerializer, diffableValueSerializer());
             }
         }.execute();
@@ -398,7 +398,7 @@ public class DiffableTests extends ESTestCase {
 
             @Override
             public Diff<TestDiffable> readDiff(StreamInput in, K key) throws IOException {
-                return AbstractDiffable.readDiffFrom(TestDiffable.PROTO, in);
+                return AbstractDiffable.readDiffFrom(TestDiffable::readFrom, in);
             }
         };
     }
@@ -419,8 +419,6 @@ public class DiffableTests extends ESTestCase {
 
     public static class TestDiffable extends AbstractDiffable<TestDiffable> {
 
-        public static final TestDiffable PROTO = new TestDiffable("");
-
         private final String value;
 
         public TestDiffable(String value) {
@@ -431,11 +429,14 @@ public class DiffableTests extends ESTestCase {
             return value;
         }
 
-        @Override
-        public TestDiffable readFrom(StreamInput in) throws IOException {
+        public static TestDiffable readFrom(StreamInput in) throws IOException {
             return new TestDiffable(in.readString());
         }
 
+        public static Diff<TestDiffable> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TestDiffable::readFrom, in);
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(value);
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index 57086a0d1bf..68815399398 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.LocalNodeMasterListener;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -121,22 +122,23 @@ public class ClusterServiceTests extends ESTestCase {
     TimedClusterService createTimedClusterService(boolean makeMaster) throws InterruptedException {
         TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
             "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool);
-        timedClusterService.setLocalNode(new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(),
+            threadPool, () -> new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(),
             emptySet(), Version.CURRENT));
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
-            public void connectToNodes(List<DiscoveryNode> addedNodes) {
+            public void connectToNodes(Iterable<DiscoveryNode> discoveryNodes) {
                 // skip
             }
 
             @Override
-            public void disconnectFromNodes(List<DiscoveryNode> removedNodes) {
+            public void disconnectFromNodesExcept(Iterable<DiscoveryNode> nodesToKeep) {
                 // skip
             }
         });
         timedClusterService.setClusterStatePublisher((event, ackListener) -> {
         });
+        timedClusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         timedClusterService.start();
         ClusterState state = timedClusterService.state();
         final DiscoveryNodes nodes = state.nodes();
@@ -284,17 +286,12 @@ public class ClusterServiceTests extends ESTestCase {
 
         taskFailed[0] = true;
         final CountDownLatch latch2 = new CountDownLatch(1);
-        nonMaster.submitStateUpdateTask("test", new ClusterStateUpdateTask() {
+        nonMaster.submitStateUpdateTask("test", new LocalClusterUpdateTask() {
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 taskFailed[0] = false;
                 latch2.countDown();
-                return currentState;
+                return unchanged();
             }
 
             @Override
@@ -324,14 +321,9 @@ public class ClusterServiceTests extends ESTestCase {
             ClusterStateTaskConfig.build(Priority.NORMAL),
             new ClusterStateTaskExecutor<Object>() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                public ClusterTasksResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
                     ClusterState newClusterState = ClusterState.builder(currentState).build();
-                    return BatchResult.builder().successes(tasks).build(newClusterState);
+                    return ClusterTasksResult.builder().successes(tasks).build(newClusterState);
                 }
 
                 @Override
@@ -367,19 +359,9 @@ public class ClusterServiceTests extends ESTestCase {
             ClusterStateTaskConfig.build(Priority.NORMAL),
             new ClusterStateTaskExecutor<Object>() {
                 @Override
-                public boolean runOnlyOnMaster() {
-                    return false;
-                }
-
-                @Override
-                public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                public ClusterTasksResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
                     ClusterState newClusterState = ClusterState.builder(currentState).build();
-                    return BatchResult.builder().successes(tasks).build(newClusterState);
-                }
-
-                @Override
-                public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
-                    assertNotNull(assertionRef.get());
+                    return ClusterTasksResult.builder().successes(tasks).build(newClusterState);
                 }
             },
             new ClusterStateTaskListener() {
@@ -419,16 +401,11 @@ public class ClusterServiceTests extends ESTestCase {
         class TaskExecutor implements ClusterStateTaskExecutor<String> {
 
             @Override
-            public BatchResult<String> execute(ClusterState currentState, List<String> tasks) throws Exception {
+            public ClusterTasksResult<String> execute(ClusterState currentState, List<String> tasks) throws Exception {
                 executionOrder.addAll(tasks); // do this first, so startedProcessing can be used as a notification that this is done.
                 startedProcessing.release(tasks.size());
                 allowProcessing.acquire(tasks.size());
-                return BatchResult.<String>builder().successes(tasks).build(ClusterState.builder(currentState).build());
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return ClusterTasksResult.<String>builder().successes(tasks).build(ClusterState.builder(currentState).build());
             }
         }
 
@@ -478,14 +455,9 @@ public class ClusterServiceTests extends ESTestCase {
             List<Integer> tasks = new ArrayList<>();
 
             @Override
-            public BatchResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
+            public ClusterTasksResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
                 this.tasks.addAll(tasks);
-                return BatchResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return ClusterTasksResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
             }
         }
 
@@ -573,7 +545,7 @@ public class ClusterServiceTests extends ESTestCase {
             (currentState, taskList) -> {
                 assertThat(taskList.size(), equalTo(tasks.size()));
                 assertThat(taskList.stream().collect(Collectors.toSet()), equalTo(tasks.keySet()));
-                return ClusterStateTaskExecutor.BatchResult.<Integer>builder().successes(taskList).build(currentState);
+                return ClusterStateTaskExecutor.ClusterTasksResult.<Integer>builder().successes(taskList).build(currentState);
             });
 
         latch.await();
@@ -632,12 +604,12 @@ public class ClusterServiceTests extends ESTestCase {
             private AtomicInteger batches = new AtomicInteger();
             private AtomicInteger published = new AtomicInteger();
 
-            public TaskExecutor(List<Set<Task>> taskGroups) {
+            TaskExecutor(List<Set<Task>> taskGroups) {
                 this.taskGroups = taskGroups;
             }
 
             @Override
-            public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
+            public ClusterTasksResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
                 for (Set<Task> expectedSet : taskGroups) {
                     long count = tasks.stream().filter(expectedSet::contains).count();
                     assertThat("batched set should be executed together or not at all. Expected " + expectedSet + "s. Executing " + tasks,
@@ -651,12 +623,7 @@ public class ClusterServiceTests extends ESTestCase {
                     batches.incrementAndGet();
                     semaphore.acquire();
                 }
-                return BatchResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return ClusterTasksResult.<Task>builder().successes(tasks).build(maybeUpdatedClusterState);
             }
 
             @Override
@@ -812,7 +779,7 @@ public class ClusterServiceTests extends ESTestCase {
             clusterService.submitStateUpdateTask("blocking", blockingTask);
 
             ClusterStateTaskExecutor<SimpleTask> executor = (currentState, tasks) ->
-                ClusterStateTaskExecutor.BatchResult.<SimpleTask>builder().successes(tasks).build(currentState);
+                ClusterStateTaskExecutor.ClusterTasksResult.<SimpleTask>builder().successes(tasks).build(currentState);
 
             SimpleTask task = new SimpleTask(1);
             ClusterStateTaskListener listener = new ClusterStateTaskListener() {
@@ -1087,20 +1054,20 @@ public class ClusterServiceTests extends ESTestCase {
     public void testDisconnectFromNewlyAddedNodesIfClusterStatePublishingFails() throws InterruptedException {
         TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
             "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool);
-        timedClusterService.setLocalNode(new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(),
+            threadPool, () -> new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(),
             emptySet(), Version.CURRENT));
-        Set<DiscoveryNode> currentNodes = Collections.synchronizedSet(new HashSet<>());
-        currentNodes.add(timedClusterService.localNode());
+        Set<DiscoveryNode> currentNodes = new HashSet<>();
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
-            public void connectToNodes(List<DiscoveryNode> addedNodes) {
-                currentNodes.addAll(addedNodes);
+            public void connectToNodes(Iterable<DiscoveryNode> discoveryNodes) {
+                discoveryNodes.forEach(currentNodes::add);
             }
 
             @Override
-            public void disconnectFromNodes(List<DiscoveryNode> removedNodes) {
-                currentNodes.removeAll(removedNodes);
+            public void disconnectFromNodesExcept(Iterable<DiscoveryNode> nodesToKeep) {
+                Set<DiscoveryNode> nodeSet = new HashSet<>();
+                nodesToKeep.iterator().forEachRemaining(nodeSet::add);
+                currentNodes.removeIf(node -> nodeSet.contains(node) == false);
             }
         });
         AtomicBoolean failToCommit = new AtomicBoolean();
@@ -1109,6 +1076,8 @@ public class ClusterServiceTests extends ESTestCase {
                 throw new Discovery.FailedToCommitClusterStateException("just to test this");
             }
         });
+        timedClusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         timedClusterService.start();
         ClusterState state = timedClusterService.state();
         final DiscoveryNodes nodes = state.nodes();
@@ -1254,7 +1223,7 @@ public class ClusterServiceTests extends ESTestCase {
     private static class BlockingTask extends ClusterStateUpdateTask implements Releasable {
         private final CountDownLatch latch = new CountDownLatch(1);
 
-        public BlockingTask(Priority priority) {
+        BlockingTask(Priority priority) {
             super(priority);
         }
 
@@ -1302,8 +1271,9 @@ public class ClusterServiceTests extends ESTestCase {
 
         public volatile Long currentTimeOverride = null;
 
-        public TimedClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-            super(settings, clusterSettings, threadPool);
+        TimedClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool,
+                                   Supplier<DiscoveryNode> localNodeSupplier) {
+            super(settings, clusterSettings, threadPool, localNodeSupplier);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/common/BooleansTests.java b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
index 176c4c75dc7..2e14159c07c 100644
--- a/core/src/test/java/org/elasticsearch/common/BooleansTests.java
+++ b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
@@ -28,59 +28,103 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class BooleansTests extends ESTestCase {
+    private static final String[] NON_BOOLEANS = new String[]{"11", "00", "sdfsdfsf", "F", "T", "on", "off", "yes", "no", "0", "1",
+        "True", "False"};
+    private static final String[] BOOLEANS = new String[]{"true", "false"};
+
     public void testIsBoolean() {
-        String[] booleans = new String[]{"true", "false", "on", "off", "yes", "no", "0", "1"};
-        String[] notBooleans = new String[]{"11", "00", "sdfsdfsf", "F", "T"};
+        for (String b : BOOLEANS) {
+            String t = "prefix" + b + "suffix";
+            assertTrue("failed to recognize [" + b + "] as boolean",
+                Booleans.isBoolean(t.toCharArray(), "prefix".length(), b.length()));
+            assertTrue("failed to recognize [" + b + "] as boolean", Booleans.isBoolean(b));
+        }
+    }
+
+    public void testIsNonBoolean() {
         assertThat(Booleans.isBoolean(null, 0, 1), is(false));
 
-        for (String b : booleans) {
-            String t = "prefix" + b + "suffix";
-            assertThat("failed to recognize [" + b + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), b.length()), Matchers.equalTo(true));
-        }
-
-        for (String nb : notBooleans) {
+        for (String nb : NON_BOOLEANS) {
             String t = "prefix" + nb + "suffix";
-            assertThat("recognized [" + nb + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()), Matchers.equalTo(false));
+            assertFalse("recognized [" + nb + "] as boolean", Booleans.isBoolean(t.toCharArray(), "prefix".length(), nb.length()));
+            assertFalse("recognized [" + nb + "] as boolean", Booleans.isBoolean(t));
+        }
+    }
+
+    public void testParseBooleanWithFallback() {
+        assertFalse(Booleans.parseBoolean(null, false));
+        assertTrue(Booleans.parseBoolean(null, true));
+        assertNull(Booleans.parseBoolean(null, null));
+        assertFalse(Booleans.parseBoolean(null, Boolean.FALSE));
+        assertTrue(Booleans.parseBoolean(null, Boolean.TRUE));
+
+        assertTrue(Booleans.parseBoolean("true", randomFrom(Boolean.TRUE, Boolean.FALSE, null)));
+        assertFalse(Booleans.parseBoolean("false", randomFrom(Boolean.TRUE, Boolean.FALSE, null)));
+    }
+
+    public void testParseNonBooleanWithFallback() {
+        for (String nonBoolean : NON_BOOLEANS) {
+            boolean defaultValue = randomFrom(Boolean.TRUE, Boolean.FALSE);
+
+            expectThrows(IllegalArgumentException.class,
+                () -> Booleans.parseBoolean(nonBoolean, defaultValue));
+            expectThrows(IllegalArgumentException.class,
+                () -> Booleans.parseBoolean(nonBoolean.toCharArray(), 0, nonBoolean.length(), defaultValue));
         }
     }
 
     public void testParseBoolean() {
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomBoolean()), is(true));
-        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomBoolean()), is(false));
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT), randomBoolean()), is(true));
-        assertThat(Booleans.parseBoolean(null, false), is(false));
-        assertThat(Booleans.parseBoolean(null, true), is(true));
-
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
-        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(false));
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
-        assertThat(Booleans.parseBoolean(null, Boolean.FALSE), is(false));
-        assertThat(Booleans.parseBoolean(null, Boolean.TRUE), is(true));
-        assertThat(Booleans.parseBoolean(null, null), nullValue());
-
-        char[] chars = randomFrom("true", "on", "yes", "1").toCharArray();
-        assertThat(Booleans.parseBoolean(chars, 0, chars.length, randomBoolean()), is(true));
-        chars = randomFrom("false", "off", "no", "0").toCharArray();
-        assertThat(Booleans.parseBoolean(chars,0, chars.length, randomBoolean()), is(false));
-        chars = randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT).toCharArray();
-        assertThat(Booleans.parseBoolean(chars,0, chars.length, randomBoolean()), is(true));
+        assertTrue(Booleans.parseBoolean("true"));
+        assertFalse(Booleans.parseBoolean("false"));
     }
 
-    public void testParseBooleanExact() {
-        assertThat(Booleans.parseBooleanExact(randomFrom("true", "on", "yes", "1")), is(true));
-        assertThat(Booleans.parseBooleanExact(randomFrom("false", "off", "no", "0")), is(false));
-        try {
-            Booleans.parseBooleanExact(randomFrom("fred", "foo", "barney", null));
-            fail("Expected exception while parsing invalid boolean value ");
-        } catch (Exception ex) {
-            assertTrue(ex instanceof IllegalArgumentException);
+    public void testParseNonBoolean() {
+        expectThrows(IllegalArgumentException.class, () -> Booleans.parseBoolean(null));
+        for (String nonBoolean : NON_BOOLEANS) {
+            expectThrows(IllegalArgumentException.class, () -> Booleans.parseBoolean(nonBoolean));
         }
     }
 
-    public void testIsExplicit() {
-        assertThat(Booleans.isExplicitFalse(randomFrom("true", "on", "yes", "1", "foo", null)), is(false));
-        assertThat(Booleans.isExplicitFalse(randomFrom("false", "off", "no", "0")), is(true));
-        assertThat(Booleans.isExplicitTrue(randomFrom("true", "on", "yes", "1")), is(true));
-        assertThat(Booleans.isExplicitTrue(randomFrom("false", "off", "no", "0", "foo", null)), is(false));
+    public void testIsBooleanLenient() {
+        String[] booleans = new String[]{"true", "false", "on", "off", "yes", "no", "0", "1"};
+        String[] notBooleans = new String[]{"11", "00", "sdfsdfsf", "F", "T"};
+        assertThat(Booleans.isBooleanLenient(null, 0, 1), is(false));
+
+        for (String b : booleans) {
+            String t = "prefix" + b + "suffix";
+            assertTrue("failed to recognize [" + b + "] as boolean",
+                Booleans.isBooleanLenient(t.toCharArray(), "prefix".length(), b.length()));
+        }
+
+        for (String nb : notBooleans) {
+            String t = "prefix" + nb + "suffix";
+            assertFalse("recognized [" + nb + "] as boolean",
+                Booleans.isBooleanLenient(t.toCharArray(), "prefix".length(), nb.length()));
+        }
+    }
+
+    public void testParseBooleanLenient() {
+        assertThat(Booleans.parseBooleanLenient(randomFrom("true", "on", "yes", "1"), randomBoolean()), is(true));
+        assertThat(Booleans.parseBooleanLenient(randomFrom("false", "off", "no", "0"), randomBoolean()), is(false));
+        assertThat(Booleans.parseBooleanLenient(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT), randomBoolean()), is(true));
+        assertThat(Booleans.parseBooleanLenient(null, false), is(false));
+        assertThat(Booleans.parseBooleanLenient(null, true), is(true));
+
+        assertThat(Booleans.parseBooleanLenient(
+            randomFrom("true", "on", "yes", "1"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
+        assertThat(Booleans.parseBooleanLenient(
+            randomFrom("false", "off", "no", "0"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(false));
+        assertThat(Booleans.parseBooleanLenient(
+            randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
+        assertThat(Booleans.parseBooleanLenient(null, Boolean.FALSE), is(false));
+        assertThat(Booleans.parseBooleanLenient(null, Boolean.TRUE), is(true));
+        assertThat(Booleans.parseBooleanLenient(null, null), nullValue());
+
+        char[] chars = randomFrom("true", "on", "yes", "1").toCharArray();
+        assertThat(Booleans.parseBooleanLenient(chars, 0, chars.length, randomBoolean()), is(true));
+        chars = randomFrom("false", "off", "no", "0").toCharArray();
+        assertThat(Booleans.parseBooleanLenient(chars,0, chars.length, randomBoolean()), is(false));
+        chars = randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT).toCharArray();
+        assertThat(Booleans.parseBooleanLenient(chars,0, chars.length, randomBoolean()), is(true));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
index c0cb3482b0e..aeb86c73470 100644
--- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
@@ -165,7 +165,7 @@ public class ChannelsTests extends ESTestCase {
 
         FileChannel delegate;
 
-        public MockFileChannel(FileChannel delegate) {
+        MockFileChannel(FileChannel delegate) {
             this.delegate = delegate;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java
index 74427281894..6369ce2d6ea 100644
--- a/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java
@@ -95,7 +95,7 @@ public class FieldMemoryStatsTests extends ESTestCase {
         ObjectLongHashMap<String> map = new ObjectLongHashMap<>();
         int keys = randomIntBetween(1, 1000);
         for (int i = 0; i < keys; i++) {
-            map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomPositiveLong());
+            map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomNonNegativeLong());
         }
         return new FieldMemoryStats(map);
     }
diff --git a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
index 1f348000ee4..ab70bd6ecae 100644
--- a/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ParseFieldTests.java
@@ -20,7 +20,6 @@ package org.elasticsearch.common;
 
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.sameInstance;
@@ -33,64 +32,31 @@ public class ParseFieldTests extends ESTestCase {
         String[] deprecated = new String[]{"barFoo", "bar_foo", "Foobar"};
         ParseField withDeprecations = field.withDeprecation(deprecated);
         assertThat(field, not(sameInstance(withDeprecations)));
-        assertThat(field.match(name, false), is(true));
-        assertThat(field.match("foo bar", false), is(false));
+        assertThat(field.match(name), is(true));
+        assertThat(field.match("foo bar"), is(false));
         for (String deprecatedName : deprecated) {
-            assertThat(field.match(deprecatedName, false), is(false));
+            assertThat(field.match(deprecatedName), is(false));
         }
 
-        assertThat(withDeprecations.match(name, false), is(true));
-        assertThat(withDeprecations.match("foo bar", false), is(false));
+        assertThat(withDeprecations.match(name), is(true));
+        assertThat(withDeprecations.match("foo bar"), is(false));
         for (String deprecatedName : deprecated) {
-            assertThat(withDeprecations.match(deprecatedName, false), is(true));
-        }
-
-        // now with strict mode
-        assertThat(field.match(name, true), is(true));
-        assertThat(field.match("foo bar", true), is(false));
-        for (String deprecatedName : deprecated) {
-            assertThat(field.match(deprecatedName, true), is(false));
-        }
-
-        assertThat(withDeprecations.match(name, true), is(true));
-        assertThat(withDeprecations.match("foo bar", true), is(false));
-        for (String deprecatedName : deprecated) {
-            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-                withDeprecations.match(deprecatedName, true);
-            });
-            assertThat(e.getMessage(), containsString("used, expected [foo_bar] instead"));
+            assertThat(withDeprecations.match(deprecatedName), is(true));
+            assertWarnings("Deprecated field [" + deprecatedName + "] used, expected [foo_bar] instead");
         }
     }
 
     public void testAllDeprecated() {
         String name = "like_text";
-
-        boolean withDeprecatedNames = randomBoolean();
         String[] deprecated = new String[]{"text", "same_as_text"};
-        String[] allValues;
-        if (withDeprecatedNames) {
-            String[] newArray = new String[1 + deprecated.length];
-            newArray[0] = name;
-            System.arraycopy(deprecated, 0, newArray, 1, deprecated.length);
-            allValues = newArray;
-        } else {
-            allValues = new String[] {name};
-        }
-
-        ParseField field;
-        if (withDeprecatedNames) {
-            field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
-        } else {
-            field = new ParseField(name).withAllDeprecated("like");
-        }
-
-        // strict mode off
-        assertThat(field.match(randomFrom(allValues), false), is(true));
-        assertThat(field.match("not a field name", false), is(false));
-
-        // now with strict mode
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> field.match(randomFrom(allValues), true));
-        assertThat(e.getMessage(), containsString(" used, replaced by [like]"));
+        ParseField field = new ParseField(name).withDeprecation(deprecated).withAllDeprecated("like");
+        assertFalse(field.match("not a field name"));
+        assertTrue(field.match("text"));
+        assertWarnings("Deprecated field [text] used, replaced by [like]");
+        assertTrue(field.match("same_as_text"));
+        assertWarnings("Deprecated field [same_as_text] used, replaced by [like]");
+        assertTrue(field.match("like_text"));
+        assertWarnings("Deprecated field [like_text] used, replaced by [like]");
     }
 
     public void testGetAllNamesIncludedDeprecated() {
diff --git a/core/src/test/java/org/elasticsearch/common/StringsTests.java b/core/src/test/java/org/elasticsearch/common/StringsTests.java
index 1b987d47796..28dd134adf0 100644
--- a/core/src/test/java/org/elasticsearch/common/StringsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/StringsTests.java
@@ -21,11 +21,9 @@ package org.elasticsearch.common;
 
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.IOException;
-
 import static org.hamcrest.Matchers.containsString;
 
 public class StringsTests extends ESTestCase {
@@ -58,21 +56,36 @@ public class StringsTests extends ESTestCase {
         assertEquals("", Strings.cleanTruncate("foo", 0));
     }
 
-    public void testEvilToString() {
-        ToXContent needsEnclosingObject = new ToXContent() {
-            @Override
-            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                return builder.field("ok", "here").field("catastrophe", "");
+    public void testToStringToXContent() {
+        final ToXContent toXContent;
+        final boolean error;
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                error = false;
+                toXContent  = (builder, params) -> builder.field("ok", "here").field("catastrophe", "");
+            } else {
+                error = true;
+                toXContent  = (builder, params) ->
+                        builder.startObject().field("ok", "here").field("catastrophe", "").endObject();
             }
-        };
-        String toString = Strings.toString(needsEnclosingObject);
-        assertThat(toString, containsString("Error building toString out of XContent"));
-        assertThat(toString, containsString("Can not write a field name, expecting a value"));
+        } else {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (ToXContentObject) (builder, params) ->
+                        builder.startObject().field("ok", "here").field("catastrophe", "").endObject();
+            } else {
+                error = true;
+                toXContent = (ToXContentObject) (builder, params) -> builder.field("ok", "here").field("catastrophe", "");
+            }
+        }
 
-        // We can salvage it!
-        toString = Strings.toString(needsEnclosingObject, true);
-        assertThat(toString, containsString("\"ok\":\"here\""));
-        assertThat(toString, containsString("\"catastrophe\":\"\""));
+        String toString = Strings.toString(toXContent);
+        if (error) {
+            assertThat(toString, containsString("Error building toString out of XContent"));
+        } else {
+            assertThat(toString, containsString("\"ok\":\"here\""));
+            assertThat(toString, containsString("\"catastrophe\":\"\""));
+        }
     }
 
     public void testSplitStringToSet() {
diff --git a/core/src/test/java/org/elasticsearch/common/UUIDTests.java b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
index f82e1a464d9..62ad47868a6 100644
--- a/core/src/test/java/org/elasticsearch/common/UUIDTests.java
+++ b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
@@ -58,7 +58,7 @@ public class UUIDTests extends ESTestCase {
         public Set<String> uuidSet = null;
         UUIDGenerator uuidSource;
 
-        public UUIDGenRunner(int count, UUIDGenerator uuidSource) {
+        UUIDGenRunner(int count, UUIDGenerator uuidSource) {
             this.count = count;
             this.uuidSource = uuidSource;
         }
diff --git a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
index f10a0da3029..ff93e04bf92 100644
--- a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
@@ -235,6 +235,9 @@ public class MemoryCircuitBreakerTests extends ESTestCase {
             fail("should never reach this");
         } catch (CircuitBreakingException cbe) {
             assertThat("breaker was tripped exactly twice", breaker.getTrippedCount(), equalTo(2L));
+
+            long newUsed = (long)(breaker.getUsed() * breaker.getOverhead());
+            assertThat(cbe.getMessage().contains("would be [" + newUsed + "/"), equalTo(true));
             assertThat(cbe.getMessage().contains("field [" + field + "]"), equalTo(true));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
index d7bb90f6c00..c2f27beb04e 100644
--- a/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
+++ b/core/src/test/java/org/elasticsearch/common/cache/CacheTests.java
@@ -416,7 +416,7 @@ public class CacheTests extends ESTestCase {
             private String value;
             private long weight;
 
-            public Value(String value, long weight) {
+            Value(String value, long weight) {
                 this.value = value;
                 this.weight = weight;
             }
@@ -552,7 +552,7 @@ public class CacheTests extends ESTestCase {
         class Key {
             private final int key;
 
-            public Key(int key) {
+            Key(int key) {
                 this.key = key;
             }
 
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
index 416299f8e7e..6624a91a6c2 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.geo.Rectangle;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.unit.DistanceUnit;
@@ -38,12 +39,10 @@ public class GeoDistanceTests extends ESTestCase {
     public void testGeoDistanceSerialization() throws IOException  {
         // make sure that ordinals don't change, because we rely on then in serialization
         assertThat(GeoDistance.PLANE.ordinal(), equalTo(0));
-        assertThat(GeoDistance.FACTOR.ordinal(), equalTo(1));
-        assertThat(GeoDistance.ARC.ordinal(), equalTo(2));
-        assertThat(GeoDistance.SLOPPY_ARC.ordinal(), equalTo(3));
-        assertThat(GeoDistance.values().length, equalTo(4));
+        assertThat(GeoDistance.ARC.ordinal(), equalTo(1));
+        assertThat(GeoDistance.values().length, equalTo(2));
 
-        GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.FACTOR, GeoDistance.ARC, GeoDistance.SLOPPY_ARC);
+        GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.ARC);
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             geoDistance.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {;
@@ -70,16 +69,24 @@ public class GeoDistanceTests extends ESTestCase {
 
     public void testDistanceCheck() {
         // Note, is within is an approximation, so, even though 0.52 is outside 50mi, we still get "true"
-        GeoDistance.DistanceBoundingCheck check = GeoDistance.distanceBoundingCheck(0, 0, 50, DistanceUnit.MILES);
-        assertThat(check.isWithin(0.5, 0.5), equalTo(true));
-        assertThat(check.isWithin(0.52, 0.52), equalTo(true));
-        assertThat(check.isWithin(1, 1), equalTo(false));
+        double radius = DistanceUnit.convert(50, DistanceUnit.MILES, DistanceUnit.METERS);
+        Rectangle box = Rectangle.fromPointDistance(0, 0, radius);
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0.5, 0.5), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0.52, 0.52), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 1, 1), equalTo(false));
 
-        check = GeoDistance.distanceBoundingCheck(0, 179, 200, DistanceUnit.MILES);
-        assertThat(check.isWithin(0, -179), equalTo(true));
-        assertThat(check.isWithin(0, -178), equalTo(false));
+        radius = DistanceUnit.convert(200, DistanceUnit.MILES, DistanceUnit.METERS);
+        box = Rectangle.fromPointDistance(0, 179, radius);
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0, -179), equalTo(true));
+        assertThat(GeoUtils.rectangleContainsPoint(box, 0, -178), equalTo(false));
     }
 
+    /**
+     * The old plane calculation in 1.x/2.x incorrectly computed the plane distance in decimal degrees. This test is
+     * well intended but bogus. todo: fix w/ new plane distance calculation
+     * note: plane distance error varies by latitude so the test will need to correctly estimate expected error
+     */
+    @AwaitsFix(bugUrl = "old plane calculation incorrectly computed everything in degrees. fix this bogus test")
     public void testArcDistanceVsPlaneInEllipsis() {
         GeoPoint centre = new GeoPoint(48.8534100, 2.3488000);
         GeoPoint northernPoint = new GeoPoint(48.8801108681, 2.35152032666);
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index d1340af0b22..4a02a1f345b 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.common.io.stream;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.UnicodeUtil;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -33,6 +33,7 @@ import org.joda.time.DateTimeZone;
 import java.io.EOFException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -530,7 +531,7 @@ public class BytesStreamsTests extends ESTestCase {
             this.field2 = field2;
         }
 
-        public TestNamedWriteable(StreamInput in) throws IOException {
+        TestNamedWriteable(StreamInput in) throws IOException {
             field1 = in.readString();
             field2 = in.readString();
         }
@@ -612,9 +613,9 @@ public class BytesStreamsTests extends ESTestCase {
 
         private boolean value;
 
-        public TestStreamable() { }
+        TestStreamable() { }
 
-        public TestStreamable(boolean value) {
+        TestStreamable(boolean value) {
             this.value = value;
         }
 
@@ -778,4 +779,37 @@ public class BytesStreamsTests extends ESTestCase {
             }
         }
     }
+
+    public void testVInt() throws IOException {
+        final int value = randomInt();
+        BytesStreamOutput output = new BytesStreamOutput();
+        output.writeVInt(value);
+        StreamInput input = output.bytes().streamInput();
+        assertEquals(value, input.readVInt());
+    }
+
+    public void testVLong() throws IOException {
+        final long value = randomLong();
+        {
+            // Read works for positive and negative numbers
+            BytesStreamOutput output = new BytesStreamOutput();
+            output.writeVLongNoCheck(value); // Use NoCheck variant so we can write negative numbers
+            StreamInput input = output.bytes().streamInput();
+            assertEquals(value, input.readVLong());
+        }
+        if (value < 0) {
+            // Write doesn't work for negative numbers
+            BytesStreamOutput output = new BytesStreamOutput();
+            Exception e = expectThrows(IllegalStateException.class, () -> output.writeVLong(value));
+            assertEquals("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + value + "]", e.getMessage());
+        }
+
+        assertTrue("If we're not compatible with 5.1.1 we can drop the assertion below",
+                Version.CURRENT.minimumCompatibilityVersion().onOrBefore(Version.V_5_1_1_UNRELEASED));
+        /* Read -1 as serialized by a version of Elasticsearch that supported writing negative numbers with writeVLong. Note that this
+         * should be the same test as the first case (when value is negative) but we've kept some bytes so no matter what we do to
+         * writeVLong in the future we can be sure we can read bytes as written by Elasticsearch before 5.1.2 */
+        StreamInput in = new BytesArray(Base64.getDecoder().decode("////////////AQAAAAAAAA==")).streamInput();
+        assertEquals(-1, in.readVLong());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
index 8442e793a92..682615d6f1a 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/NamedWriteableRegistryTests.java
@@ -28,7 +28,7 @@ import org.elasticsearch.test.ESTestCase;
 public class NamedWriteableRegistryTests extends ESTestCase {
 
     private static class DummyNamedWriteable implements NamedWriteable {
-        public DummyNamedWriteable(StreamInput in) {}
+        DummyNamedWriteable(StreamInput in) {}
         @Override
         public String getWriteableName() {
             return "test";
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index d40a2b7fcb4..9d885fe131c 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -216,11 +216,11 @@ public class StreamTests extends ESTestCase {
     static final class WriteableString implements Writeable {
         final String string;
 
-        public WriteableString(String string) {
+        WriteableString(String string) {
             this.string = string;
         }
 
-        public WriteableString(StreamInput in) throws IOException {
+        WriteableString(StreamInput in) throws IOException {
             this(in.readString());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
index f75e73ced2c..d0e1b807baf 100644
--- a/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
@@ -41,6 +41,12 @@ public class DeprecationLoggerTests extends ESTestCase {
 
     private final DeprecationLogger logger = new DeprecationLogger(Loggers.getLogger(getClass()));
 
+    @Override
+    protected boolean enableWarningsCheck() {
+        //this is a low level test for the deprecation logger, setup and checks are done manually
+        return false;
+    }
+
     public void testAddsHeaderWithThreadContext() throws IOException {
         String msg = "A simple message [{}]";
         String param = randomAsciiOfLengthBetween(1, 5);
@@ -54,7 +60,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
 
             assertEquals(1, responseHeaders.size());
-            assertEquals(formatted, responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER).get(0));
+            assertEquals(formatted, responseHeaders.get(DeprecationLogger.WARNING_HEADER).get(0));
         }
     }
 
@@ -74,7 +80,7 @@ public class DeprecationLoggerTests extends ESTestCase {
 
             assertEquals(1, responseHeaders.size());
 
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            List<String> responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             assertEquals(2, responses.size());
             assertEquals(formatted, responses.get(0));
@@ -93,7 +99,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             logger.deprecated(expected);
 
             Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
-            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            List<String> responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             // ensure it works (note: concurrent tests may be adding to it, but in different threads, so it should have no impact)
             assertThat(responses, hasSize(atLeast(1)));
@@ -104,7 +110,7 @@ public class DeprecationLoggerTests extends ESTestCase {
             logger.deprecated(unexpected);
 
             responseHeaders = threadContext.getResponseHeaders();
-            responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+            responses = responseHeaders.get(DeprecationLogger.WARNING_HEADER);
 
             assertThat(responses, hasSize(atLeast(1)));
             assertThat(responses, hasItem(expected));
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 3e29068e672..7c73520173f 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -27,8 +27,9 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -36,9 +37,12 @@ import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
+import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.transport.TransportRequestHandler;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -46,12 +50,27 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
 public class NetworkModuleTests extends ModuleTestCase {
+    private ThreadPool threadPool;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(NetworkModuleTests.class.getName());
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+    }
 
     static class FakeHttpTransport extends AbstractLifecycleComponent implements HttpServerTransport {
-        public FakeHttpTransport() {
+        FakeHttpTransport() {
             super(null);
         }
         @Override
@@ -72,13 +91,11 @@ public class NetworkModuleTests extends ModuleTestCase {
         public HttpStats stats() {
             return null;
         }
-        @Override
-        public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {}
     }
 
 
     static class FakeRestHandler extends BaseRestHandler {
-        public FakeRestHandler() {
+        FakeRestHandler() {
             super(null);
         }
         @Override
@@ -86,7 +103,7 @@ public class NetworkModuleTests extends ModuleTestCase {
     }
 
     static class FakeCatRestHandler extends AbstractCatAction {
-        public FakeCatRestHandler() {
+        FakeCatRestHandler() {
             super(null);
         }
         @Override
@@ -137,7 +154,9 @@ public class NetworkModuleTests extends ModuleTestCase {
                                                                                 BigArrays bigArrays,
                                                                                 CircuitBreakerService circuitBreakerService,
                                                                                 NamedWriteableRegistry namedWriteableRegistry,
-                                                                                NetworkService networkService) {
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
                 return Collections.singletonMap("custom", custom);
             }
         });
@@ -176,7 +195,9 @@ public class NetworkModuleTests extends ModuleTestCase {
                                                                                 BigArrays bigArrays,
                                                                                 CircuitBreakerService circuitBreakerService,
                                                                                 NamedWriteableRegistry namedWriteableRegistry,
-                                                                                NetworkService networkService) {
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
                 Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
                 supplierMap.put("custom", custom);
                 supplierMap.put("default_custom", def);
@@ -208,7 +229,9 @@ public class NetworkModuleTests extends ModuleTestCase {
                                                                                 BigArrays bigArrays,
                                                                                 CircuitBreakerService circuitBreakerService,
                                                                                 NamedWriteableRegistry namedWriteableRegistry,
-                                                                                NetworkService networkService) {
+                                                                                NamedXContentRegistry xContentRegistry,
+                                                                                NetworkService networkService,
+                                                                                HttpServerTransport.Dispatcher requestDispatcher) {
                 Map<String, Supplier<HttpServerTransport>> supplierMap = new HashMap<>();
                 supplierMap.put("custom", custom);
                 supplierMap.put("default_custom", def);
@@ -224,17 +247,37 @@ public class NetworkModuleTests extends ModuleTestCase {
         Settings settings = Settings.builder()
             .put(NetworkModule.HTTP_ENABLED.getKey(), false)
             .put(NetworkModule.TRANSPORT_TYPE_KEY, "local").build();
+        AtomicInteger called = new AtomicInteger(0);
 
         TransportInterceptor interceptor = new TransportInterceptor() {
+            @Override
+            public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                            boolean forceExecution,
+                                                                                            TransportRequestHandler<T> actualHandler) {
+                called.incrementAndGet();
+                if ("foo/bar/boom".equals(action)) {
+                    assertTrue(forceExecution);
+                } else {
+                    assertFalse(forceExecution);
+                }
+                return actualHandler;
+            }
         };
         NetworkModule module = newNetworkModule(settings, false, new NetworkPlugin() {
                 @Override
-                public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+                public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                           ThreadContext threadContext) {
+                    assertNotNull(threadContext);
                     return Collections.singletonList(interceptor);
                 }
             });
 
         TransportInterceptor transportInterceptor = module.getTransportInterceptor();
+        assertEquals(0, called.get());
+        transportInterceptor.interceptHandler("foo/bar/boom", null, true, null);
+        assertEquals(1, called.get());
+        transportInterceptor.interceptHandler("foo/baz/boom", null, false, null);
+        assertEquals(2, called.get());
         assertTrue(transportInterceptor instanceof  NetworkModule.CompositeTransportInterceptor);
         assertEquals(((NetworkModule.CompositeTransportInterceptor)transportInterceptor).transportInterceptors.size(), 1);
         assertSame(((NetworkModule.CompositeTransportInterceptor)transportInterceptor).transportInterceptors.get(0), interceptor);
@@ -242,7 +285,9 @@ public class NetworkModuleTests extends ModuleTestCase {
         NullPointerException nullPointerException = expectThrows(NullPointerException.class, () -> {
             newNetworkModule(settings, false, new NetworkPlugin() {
                 @Override
-                public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+                public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                           ThreadContext threadContext) {
+                    assertNotNull(threadContext);
                     return Collections.singletonList(null);
                 }
             });
@@ -252,6 +297,7 @@ public class NetworkModuleTests extends ModuleTestCase {
     }
 
     private NetworkModule newNetworkModule(Settings settings, boolean transportClient, NetworkPlugin... plugins) {
-        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), null, null, null, null, null);
+        return new NetworkModule(settings, transportClient, Arrays.asList(plugins), threadPool, null, null, null, xContentRegistry(), null,
+            (a, b, c) -> {});
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
new file mode 100644
index 00000000000..ef732c1e29c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/AddStringKeyStoreCommandTests.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
+    InputStream input;
+
+    @Override
+    protected Command newCommand() {
+        return new AddStringKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+            @Override
+            InputStream getStdin() {
+                return input;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testOverwritePromptDefault() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("");
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitNo() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("n"); // explicit no
+        execute("foo");
+        assertSecureString("foo", "bar");
+    }
+
+    public void testOverwritePromptExplicitYes() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addTextInput("y");
+        terminal.addSecretInput("newvalue");
+        execute("foo");
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceShort() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("newvalue");
+        execute("-f", "foo"); // force
+        assertSecureString("foo", "newvalue");
+    }
+
+    public void testOverwriteForceLong() throws Exception {
+        createKeystore("", "foo", "bar");
+        terminal.addSecretInput("and yet another secret value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "and yet another secret value");
+    }
+
+    public void testForceNonExistent() throws Exception {
+        createKeystore("");
+        terminal.addSecretInput("value");
+        execute("--force", "foo"); // force
+        assertSecureString("foo", "value");
+    }
+
+    public void testPromptForValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("secret value");
+        execute("foo");
+        assertSecureString("foo", "secret value");
+    }
+
+    public void testStdinShort() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 1");
+        execute("-x", "foo");
+        assertSecureString("foo", "secret value 1");
+    }
+
+    public void testStdinLong() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        setInput("secret value 2");
+        execute("--stdin", "foo");
+        assertSecureString("foo", "secret value 2");
+    }
+
+    public void testNonAsciiValue() throws Exception {
+        KeyStoreWrapper.create(new char[0]).save(env.configFile());
+        terminal.addSecretInput("non-äsčîï");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertEquals("String value must contain only ASCII", e.getMessage());
+    }
+
+    public void testNpe() throws Exception {
+        createKeystore("");
+        terminal.addTextInput("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("The setting name can not be null"));
+    }
+
+    void setInput(String inputStr) {
+        input = new ByteArrayInputStream(inputStr.getBytes(StandardCharsets.UTF_8));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java
new file mode 100644
index 00000000000..8584d4d1555
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/CreateKeyStoreCommandTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.env.Environment;
+
+public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new CreateKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testPosix() throws Exception {
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testNotPosix() throws Exception {
+        setupEnv(false);
+        execute();
+        Path configDir = env.configFile();
+        assertNotNull(KeyStoreWrapper.load(configDir));
+    }
+
+    public void testOverwrite() throws Exception {
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8);
+        Files.write(keystoreFile, content);
+
+        terminal.addTextInput(""); // default is no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("n"); // explicit no
+        execute();
+        assertArrayEquals(content, Files.readAllBytes(keystoreFile));
+
+        terminal.addTextInput("y");
+        execute();
+        assertNotNull(KeyStoreWrapper.load(env.configFile()));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java
new file mode 100644
index 00000000000..1e4d24a344e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/KeyStoreCommandTestCase.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.jimfs.Configuration;
+import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.cli.CommandTestCase;
+import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.env.Environment;
+import org.junit.After;
+import org.junit.Before;
+
+/**
+ * Base test case for manipulating the ES keystore.
+ */
+@LuceneTestCase.SuppressFileSystems("*") // we do our own mocking
+public abstract class KeyStoreCommandTestCase extends CommandTestCase {
+
+    Environment env;
+
+    List<FileSystem> fileSystems = new ArrayList<>();
+
+    @After
+    public void closeMockFileSystems() throws IOException {
+        IOUtils.close(fileSystems);
+    }
+
+    @Before
+    public void setupEnv() throws IOException {
+        setupEnv(true); // default to posix, but tests may call setupEnv(false) to overwrite
+    }
+
+    void setupEnv(boolean posix) throws IOException {
+        final Configuration configuration;
+        if (posix) {
+            configuration = Configuration.unix().toBuilder().setAttributeViews("basic", "owner", "posix", "unix").build();
+        } else {
+            configuration = Configuration.unix();
+        }
+        FileSystem fs = Jimfs.newFileSystem(configuration);
+        fileSystems.add(fs);
+        PathUtilsForTesting.installMock(fs); // restored by restoreFileSystem in ESTestCase
+        Path home = fs.getPath("/", "test-home");
+        Files.createDirectories(home.resolve("config"));
+        env = new Environment(Settings.builder().put("path.home", home).build());
+    }
+
+    KeyStoreWrapper createKeystore(String password, String... settings) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.create(password.toCharArray());
+        assertEquals(0, settings.length % 2);
+        for (int i = 0; i < settings.length; i += 2) {
+            keystore.setString(settings[i], settings[i + 1].toCharArray());
+        }
+        keystore.save(env.configFile());
+        return keystore;
+    }
+
+    KeyStoreWrapper loadKeystore(String password) throws Exception {
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        keystore.decrypt(password.toCharArray());
+        return keystore;
+    }
+
+    void assertSecureString(String setting, String value) throws Exception {
+        assertSecureString(loadKeystore(""), setting, value);
+    }
+
+    void assertSecureString(KeyStoreWrapper keystore, String setting, String value) throws Exception {
+        assertEquals(value, keystore.getString(setting).toString());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java
new file mode 100644
index 00000000000..1a8bdfc077d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/ListKeyStoreCommandTests.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.util.Map;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class ListKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new ListKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testEmpty() throws Exception {
+        createKeystore("");
+        execute();
+        assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute();
+        assertEquals("foo\n", terminal.getOutput());
+    }
+
+    public void testMultiple() throws Exception {
+        createKeystore("", "foo", "1", "baz", "2", "bar", "3");
+        execute();
+        assertEquals("bar\nbaz\nfoo\n", terminal.getOutput()); // sorted
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java b/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java
new file mode 100644
index 00000000000..e1d526c88be
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/settings/RemoveSettingKeyStoreCommandTests.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import javax.crypto.SecretKeyFactory;
+import java.security.Provider;
+import java.security.Security;
+import java.util.Map;
+import java.util.Set;
+
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.env.Environment;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class RemoveSettingKeyStoreCommandTests extends KeyStoreCommandTestCase {
+
+    @Override
+    protected Command newCommand() {
+        return new RemoveSettingKeyStoreCommand() {
+            @Override
+            protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                return env;
+            }
+        };
+    }
+
+    public void testMissing() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
+        assertThat(e.getMessage(), containsString("keystore not found"));
+    }
+
+    public void testNoSettings() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, this::execute);
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertThat(e.getMessage(), containsString("Must supply at least one setting"));
+    }
+
+    public void testNonExistentSetting() throws Exception {
+        createKeystore("");
+        UserException e = expectThrows(UserException.class, () -> execute("foo"));
+        assertEquals(ExitCodes.CONFIG, e.exitCode);
+        assertThat(e.getMessage(), containsString("[foo] does not exist"));
+    }
+
+    public void testOne() throws Exception {
+        createKeystore("", "foo", "bar");
+        execute("foo");
+        assertFalse(loadKeystore("").getSettings().contains("foo"));
+    }
+
+    public void testMany() throws Exception {
+        createKeystore("", "foo", "1", "bar", "2", "baz", "3");
+        execute("foo", "baz");
+        Set<String> settings = loadKeystore("").getSettings();
+        assertFalse(settings.contains("foo"));
+        assertFalse(settings.contains("baz"));
+        assertTrue(settings.contains("bar"));
+        assertEquals(1, settings.size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index 851ea26a19d..0606584d0e1 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -33,12 +33,15 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -126,6 +129,52 @@ public class ScopedSettingsTests extends ESTestCase {
         assertEquals(0, consumer2.get());
     }
 
+    public void testAddConsumerAffix() {
+        Setting.AffixSetting<Integer> intSetting = Setting.affixKeySetting("foo.", "bar",
+            (k) ->  Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope));
+        Setting.AffixSetting<List<Integer>> listSetting = Setting.affixKeySetting("foo.", "list",
+            (k) -> Setting.listSetting(k, Arrays.asList("1"), Integer::parseInt, Property.Dynamic, Property.NodeScope));
+        AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY,new HashSet<>(Arrays.asList(intSetting, listSetting)));
+        Map<String, List<Integer>> listResults = new HashMap<>();
+        Map<String, Integer> intResults = new HashMap<>();
+
+        BiConsumer<String, Integer> intConsumer = intResults::put;
+        BiConsumer<String, List<Integer>> listConsumer = listResults::put;
+
+        service.addAffixUpdateConsumer(listSetting, listConsumer, (s, k) -> {});
+        service.addAffixUpdateConsumer(intSetting, intConsumer, (s, k) -> {});
+        assertEquals(0, listResults.size());
+        assertEquals(0, intResults.size());
+        service.applySettings(Settings.builder()
+            .put("foo.test.bar", 2)
+            .put("foo.test_1.bar", 7)
+            .putArray("foo.test_list.list", "16", "17")
+            .putArray("foo.test_list_1.list", "18", "19", "20")
+            .build());
+        assertEquals(2, intResults.get("test").intValue());
+        assertEquals(7, intResults.get("test_1").intValue());
+        assertEquals(Arrays.asList(16, 17), listResults.get("test_list"));
+        assertEquals(Arrays.asList(18, 19, 20), listResults.get("test_list_1"));
+        assertEquals(2, listResults.size());
+        assertEquals(2, intResults.size());
+
+        listResults.clear();
+        intResults.clear();
+
+        service.applySettings(Settings.builder()
+            .put("foo.test.bar", 2)
+            .put("foo.test_1.bar", 8)
+            .putArray("foo.test_list.list", "16", "17")
+            .putNull("foo.test_list_1.list")
+            .build());
+        assertNull("test wasn't changed", intResults.get("test"));
+        assertEquals(8, intResults.get("test_1").intValue());
+        assertNull("test_list wasn't changed", listResults.get("test_list"));
+        assertEquals(Arrays.asList(1), listResults.get("test_list_1")); // reset to default
+        assertEquals(1, listResults.size());
+        assertEquals(1, intResults.size());
+    }
+
     public void testApply() {
         Setting<Integer> testSetting = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
         Setting<Integer> testSetting2 = Setting.intSetting("foo.bar.baz", 1, Property.Dynamic, Property.NodeScope);
@@ -214,27 +263,28 @@ public class ScopedSettingsTests extends ESTestCase {
         Setting<Integer> fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
         Setting<Integer> fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
         Setting<Settings> someGroup = Setting.groupSetting("some.group.", Property.Dynamic, Property.NodeScope);
-        Setting<Boolean> someAffix = Setting.affixKeySetting("some.prefix.", "somekey", "true", Boolean::parseBoolean, Property.NodeScope);
+        Setting<Boolean> someAffix = Setting.affixKeySetting("some.prefix.", "somekey", (key) -> Setting.boolSetting(key, true,
+            Property.NodeScope));
         Setting<List<String>> foorBarQuux =
                 Setting.listSetting("foo.bar.quux", Arrays.asList("a", "b", "c"), Function.identity(), Property.NodeScope);
         ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fooBar, fooBarBaz, foorBarQuux,
             someGroup, someAffix)));
         Settings diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.EMPTY);
-        assertEquals(4, diff.getAsMap().size()); // 4 since foo.bar.quux has 3 values essentially
+        assertEquals(4, diff.size()); // 4 since foo.bar.quux has 3 values essentially
         assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
         assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
 
         diff = settings.diff(
                 Settings.builder().put("foo.bar", 5).build(),
                 Settings.builder().put("foo.bar.baz", 17).putArray("foo.bar.quux", "d", "e", "f").build());
-        assertEquals(4, diff.getAsMap().size()); // 4 since foo.bar.quux has 3 values essentially
+        assertEquals(4, diff.size()); // 4 since foo.bar.quux has 3 values essentially
         assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(17));
         assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"d", "e", "f"});
 
         diff = settings.diff(
             Settings.builder().put("some.group.foo", 5).build(),
             Settings.builder().put("some.group.foobar", 17, "some.group.foo", 25).build());
-        assertEquals(6, diff.getAsMap().size()); // 6 since foo.bar.quux has 3 values essentially
+        assertEquals(6, diff.size()); // 6 since foo.bar.quux has 3 values essentially
         assertThat(diff.getAsInt("some.group.foobar", null), equalTo(17));
         assertNull(diff.get("some.group.foo"));
         assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
@@ -245,7 +295,7 @@ public class ScopedSettingsTests extends ESTestCase {
             Settings.builder().put("some.prefix.foo.somekey", 5).build(),
             Settings.builder().put("some.prefix.foobar.somekey", 17,
                 "some.prefix.foo.somekey", 18).build());
-        assertEquals(6, diff.getAsMap().size()); // 6 since foo.bar.quux has 3 values essentially
+        assertEquals(6, diff.size()); // 6 since foo.bar.quux has 3 values essentially
         assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
         assertNull(diff.get("some.prefix.foo.somekey"));
         assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"a", "b", "c"});
@@ -253,6 +303,65 @@ public class ScopedSettingsTests extends ESTestCase {
         assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
     }
 
+    public void testDiffWithAffixAndComplexMatcher() {
+        Setting<Integer> fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
+        Setting<Integer> fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);
+        Setting<Settings> someGroup = Setting.groupSetting("some.group.", Property.Dynamic, Property.NodeScope);
+        Setting<Boolean> someAffix = Setting.affixKeySetting("some.prefix.", "somekey", (key) -> Setting.boolSetting(key, true,
+            Property.NodeScope));
+        Setting<List<String>> foorBarQuux = Setting.affixKeySetting("foo.", "quux",
+            (key) -> Setting.listSetting(key,  Arrays.asList("a", "b", "c"), Function.identity(), Property.NodeScope));
+        ClusterSettings settings = new ClusterSettings(Settings.EMPTY, new HashSet<>(Arrays.asList(fooBar, fooBarBaz, foorBarQuux,
+            someGroup, someAffix)));
+        Settings diff = settings.diff(Settings.builder().put("foo.bar", 5).build(), Settings.EMPTY);
+        assertEquals(1, diff.size());
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+
+        diff = settings.diff(
+            Settings.builder().put("foo.bar", 5).build(),
+            Settings.builder().put("foo.bar.baz", 17).putArray("foo.bar.quux", "d", "e", "f").build());
+        assertEquals(4, diff.size());
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(17));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"d", "e", "f"});
+
+        diff = settings.diff(
+            Settings.builder().put("some.group.foo", 5).build(),
+            Settings.builder().put("some.group.foobar", 17, "some.group.foo", 25).build());
+        assertEquals(3, diff.size());
+        assertThat(diff.getAsInt("some.group.foobar", null), equalTo(17));
+        assertNull(diff.get("some.group.foo"));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+
+        diff = settings.diff(
+            Settings.builder().put("some.prefix.foo.somekey", 5).build(),
+            Settings.builder().put("some.prefix.foobar.somekey", 17,
+                "some.prefix.foo.somekey", 18).build());
+        assertEquals(3, diff.size());
+        assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
+        assertNull(diff.get("some.prefix.foo.somekey"));
+        assertNull(diff.getAsArray("foo.bar.quux", null)); // affix settings don't know their concrete keys
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+
+        diff = settings.diff(
+            Settings.builder().put("some.prefix.foo.somekey", 5).build(),
+            Settings.builder().put("some.prefix.foobar.somekey", 17,
+                "some.prefix.foo.somekey", 18)
+            .putArray("foo.bar.quux", "x", "y", "z")
+            .putArray("foo.baz.quux", "d", "e", "f")
+                .build());
+        assertEquals(9, diff.size());
+        assertThat(diff.getAsInt("some.prefix.foobar.somekey", null), equalTo(17));
+        assertNull(diff.get("some.prefix.foo.somekey"));
+        assertArrayEquals(diff.getAsArray("foo.bar.quux", null), new String[] {"x", "y", "z"});
+        assertArrayEquals(diff.getAsArray("foo.baz.quux", null), new String[] {"d", "e", "f"});
+        assertThat(diff.getAsInt("foo.bar.baz", null), equalTo(1));
+        assertThat(diff.getAsInt("foo.bar", null), equalTo(1));
+    }
+
     public void testUpdateTracer() {
         ClusterSettings settings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         AtomicReference<List<String>> ref = new AtomicReference<>();
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 2bd5dea3c10..24b48cbf368 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -31,6 +31,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -125,7 +127,7 @@ public class SettingTests extends ESTestCase {
             settingUpdater.apply(build, Settings.EMPTY);
             fail("not a boolean");
         } catch (IllegalArgumentException ex) {
-            assertEquals("Failed to parse value [I am not a boolean] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]",
+            assertEquals("Failed to parse value [I am not a boolean] as only [true] or [false] are allowed.",
                     ex.getMessage());
         }
     }
@@ -426,7 +428,7 @@ public class SettingTests extends ESTestCase {
     }
 
     public void testDynamicKeySetting() {
-        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", "false", Boolean::parseBoolean, Property.NodeScope);
+        Setting<Boolean> setting = Setting.prefixKeySetting("foo.", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar"));
         assertFalse(setting.match("foo"));
@@ -444,11 +446,11 @@ public class SettingTests extends ESTestCase {
 
     public void testAffixKeySetting() {
         Setting<Boolean> setting =
-            Setting.affixKeySetting("foo.", "enable", "false", Boolean::parseBoolean, Property.NodeScope);
+            Setting.affixKeySetting("foo.", "enable", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
         assertTrue(setting.hasComplexMatcher());
         assertTrue(setting.match("foo.bar.enable"));
         assertTrue(setting.match("foo.baz.enable"));
-        assertTrue(setting.match("foo.bar.baz.enable"));
+        assertFalse(setting.match("foo.bar.baz.enable"));
         assertFalse(setting.match("foo.bar"));
         assertFalse(setting.match("foo.bar.baz.enabled"));
         assertFalse(setting.match("foo"));
@@ -459,9 +461,48 @@ public class SettingTests extends ESTestCase {
         IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> setting.getConcreteSetting("foo"));
         assertEquals("key [foo] must match [foo.*.enable] but didn't.", exc.getMessage());
 
-        exc = expectThrows(IllegalArgumentException.class, () -> Setting.affixKeySetting("foo", "enable", "false",
-            Boolean::parseBoolean, Property.NodeScope));
+        exc = expectThrows(IllegalArgumentException.class, () -> Setting.affixKeySetting("foo", "enable",
+            (key) -> Setting.boolSetting(key, false, Property.NodeScope)));
         assertEquals("prefix must end with a '.'", exc.getMessage());
+
+        Setting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
+
+        assertTrue(listAffixSetting.hasComplexMatcher());
+        assertTrue(listAffixSetting.match("foo.test.bar"));
+        assertTrue(listAffixSetting.match("foo.test_1.bar"));
+        assertFalse(listAffixSetting.match("foo.buzz.baz.bar"));
+        assertFalse(listAffixSetting.match("foo.bar"));
+        assertFalse(listAffixSetting.match("foo.baz"));
+        assertFalse(listAffixSetting.match("foo"));
+    }
+
+    public void testGetAllConcreteSettings() {
+        Setting.AffixSetting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.emptyList(), Function.identity(), Property.NodeScope));
+
+        Settings settings = Settings.builder()
+            .putArray("foo.1.bar", "1", "2")
+            .putArray("foo.2.bar", "3", "4", "5")
+            .putArray("foo.bar", "6")
+            .putArray("some.other", "6")
+            .putArray("foo.3.bar", "6")
+            .build();
+        Stream<Setting<List<String>>> allConcreteSettings = listAffixSetting.getAllConcreteSettings(settings);
+        Map<String, List<String>> collect = allConcreteSettings.collect(Collectors.toMap(Setting::getKey, (s) -> s.get(settings)));
+        assertEquals(3, collect.size());
+        assertEquals(Arrays.asList("1", "2"), collect.get("foo.1.bar"));
+        assertEquals(Arrays.asList("3", "4", "5"), collect.get("foo.2.bar"));
+        assertEquals(Arrays.asList("6"), collect.get("foo.3.bar"));
+    }
+
+    public void testAffixSettingsFailOnGet() {
+        Setting.AffixSetting<List<String>> listAffixSetting = Setting.affixKeySetting("foo.", "bar",
+            (key) -> Setting.listSetting(key, Collections.singletonList("testelement"), Function.identity(), Property.NodeScope));
+        expectThrows(UnsupportedOperationException.class, () -> listAffixSetting.get(Settings.EMPTY));
+        expectThrows(UnsupportedOperationException.class, () -> listAffixSetting.getRaw(Settings.EMPTY));
+        assertEquals(Collections.singletonList("testelement"), listAffixSetting.getDefault(Settings.EMPTY));
+        assertEquals("[\"testelement\"]", listAffixSetting.getDefaultRaw(Settings.EMPTY));
     }
 
     public void testMinMaxInt() {
@@ -530,4 +571,5 @@ public class SettingTests extends ESTestCase {
         assertThat(setting.get(Settings.builder().put("foo", "12h").build()), equalTo(TimeValue.timeValueHours(12)));
         assertThat(setting.get(Settings.EMPTY).getMillis(), equalTo(random.getMillis() * factor));
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
index 14e93a3bbc3..93f745d290e 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsFilterTests.java
@@ -118,7 +118,7 @@ public class SettingsFilterTests extends ESTestCase {
         source.toXContent(xContentBuilder, request);
         xContentBuilder.endObject();
         String filteredSettingsString = xContentBuilder.string();
-        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build();
+        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build();
         assertThat(filteredSettings.getAsMap().entrySet(), equalTo(filtered.getAsMap().entrySet()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index dc0545624d6..6da023aef2b 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -118,7 +118,7 @@ public class SettingsModuleTests extends ModuleTestCase {
                 new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
-                assertEquals("Failed to parse value [BOOM] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]",
+                assertEquals("Failed to parse value [BOOM] as only [true] or [false] are allowed.",
                         ex.getMessage());
             }
         }
@@ -163,7 +163,7 @@ public class SettingsModuleTests extends ModuleTestCase {
             Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered),
             Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*"));
         assertInstanceBinding(module, Settings.class, (s) -> s == settings);
-        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().size() == 1);
+        assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).size() == 1);
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().containsKey("bar.baz"));
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().get("bar.baz").equals("false"));
 
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 346c5bc60de..308b7883354 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -19,13 +19,20 @@
 
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.loader.YamlSettingsLoader;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.allOf;
@@ -131,15 +138,115 @@ public class SettingsTests extends ESTestCase {
 
     public void testGetAsSettings() {
         Settings settings = Settings.builder()
+                .put("bar", "hello world")
                 .put("foo", "abc")
                 .put("foo.bar", "def")
                 .put("foo.baz", "ghi").build();
 
         Settings fooSettings = settings.getAsSettings("foo");
+        assertFalse(fooSettings.isEmpty());
+        assertEquals(2, fooSettings.size());
         assertThat(fooSettings.get("bar"), equalTo("def"));
         assertThat(fooSettings.get("baz"), equalTo("ghi"));
     }
 
+    @SuppressWarnings("deprecation") //#getAsBooleanLenientForPreEs6Indices is the test subject
+    public void testLenientBooleanForPreEs6Index() throws IOException {
+        // time to say goodbye?
+        assertTrue(
+            "It's time to implement #22298. Please delete this test and Settings#getAsBooleanLenientForPreEs6Indices().",
+            Version.CURRENT.minimumCompatibilityVersion().before(Version.V_6_0_0_alpha1_UNRELEASED));
+
+
+        String falsy = randomFrom("false", "off", "no", "0");
+        String truthy = randomFrom("true", "on", "yes", "1");
+
+        Settings settings = Settings.builder()
+            .put("foo", falsy)
+            .put("bar", truthy).build();
+
+        final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger("testLenientBooleanForPreEs6Index"));
+
+        assertFalse(settings.getAsBooleanLenientForPreEs6Indices(Version.V_5_0_0, "foo", null, deprecationLogger));
+        assertTrue(settings.getAsBooleanLenientForPreEs6Indices(Version.V_5_0_0, "bar", null, deprecationLogger));
+        assertTrue(settings.getAsBooleanLenientForPreEs6Indices(Version.V_5_0_0, "baz", true, deprecationLogger));
+
+        List<String> expectedDeprecationWarnings = new ArrayList<>();
+        if (Booleans.isBoolean(falsy) == false) {
+            expectedDeprecationWarnings.add(
+                "The value [" + falsy + "] of setting [foo] is not coerced into boolean anymore. Please change this value to [false].");
+        }
+        if (Booleans.isBoolean(truthy) == false) {
+            expectedDeprecationWarnings.add(
+                "The value [" + truthy + "] of setting [bar] is not coerced into boolean anymore. Please change this value to [true].");
+        }
+
+        if (expectedDeprecationWarnings.isEmpty() == false) {
+            assertWarnings(expectedDeprecationWarnings.toArray(new String[1]));
+        }
+    }
+
+    @SuppressWarnings("deprecation") //#getAsBooleanLenientForPreEs6Indices is the test subject
+    public void testInvalidLenientBooleanForCurrentIndexVersion() {
+        String falsy = randomFrom("off", "no", "0");
+        String truthy = randomFrom("on", "yes", "1");
+
+        Settings settings = Settings.builder()
+            .put("foo", falsy)
+            .put("bar", truthy).build();
+
+        final DeprecationLogger deprecationLogger =
+            new DeprecationLogger(ESLoggerFactory.getLogger("testInvalidLenientBooleanForCurrentIndexVersion"));
+        expectThrows(IllegalArgumentException.class,
+            () -> settings.getAsBooleanLenientForPreEs6Indices(Version.CURRENT, "foo", null, deprecationLogger));
+        expectThrows(IllegalArgumentException.class,
+            () -> settings.getAsBooleanLenientForPreEs6Indices(Version.CURRENT, "bar", null, deprecationLogger));
+    }
+
+    @SuppressWarnings("deprecation") //#getAsBooleanLenientForPreEs6Indices is the test subject
+    public void testValidLenientBooleanForCurrentIndexVersion() {
+        Settings settings = Settings.builder()
+            .put("foo", "false")
+            .put("bar", "true").build();
+
+        final DeprecationLogger deprecationLogger =
+            new DeprecationLogger(ESLoggerFactory.getLogger("testValidLenientBooleanForCurrentIndexVersion"));
+        assertFalse(settings.getAsBooleanLenientForPreEs6Indices(Version.CURRENT, "foo", null, deprecationLogger));
+        assertTrue(settings.getAsBooleanLenientForPreEs6Indices(Version.CURRENT, "bar", null, deprecationLogger));
+        assertTrue(settings.getAsBooleanLenientForPreEs6Indices(Version.CURRENT, "baz", true, deprecationLogger));
+    }
+
+    public void testMultLevelGetPrefix() {
+        Settings settings = Settings.builder()
+            .put("1.2.3", "hello world")
+            .put("1.2.3.4", "abc")
+            .put("2.3.4", "def")
+            .put("3.4", "ghi").build();
+
+        Settings firstLevelSettings = settings.getByPrefix("1.");
+        assertFalse(firstLevelSettings.isEmpty());
+        assertEquals(2, firstLevelSettings.size());
+        assertThat(firstLevelSettings.get("2.3.4"), equalTo("abc"));
+        assertThat(firstLevelSettings.get("2.3"), equalTo("hello world"));
+
+        Settings secondLevelSetting = firstLevelSettings.getByPrefix("2.");
+        assertFalse(secondLevelSetting.isEmpty());
+        assertEquals(2, secondLevelSetting.size());
+        assertNull(secondLevelSetting.get("2.3.4"));
+        assertNull(secondLevelSetting.get("1.2.3.4"));
+        assertNull(secondLevelSetting.get("1.2.3"));
+        assertThat(secondLevelSetting.get("3.4"), equalTo("abc"));
+        assertThat(secondLevelSetting.get("3"), equalTo("hello world"));
+
+        Settings thirdLevelSetting = secondLevelSetting.getByPrefix("3.");
+        assertFalse(thirdLevelSetting.isEmpty());
+        assertEquals(1, thirdLevelSetting.size());
+        assertNull(thirdLevelSetting.get("2.3.4"));
+        assertNull(thirdLevelSetting.get("3.4"));
+        assertNull(thirdLevelSetting.get("1.2.3"));
+        assertThat(thirdLevelSetting.get("4"), equalTo("abc"));
+    }
+
     public void testNames() {
         Settings settings = Settings.builder()
                 .put("bar", "baz")
@@ -273,7 +380,7 @@ public class SettingsTests extends ESTestCase {
                 .normalizePrefix("foo.")
                 .build();
 
-        assertThat(settings.getAsMap().size(), equalTo(1));
+        assertThat(settings.size(), equalTo(1));
         assertThat(settings.get("bar"), nullValue());
         assertThat(settings.get("foo.bar"), equalTo("baz"));
 
@@ -284,7 +391,7 @@ public class SettingsTests extends ESTestCase {
                 .normalizePrefix("foo.")
                 .build();
 
-        assertThat(settings.getAsMap().size(), equalTo(2));
+        assertThat(settings.size(), equalTo(2));
         assertThat(settings.get("bar"), nullValue());
         assertThat(settings.get("foo.bar"), equalTo("baz"));
         assertThat(settings.get("foo.test"), equalTo("test"));
@@ -295,7 +402,151 @@ public class SettingsTests extends ESTestCase {
                 .build();
 
 
-        assertThat(settings.getAsMap().size(), equalTo(1));
+        assertThat(settings.size(), equalTo(1));
         assertThat(settings.get("foo.test"), equalTo("test"));
     }
+
+    public void testFilteredMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+
+        Map<String, String> fiteredMap = builder.build().filter((k) -> k.startsWith("a.b")).getAsMap();
+        assertEquals(3, fiteredMap.size());
+        int numKeys = 0;
+        for (String k : fiteredMap.keySet()) {
+            numKeys++;
+            assertTrue(k.startsWith("a.b"));
+        }
+
+        assertEquals(3, numKeys);
+        int numValues = 0;
+
+        for (String v : fiteredMap.values()) {
+            numValues++;
+            assertTrue(v.startsWith("ab"));
+        }
+        assertEquals(3, numValues);
+        assertFalse(fiteredMap.containsKey("a.c"));
+        assertFalse(fiteredMap.containsKey("a"));
+        assertTrue(fiteredMap.containsKey("a.b"));
+        assertTrue(fiteredMap.containsKey("a.b.c"));
+        assertTrue(fiteredMap.containsKey("a.b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            fiteredMap.remove("a.b"));
+        assertEquals("ab1", fiteredMap.get("a.b"));
+        assertEquals("ab2", fiteredMap.get("a.b.c"));
+        assertEquals("ab3", fiteredMap.get("a.b.c.d"));
+
+        Iterator<String> iterator = fiteredMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b", iterator.next());
+        if (randomBoolean()) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b.c", iterator.next());
+        if (randomBoolean()) {
+            assertTrue(iterator.hasNext());
+        }
+        assertEquals("a.b.c.d", iterator.next());
+        assertFalse(iterator.hasNext());
+        expectThrows(NoSuchElementException.class, () -> iterator.next());
+
+    }
+
+    public void testPrefixMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+        Map<String, String> prefixMap = builder.build().getByPrefix("a.").getAsMap();
+        assertEquals(4, prefixMap.size());
+        int numKeys = 0;
+        for (String k : prefixMap.keySet()) {
+            numKeys++;
+            assertTrue(k, k.startsWith("b") || k.startsWith("c"));
+        }
+
+        assertEquals(4, numKeys);
+        int numValues = 0;
+
+        for (String v : prefixMap.values()) {
+            numValues++;
+            assertTrue(v, v.startsWith("ab") || v.startsWith("ac"));
+        }
+        assertEquals(4, numValues);
+        assertFalse(prefixMap.containsKey("a"));
+        assertTrue(prefixMap.containsKey("c"));
+        assertTrue(prefixMap.containsKey("b"));
+        assertTrue(prefixMap.containsKey("b.c"));
+        assertTrue(prefixMap.containsKey("b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            prefixMap.remove("a.b"));
+        assertEquals("ab1", prefixMap.get("b"));
+        assertEquals("ab2", prefixMap.get("b.c"));
+        assertEquals("ab3", prefixMap.get("b.c.d"));
+        Iterator<String> prefixIterator = prefixMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b.c", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("b.c.d", prefixIterator.next());
+        if (randomBoolean()) {
+            assertTrue(prefixIterator.hasNext());
+        }
+        assertEquals("c", prefixIterator.next());
+        assertFalse(prefixIterator.hasNext());
+        expectThrows(NoSuchElementException.class, () -> prefixIterator.next());
+    }
+
+    public void testEmptyFilterMap() {
+        Settings.Builder builder = Settings.builder();
+        builder.put("a", "a1");
+        builder.put("a.b", "ab1");
+        builder.put("a.b.c", "ab2");
+        builder.put("a.c", "ac1");
+        builder.put("a.b.c.d", "ab3");
+
+        Map<String, String> fiteredMap = builder.build().filter((k) -> false).getAsMap();
+        assertEquals(0, fiteredMap.size());
+        for (String k : fiteredMap.keySet()) {
+            fail("no element");
+
+        }
+        for (String v : fiteredMap.values()) {
+            fail("no element");
+        }
+        assertFalse(fiteredMap.containsKey("a.c"));
+        assertFalse(fiteredMap.containsKey("a"));
+        assertFalse(fiteredMap.containsKey("a.b"));
+        assertFalse(fiteredMap.containsKey("a.b.c"));
+        assertFalse(fiteredMap.containsKey("a.b.c.d"));
+        expectThrows(UnsupportedOperationException.class, () ->
+            fiteredMap.remove("a.b"));
+        assertNull(fiteredMap.get("a.b"));
+        assertNull(fiteredMap.get("a.b.c"));
+        assertNull(fiteredMap.get("a.b.c.d"));
+
+        Iterator<String> iterator = fiteredMap.keySet().iterator();
+        for (int i = 0; i < 10; i++) {
+            assertFalse(iterator.hasNext());
+        }
+        expectThrows(NoSuchElementException.class, () -> iterator.next());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
index d917c0d12c0..fc1300d9413 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/JsonSettingsLoaderTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -52,7 +53,8 @@ public class JsonSettingsLoaderTests extends ESTestCase {
         assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
             XContent.isStrictDuplicateDetectionEnabled());
         final String json = "{\"foo\":\"bar\",\"foo\":\"baz\"}";
-        final SettingsException e = expectThrows(SettingsException.class, () -> Settings.builder().loadFromSource(json).build());
+        final SettingsException e = expectThrows(SettingsException.class,
+            () -> Settings.builder().loadFromSource(json, XContentType.JSON).build());
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
         assertThat(
                 e.toString(),
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index eeb2df7e1d6..e4b4de0ceb6 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.charset.StandardCharsets;
@@ -74,7 +75,7 @@ public class YamlSettingsLoaderTests extends ESTestCase {
 
         String yaml = "foo: bar\nfoo: baz";
         SettingsException e = expectThrows(SettingsException.class, () -> {
-            Settings.builder().loadFromSource(yaml);
+            Settings.builder().loadFromSource(yaml, XContentType.YAML);
         });
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
         String msg = e.getCause().getMessage();
diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
index 6ad5a1ce32a..9d0a3e67e9d 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java
@@ -150,7 +150,7 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testCompareEquality() {
-        long firstRandom = randomPositiveLong();
+        long firstRandom = randomNonNegativeLong();
         ByteSizeUnit randomUnit = randomFrom(ByteSizeUnit.values());
         ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, randomUnit);
         ByteSizeValue secondByteValue = new ByteSizeValue(firstRandom, randomUnit);
@@ -158,8 +158,8 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testCompareValue() {
-        long firstRandom = randomPositiveLong();
-        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong);
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
         ByteSizeUnit unit = randomFrom(ByteSizeUnit.values());
         ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, unit);
         ByteSizeValue secondByteValue = new ByteSizeValue(secondRandom, unit);
@@ -168,7 +168,7 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testCompareUnits() {
-        long number = randomPositiveLong();
+        long number = randomNonNegativeLong();
         ByteSizeUnit randomUnit = randomValueOtherThan(ByteSizeUnit.PB, ()->randomFrom(ByteSizeUnit.values()));
         ByteSizeValue firstByteValue = new ByteSizeValue(number, randomUnit);
         ByteSizeValue secondByteValue = new ByteSizeValue(number, ByteSizeUnit.PB);
@@ -189,7 +189,7 @@ public class ByteSizeValueTests extends ESTestCase {
     }
 
     public void testSerialization() throws IOException {
-        ByteSizeValue byteSizeValue = new ByteSizeValue(randomPositiveLong(), randomFrom(ByteSizeUnit.values()));
+        ByteSizeValue byteSizeValue = new ByteSizeValue(randomNonNegativeLong(), randomFrom(ByteSizeUnit.values()));
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             byteSizeValue.writeTo(out);
             try (StreamInput in = out.bytes().streamInput()) {
diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
index 3f6f1848fd8..b370250bf9d 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
@@ -20,9 +20,8 @@ package org.elasticsearch.common.unit;
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -31,7 +30,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.sameInstance;
-import static org.hamcrest.number.IsCloseTo.closeTo;
 
 public class FuzzinessTests extends ESTestCase {
     public void testNumerics() {
@@ -43,12 +41,11 @@ public class FuzzinessTests extends ESTestCase {
         final int iters = randomIntBetween(10, 50);
         for (int i = 0; i < iters; i++) {
             {
-                XContent xcontent = XContentType.JSON.xContent();
                 float floatValue = randomFloat();
-                String json = jsonBuilder().startObject()
+                XContentBuilder json = jsonBuilder().startObject()
                         .field(Fuzziness.X_FIELD_NAME, floatValue)
-                        .endObject().string();
-                XContentParser parser = xcontent.createParser(json);
+                        .endObject();
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
@@ -57,17 +54,16 @@ public class FuzzinessTests extends ESTestCase {
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
             }
             {
-                XContent xcontent = XContentType.JSON.xContent();
                 Integer intValue = frequently() ? randomIntBetween(0, 2) : randomIntBetween(0, 100);
                 Float floatRep = randomFloat();
                 Number value = intValue;
                 if (randomBoolean()) {
                     value = new Float(floatRep += intValue);
                 }
-                String json = jsonBuilder().startObject()
+                XContentBuilder json = jsonBuilder().startObject()
                         .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? value.toString() : value)
-                        .endObject().string();
-                XContentParser parser = xcontent.createParser(json);
+                        .endObject();
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), anyOf(equalTo(XContentParser.Token.VALUE_NUMBER), equalTo(XContentParser.Token.VALUE_STRING)));
@@ -93,14 +89,15 @@ public class FuzzinessTests extends ESTestCase {
                 }
             }
             {
-                XContent xcontent = XContentType.JSON.xContent();
-                String json = jsonBuilder().startObject()
-                        .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? "AUTO" : "auto")
-                        .endObject().string();
+                XContentBuilder json;
                 if (randomBoolean()) {
-                    json = Fuzziness.AUTO.toXContent(jsonBuilder().startObject(), null).endObject().string();
+                    json = Fuzziness.AUTO.toXContent(jsonBuilder().startObject(), null).endObject();
+                } else {
+                    json = jsonBuilder().startObject()
+                            .field(Fuzziness.X_FIELD_NAME, randomBoolean() ? "AUTO" : "auto")
+                            .endObject();
                 }
-                XContentParser parser = xcontent.createParser(json);
+                XContentParser parser = createParser(json);
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
diff --git a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
index 3a97a11308b..4fd2af3ce54 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java
@@ -66,7 +66,7 @@ public class SizeValueTests extends ESTestCase {
     }
 
     public void testCompareEquality() {
-        long randomValue = randomPositiveLong();
+        long randomValue = randomNonNegativeLong();
         SizeUnit randomUnit = randomFrom(SizeUnit.values());
         SizeValue firstValue = new SizeValue(randomValue, randomUnit);
         SizeValue secondValue = new SizeValue(randomValue, randomUnit);
@@ -74,8 +74,8 @@ public class SizeValueTests extends ESTestCase {
     }
 
     public void testCompareValue() {
-        long firstRandom = randomPositiveLong();
-        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong);
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
         SizeUnit unit = randomFrom(SizeUnit.values());
         SizeValue firstSizeValue = new SizeValue(firstRandom, unit);
         SizeValue secondSizeValue = new SizeValue(secondRandom, unit);
@@ -84,7 +84,7 @@ public class SizeValueTests extends ESTestCase {
     }
 
     public void testCompareUnits() {
-        long number = randomPositiveLong();
+        long number = randomNonNegativeLong();
         SizeUnit randomUnit = randomValueOtherThan(SizeUnit.PETA, ()->randomFrom(SizeUnit.values()));
         SizeValue firstValue = new SizeValue(number, randomUnit);
         SizeValue secondValue = new SizeValue(number, SizeUnit.PETA);
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 1f7e876f856..cbfd98aa3b7 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -226,7 +226,7 @@ public class TimeValueTests extends ESTestCase {
     }
 
     public void testCompareEquality() {
-        long randomLong = randomPositiveLong();
+        long randomLong = randomNonNegativeLong();
         TimeUnit randomUnit = randomFrom(TimeUnit.values());
         TimeValue firstValue = new TimeValue(randomLong, randomUnit);
         TimeValue secondValue = new TimeValue(randomLong, randomUnit);
@@ -234,8 +234,8 @@ public class TimeValueTests extends ESTestCase {
     }
 
     public void testCompareValue() {
-        long firstRandom = randomPositiveLong();
-        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong);
+        long firstRandom = randomNonNegativeLong();
+        long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong);
         TimeUnit unit = randomFrom(TimeUnit.values());
         TimeValue firstValue = new TimeValue(firstRandom, unit);
         TimeValue secondValue = new TimeValue(secondRandom, unit);
@@ -244,7 +244,7 @@ public class TimeValueTests extends ESTestCase {
     }
 
     public void testCompareUnits() {
-        long number = randomPositiveLong();
+        long number = randomNonNegativeLong();
         TimeUnit randomUnit = randomValueOtherThan(TimeUnit.DAYS, ()->randomFrom(TimeUnit.values()));
         TimeValue firstValue = new TimeValue(number, randomUnit);
         TimeValue secondValue = new TimeValue(number, TimeUnit.DAYS);
diff --git a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
index 916926e36a4..581d58d896d 100644
--- a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.MetaDataStateFormat;
@@ -223,7 +224,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
             assertEquals(indexFolders.size(), 1);
 
             // ensure index metadata is moved
-            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger,
+            IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
                 nodeEnvironment.resolveIndexFolder(indexFolders.iterator().next()));
             assertNotNull(indexMetaData);
             Index index = indexMetaData.getIndex();
@@ -265,7 +266,8 @@ public class IndexFolderUpgraderTests extends ESTestCase {
                             int numIdxFiles, int numTranslogFiles) throws IOException {
         final Index index = indexSettings.getIndex();
         // ensure index state can be loaded
-        IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(index));
+        IndexMetaData loadLatestState = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+            nodeEnv.indexPaths(index));
         assertNotNull(loadLatestState);
         assertEquals(loadLatestState.getIndex(), index);
         for (int shardId = 0; shardId < indexSettings.getNumberOfShards(); shardId++) {
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
index ed4c7c9f7bb..b2664b134ed 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
@@ -128,7 +128,7 @@ public class RefCountedTests extends ESTestCase {
 
         private final AtomicBoolean closed = new AtomicBoolean(false);
 
-        public MyRefCounted() {
+        MyRefCounted() {
             super("test");
         }
 
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 3f914f61d48..582f5d58d80 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -27,6 +27,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -86,7 +87,7 @@ public class ThreadContextTests extends ESTestCase {
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
-        ThreadContext.StoredContext storedContext = threadContext.newStoredContext();
+        ThreadContext.StoredContext storedContext = threadContext.newStoredContext(false);
         threadContext.putHeader("foo.bar", "baz");
         try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
             assertNull(threadContext.getHeader("foo"));
@@ -109,6 +110,63 @@ public class ThreadContextTests extends ESTestCase {
         assertNull(threadContext.getHeader("foo.bar"));
     }
 
+    public void testRestorableContext() {
+        Settings build = Settings.builder().put("request.headers.default", "1").build();
+        ThreadContext threadContext = new ThreadContext(build);
+        threadContext.putHeader("foo", "bar");
+        threadContext.putTransient("ctx.foo", 1);
+        threadContext.addResponseHeader("resp.header", "baaaam");
+        Supplier<ThreadContext.StoredContext> contextSupplier = threadContext.newRestorableContext(true);
+
+        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals("1", threadContext.getHeader("default"));
+            threadContext.addResponseHeader("resp.header", "boom");
+            try (ThreadContext.StoredContext tmp = contextSupplier.get()) {
+                assertEquals("bar", threadContext.getHeader("foo"));
+                assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+                assertEquals("1", threadContext.getHeader("default"));
+                assertEquals(2, threadContext.getResponseHeaders().get("resp.header").size());
+                assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+                assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(1));
+            }
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("ctx.foo"));
+            assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+            assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+        }
+        assertEquals("bar", threadContext.getHeader("foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+        assertEquals("1", threadContext.getHeader("default"));
+        assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+        assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+
+        contextSupplier = threadContext.newRestorableContext(false);
+
+        try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
+            assertNull(threadContext.getHeader("foo"));
+            assertEquals("1", threadContext.getHeader("default"));
+            threadContext.addResponseHeader("resp.header", "boom");
+            try (ThreadContext.StoredContext tmp = contextSupplier.get()) {
+                assertEquals("bar", threadContext.getHeader("foo"));
+                assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+                assertEquals("1", threadContext.getHeader("default"));
+                assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+                assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+            }
+            assertNull(threadContext.getHeader("foo"));
+            assertNull(threadContext.getTransient("ctx.foo"));
+            assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+            assertEquals("boom", threadContext.getResponseHeaders().get("resp.header").get(0));
+        }
+
+        assertEquals("bar", threadContext.getHeader("foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
+        assertEquals("1", threadContext.getHeader("default"));
+        assertEquals(1, threadContext.getResponseHeaders().get("resp.header").size());
+        assertEquals("baaaam", threadContext.getResponseHeaders().get("resp.header").get(0));
+    }
+
     public void testResponseHeaders() {
         final boolean expectThird = randomBoolean();
 
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index 1c0a92dbd74..e468751cf4a 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -21,11 +21,13 @@ package org.elasticsearch.common.xcontent;
 
 import com.fasterxml.jackson.core.JsonGenerationException;
 import com.fasterxml.jackson.core.JsonGenerator;
-
 import com.fasterxml.jackson.core.JsonParseException;
+
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -751,7 +753,7 @@ public abstract class BaseXContentTestCase extends ESTestCase {
             generator.writeEndObject();
         }
 
-        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("bar", parser.currentName());
@@ -785,7 +787,7 @@ public abstract class BaseXContentTestCase extends ESTestCase {
             generator.writeRawValue(new BytesArray(rawData));
         }
 
-        XContentParser parser = xcontentType().xContent().createParser(os.toByteArray());
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("foo", parser.currentName());
@@ -801,7 +803,7 @@ public abstract class BaseXContentTestCase extends ESTestCase {
             generator.writeEndObject();
         }
 
-        parser = xcontentType().xContent().createParser(os.toByteArray());
+        parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, os.toByteArray());
         assertEquals(Token.START_OBJECT, parser.nextToken());
         assertEquals(Token.FIELD_NAME, parser.nextToken());
         assertEquals("test", parser.currentName());
@@ -829,7 +831,7 @@ public abstract class BaseXContentTestCase extends ESTestCase {
         generator.flush();
         byte[] serialized = os.toByteArray();
 
-        XContentParser parser = xcontentType().xContent().createParser(serialized);
+        XContentParser parser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, serialized);
         Map<String, Object> map = parser.map();
         assertEquals("bar", map.get("foo"));
         assertEquals(bigInteger, map.get("bigint"));
@@ -990,17 +992,50 @@ public abstract class BaseXContentTestCase extends ESTestCase {
         assumeTrue("Test only makes sense if XContent parser has strict duplicate checks enabled",
             XContent.isStrictDuplicateDetectionEnabled());
 
-        BytesReference bytes = builder()
+        XContentBuilder builder = builder()
                 .startObject()
                     .field("key", 1)
                     .field("key", 2)
-                .endObject()
-            .bytes();
+                .endObject();
 
-        JsonParseException pex = expectThrows(JsonParseException.class, () -> createParser(xcontentType().xContent(), bytes).map());
+        JsonParseException pex = expectThrows(JsonParseException.class, () -> createParser(builder).map());
         assertThat(pex.getMessage(), startsWith("Duplicate field 'key'"));
     }
 
+    public void testNamedObject() throws IOException {
+        Object test1 = new Object();
+        Object test2 = new Object();
+        NamedXContentRegistry registry = new NamedXContentRegistry(Arrays.asList(
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("test1"), p -> test1),
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("test2", "deprecated"), p -> test2),
+                new NamedXContentRegistry.Entry(Object.class, new ParseField("str"), p -> p.text())));
+        XContentBuilder b = XContentBuilder.builder(xcontentType().xContent());
+        b.value("test");
+        XContentParser p = xcontentType().xContent().createParser(registry, b.bytes());
+        assertEquals(test1, p.namedObject(Object.class, "test1", null));
+        assertEquals(test2, p.namedObject(Object.class, "test2", null));
+        assertEquals(test2, p.namedObject(Object.class, "deprecated", null));
+        assertWarnings("Deprecated field [deprecated] used, expected [test2] instead");
+        {
+            p.nextToken();
+            assertEquals("test", p.namedObject(Object.class, "str", null));
+            NamedXContentRegistry.UnknownNamedObjectException e = expectThrows(NamedXContentRegistry.UnknownNamedObjectException.class,
+                    () -> p.namedObject(Object.class, "unknown", null));
+            assertEquals("Unknown Object [unknown]", e.getMessage());
+            assertEquals("java.lang.Object", e.getCategoryClass());
+            assertEquals("unknown", e.getName());
+        }
+        {
+            Exception e = expectThrows(ElasticsearchException.class, () -> p.namedObject(String.class, "doesn't matter", null));
+            assertEquals("Unknown namedObject category [java.lang.String]", e.getMessage());
+        }
+        {
+            XContentParser emptyRegistryParser = xcontentType().xContent().createParser(NamedXContentRegistry.EMPTY, new byte[] {});
+            Exception e = expectThrows(ElasticsearchException.class,
+                    () -> emptyRegistryParser.namedObject(String.class, "doesn't matter", null));
+            assertEquals("namedObject is not supported for this parser", e.getMessage());
+        }
+    }
 
     private static void expectUnclosedException(ThrowingRunnable runnable) {
         IllegalStateException e = expectThrows(IllegalStateException.class, runnable);
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
index 6cae59391c5..0e11e630ea9 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
@@ -19,13 +19,11 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.AbstractObjectParser.NoContextParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matcher;
@@ -43,19 +41,17 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ConstructingObjectParserTests extends ESTestCase {
-    private static final ParseFieldMatcherSupplier MATCHER = () -> ParseFieldMatcher.STRICT;
-
     public void testNullDeclares() {
-        ConstructingObjectParser<Void, ParseFieldMatcherSupplier> objectParser = new ConstructingObjectParser<>("foo", a -> null);
+        ConstructingObjectParser<Void, Void> objectParser = new ConstructingObjectParser<>("foo", a -> null);
         Exception e = expectThrows(IllegalArgumentException.class,
                 () -> objectParser.declareField(null, (r, c) -> null, new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[consumer] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
-                (o, v) -> {}, (ContextParser<ParseFieldMatcherSupplier, Object>) null,
+                (o, v) -> {}, (ContextParser<Void, Object>) null,
                 new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[parser] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
-                (o, v) -> {}, (NoContextParser<Object>) null,
+                (o, v) -> {}, (CheckedFunction<XContentParser, Object, IOException>) null,
                 new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[parser] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
@@ -82,7 +78,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
         builder = shuffleXContent(builder);
         BytesReference bytes = builder.bytes();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, bytes)) {
-            HasCtorArguments parsed = randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER);
+            HasCtorArguments parsed = randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null);
             assertEquals(expected.animal, parsed.animal);
             assertEquals(expected.vegetable, parsed.vegetable);
             assertEquals(expected.mineral, parsed.mineral);
@@ -102,9 +98,9 @@ public class ConstructingObjectParserTests extends ESTestCase {
                   "{\n"
                 + "  \"mineral\": 1\n"
                 + "}");
-        ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> objectParser = randomBoolean() ? HasCtorArguments.PARSER
+        ConstructingObjectParser<HasCtorArguments, Void> objectParser = randomBoolean() ? HasCtorArguments.PARSER
                 : HasCtorArguments.PARSER_VEGETABLE_OPTIONAL;
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> objectParser.apply(parser, MATCHER));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> objectParser.apply(parser, null));
         if (objectParser == HasCtorArguments.PARSER) {
             assertEquals("Required [animal, vegetable]", e.getMessage());
         } else {
@@ -117,7 +113,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 "{\n"
               + "  \"mineral\": 1\n"
               + "}");
-        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, null);
         assertEquals(1, parsed.mineral);
     }
 
@@ -128,7 +124,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"animal\": \"cat\"\n"
                 + "}");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> HasCtorArguments.PARSER.apply(parser, MATCHER));
+                () -> HasCtorArguments.PARSER.apply(parser, null));
         assertEquals("Required [vegetable]", e.getMessage());
     }
 
@@ -140,7 +136,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
               + "}");
         @SuppressWarnings("unchecked")
         HasCtorArguments parsed = randomFrom(HasCtorArguments.PARSER_VEGETABLE_OPTIONAL, HasCtorArguments.PARSER_ALL_OPTIONAL).apply(parser,
-                MATCHER);
+                null);
         assertEquals(1, parsed.mineral);
         assertEquals("cat", parsed.animal);
     }
@@ -153,7 +149,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "}");
         @SuppressWarnings("unchecked")
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> randomFrom(HasCtorArguments.PARSER, HasCtorArguments.PARSER_VEGETABLE_OPTIONAL).apply(parser, MATCHER));
+                () -> randomFrom(HasCtorArguments.PARSER, HasCtorArguments.PARSER_VEGETABLE_OPTIONAL).apply(parser, null));
         assertEquals("Required [animal]", e.getMessage());
     }
 
@@ -163,7 +159,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
               + "  \"mineral\": 1,\n"
               + "  \"vegetable\": 2\n"
               + "}");
-        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, null);
         assertEquals(1, parsed.mineral);
         assertEquals((Integer) 2, parsed.vegetable);
     }
@@ -176,7 +172,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"vegetable\": 1,\n"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        Throwable e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        Throwable e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         e = e.getCause();
         assertThat(e, instanceOf(IllegalArgumentException.class));
@@ -190,7 +186,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"vegetable\": 2,\n"
                 + "  \"a\": \"supercalifragilisticexpialidocious\"\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [a]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         assertEquals("[a] must be less than 10 characters in length but was [supercalifragilisticexpialidocious]",
@@ -204,7 +200,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"animal\": \"cat\"\n,"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, null));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         e = (ParsingException) e.getCause();
@@ -220,7 +216,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
     public void testConstructorArgsMustBeConfigured() throws IOException {
         class NoConstructorArgs {
         }
-        ConstructingObjectParser<NoConstructorArgs, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
+        ConstructingObjectParser<NoConstructorArgs, Void> parser = new ConstructingObjectParser<>(
                 "constructor_args_required", (a) -> new NoConstructorArgs());
         try {
             parser.apply(createParser(JsonXContent.jsonXContent, "{}"), null);
@@ -237,7 +233,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
     public void testCalledOneTime() throws IOException {
         boolean ctorArgOptional = randomBoolean();
         class CalledOneTime {
-            public CalledOneTime(String yeah) {
+            CalledOneTime(String yeah) {
                 Matcher<String> yeahMatcher = equalTo("!");
                 if (ctorArgOptional) {
                     // either(yeahMatcher).or(nullValue) is broken by https://github.com/hamcrest/JavaHamcrest/issues/49
@@ -252,7 +248,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 fooSet = true;
             }
         }
-        ConstructingObjectParser<CalledOneTime, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>("one_time_test",
+        ConstructingObjectParser<CalledOneTime, Void> parser = new ConstructingObjectParser<>("one_time_test",
                 (a) -> new CalledOneTime((String) a[0]));
         parser.declareString(CalledOneTime::setFoo, new ParseField("foo"));
         parser.declareString(ctorArgOptional ? optionalConstructorArg() : constructorArg(), new ParseField("yeah"));
@@ -263,7 +259,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
               + "  \"yeah\": \"!\",\n"
               + "  \"foo\": \"foo\"\n"
               + "}");
-        CalledOneTime result = parser.apply(xcontent, MATCHER);
+        CalledOneTime result = parser.apply(xcontent, null);
         assertTrue(result.fooSet);
 
         // and ctor arg second just in case
@@ -272,7 +268,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
               + "  \"foo\": \"foo\",\n"
               + "  \"yeah\": \"!\"\n"
               + "}");
-        result = parser.apply(xcontent, MATCHER);
+        result = parser.apply(xcontent, null);
         assertTrue(result.fooSet);
 
         if (ctorArgOptional) {
@@ -281,7 +277,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                     "{\n"
                   + "  \"foo\": \"foo\"\n"
                   + "}");
-            result = parser.apply(xcontent, MATCHER);
+            result = parser.apply(xcontent, null);
         }
         assertTrue(result.fooSet);
     }
@@ -294,14 +290,14 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "}");
         class TestStruct {
             public final String test;
-            public TestStruct(String test) {
+            TestStruct(String test) {
                 this.test = test;
             }
         }
-        ConstructingObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ConstructingObjectParser<>("foo", true, a ->
+        ConstructingObjectParser<TestStruct, Void> objectParser = new ConstructingObjectParser<>("foo", true, a ->
                 new TestStruct((String) a[0]));
         objectParser.declareString(constructorArg(), new ParseField("test"));
-        TestStruct s = objectParser.apply(parser, MATCHER);
+        TestStruct s = objectParser.apply(parser, null);
         assertEquals(s.test, "foo");
     }
 
@@ -317,7 +313,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
         String c;
         boolean d;
 
-        public HasCtorArguments(@Nullable String animal, @Nullable Integer vegetable) {
+        HasCtorArguments(@Nullable String animal, @Nullable Integer vegetable) {
             this.animal = animal;
             this.vegetable = vegetable;
         }
@@ -380,17 +376,15 @@ public class ConstructingObjectParserTests extends ESTestCase {
          * It is normal just to declare a single PARSER but we use a couple of different parsers for testing so we have all of these. Don't
          * this this style is normal just because it is in the test.
          */
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER = buildParser(true, true);
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_VEGETABLE_OPTIONAL = buildParser(
-                true, false);
-        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_ALL_OPTIONAL = buildParser(false,
-                false);
-        public static final List<ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier>> ALL_PARSERS = unmodifiableList(
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER = buildParser(true, true);
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER_VEGETABLE_OPTIONAL = buildParser(true, false);
+        public static final ConstructingObjectParser<HasCtorArguments, Void> PARSER_ALL_OPTIONAL = buildParser(false, false);
+        public static final List<ConstructingObjectParser<HasCtorArguments, Void>> ALL_PARSERS = unmodifiableList(
                 Arrays.asList(PARSER, PARSER_VEGETABLE_OPTIONAL, PARSER_ALL_OPTIONAL));
 
-        private static ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> buildParser(boolean animalRequired,
+        private static ConstructingObjectParser<HasCtorArguments, Void> buildParser(boolean animalRequired,
                 boolean vegetableRequired) {
-            ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
+            ConstructingObjectParser<HasCtorArguments, Void> parser = new ConstructingObjectParser<>(
                     "has_required_arguments", a -> new HasCtorArguments((String) a[0], (Integer) a[1]));
             parser.declareString(animalRequired ? constructorArg() : optionalConstructorArg(), new ParseField("animal"));
             parser.declareInt(vegetableRequired ? constructorArg() : optionalConstructorArg(), new ParseField("vegetable"));
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index 46c0ba35723..84c9518051c 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -18,11 +18,9 @@
  */
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.AbstractObjectParser.NoContextParser;
 import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -39,8 +37,6 @@ import static org.hamcrest.Matchers.hasSize;
 
 public class ObjectParserTests extends ESTestCase {
 
-    private static final ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT;
-
     public void testBasics() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, 
                   "{\n"
@@ -60,13 +56,13 @@ public class ObjectParserTests extends ESTestCase {
                 this.ints = ints;
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
 
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
         objectParser.declareInt(TestStruct::setTestNumber, new ParseField("test_number"));
         objectParser.declareIntArray(TestStruct::setInts, new ParseField("test_array"));
-        objectParser.parse(parser, s, STRICT_PARSING);
+        objectParser.parse(parser, s, null);
         assertEquals(s.test, "foo");
         assertEquals(s.testNumber, 2);
         assertEquals(s.ints, Arrays.asList(1, 2, 3, 4));
@@ -77,16 +73,16 @@ public class ObjectParserTests extends ESTestCase {
     }
 
     public void testNullDeclares() {
-        ObjectParser<Void, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<Void, Void> objectParser = new ObjectParser<>("foo");
         Exception e = expectThrows(IllegalArgumentException.class,
                 () -> objectParser.declareField(null, (r, c) -> null, new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[consumer] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
-                (o, v) -> {}, (ContextParser<ParseFieldMatcherSupplier, Object>) null,
+                (o, v) -> {}, (ContextParser<Void, Object>) null,
                 new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[parser] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
-                (o, v) -> {}, (NoContextParser<Object>) null,
+                (o, v) -> {}, (CheckedFunction<XContentParser, Object, IOException>) null,
                 new ParseField("test"), ObjectParser.ValueType.STRING));
         assertEquals("[parser] is required", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> objectParser.declareField(
@@ -99,17 +95,17 @@ public class ObjectParserTests extends ESTestCase {
 
     public void testObjectOrDefault() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : { \"test\": 2}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test"));
         objectParser.declareObjectOrDefault(StaticTestStruct::setObject, objectParser, StaticTestStruct::new, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertEquals(s.object.test, 2);
         parser = createParser(JsonXContent.jsonXContent, "{\"object\" : false }");
-        s = objectParser.parse(parser, STRICT_PARSING);
+        s = objectParser.parse(parser, null);
         assertNull(s.object);
 
         parser = createParser(JsonXContent.jsonXContent, "{\"object\" : true }");
-        s = objectParser.parse(parser, STRICT_PARSING);
+        s = objectParser.parse(parser, null);
         assertNotNull(s.object);
         assertEquals(s.object.test, 0);
 
@@ -159,19 +155,14 @@ public class ObjectParserTests extends ESTestCase {
             }
         }
 
-        class CustomParseFieldMatchSupplier implements ParseFieldMatcherSupplier {
+        class CustomParseContext {
 
             public final ClassicParser parser;
 
-            CustomParseFieldMatchSupplier(ClassicParser parser) {
+            CustomParseContext(ClassicParser parser) {
                 this.parser = parser;
             }
 
-            @Override
-            public ParseFieldMatcher getParseFieldMatcher() {
-                return ParseFieldMatcher.EMPTY;
-            }
-
             public URI parseURI(XContentParser parser) {
                 try {
                     return this.parser.parseURI(parser);
@@ -180,12 +171,12 @@ public class ObjectParserTests extends ESTestCase {
                 }
             }
         }
-        XContentParser parser = XContentType.JSON.xContent()
-            .createParser("{\"url\" : { \"host\": \"http://foobar\", \"port\" : 80}, \"name\" : \"foobarbaz\"}");
-        ObjectParser<Foo, CustomParseFieldMatchSupplier> objectParser = new ObjectParser<>("foo");
+        XContentParser parser = createParser(JsonXContent.jsonXContent,
+                "{\"url\" : { \"host\": \"http://foobar\", \"port\" : 80}, \"name\" : \"foobarbaz\"}");
+        ObjectParser<Foo, CustomParseContext> objectParser = new ObjectParser<>("foo");
         objectParser.declareString(Foo::setName, new ParseField("name"));
         objectParser.declareObjectOrDefault(Foo::setURI, (p, s) -> s.parseURI(p), () -> null, new ParseField("url"));
-        Foo s = objectParser.parse(parser, new Foo(), new CustomParseFieldMatchSupplier(new ClassicParser()));
+        Foo s = objectParser.parse(parser, new Foo(), new CustomParseContext(new ClassicParser()));
         assertEquals(s.uri.getHost(),  "foobar");
         assertEquals(s.uri.getPort(),  80);
         assertEquals(s.name, "foobarbaz");
@@ -197,12 +188,12 @@ public class ObjectParserTests extends ESTestCase {
             public void setTest(int test) {
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("the_parser");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("the_parser");
         TestStruct s = new TestStruct();
         objectParser.declareInt(TestStruct::setTest, new ParseField("test"));
 
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("numeric value expected");
         } catch (ParsingException ex) {
             assertEquals(ex.getMessage(), "[the_parser] failed to parse field [test]");
@@ -211,34 +202,24 @@ public class ObjectParserTests extends ESTestCase {
 
         parser = createParser(JsonXContent.jsonXContent, "{\"not_supported_field\" : \"foo\"}");
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("field not supported");
         } catch (IllegalArgumentException ex) {
             assertEquals(ex.getMessage(), "[the_parser] unknown field [not_supported_field], parser not found");
         }
     }
 
-    public void testDeprecationFail() throws IOException {
-        XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"old_test\" : \"foo\"}");
+    public void testDeprecationWarnings() throws IOException {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
-
+        XContentParser parser = createParser(XContentType.JSON.xContent(), "{\"old_test\" : \"foo\"}");
         objectParser.declareField((i, v, c) -> v.test = i.text(), new ParseField("test", "old_test"), ObjectParser.ValueType.STRING);
-
-        try {
-            objectParser.parse(parser, s, STRICT_PARSING);
-            fail("deprecated value");
-        } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Deprecated field [old_test] used, expected [test] instead");
-
-        }
-        assertNull(s.test);
-        parser = createParser(JsonXContent.jsonXContent, "{\"old_test\" : \"foo\"}");
-        objectParser.parse(parser, s, () -> ParseFieldMatcher.EMPTY);
+        objectParser.parse(parser, s, null);
         assertEquals("foo", s.test);
+        assertWarnings("Deprecated field [old_test] used, expected [test] instead");
     }
 
     public void testFailOnValueType() throws IOException {
@@ -246,12 +227,12 @@ public class ObjectParserTests extends ESTestCase {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
 
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("numeric_value"), ObjectParser.ValueType.FLOAT);
         try {
-            objectParser.parse(parser, s, STRICT_PARSING);
+            objectParser.parse(parser, s, null);
             fail("wrong type - must be number");
         } catch (IllegalArgumentException ex) {
             assertEquals(ex.getMessage(), "[foo] numeric_value doesn't support values of type: VALUE_BOOLEAN");
@@ -264,40 +245,40 @@ public class ObjectParserTests extends ESTestCase {
             public int test;
             TestStruct object;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         TestStruct s = new TestStruct();
         s.object = new TestStruct();
         objectParser.declareField((i, c, x) -> c.test = i.intValue(), new ParseField("test"), ValueType.INT);
-        objectParser.declareField((i, c, x) -> objectParser.parse(parser, c.object, STRICT_PARSING), new ParseField("object"),
+        objectParser.declareField((i, c, x) -> objectParser.parse(parser, c.object, null), new ParseField("object"),
                 ValueType.OBJECT);
-        objectParser.parse(parser, s, STRICT_PARSING);
+        objectParser.parse(parser, s, null);
         assertEquals(s.test, 1);
         assertEquals(s.object.test, 2);
     }
 
     public void testParseNestedShortcut() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{ \"test\" : 1, \"object\" : { \"test\": 2}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareInt(StaticTestStruct::setTest, new ParseField("test"));
         objectParser.declareObject(StaticTestStruct::setObject, objectParser, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertEquals(s.test, 1);
         assertEquals(s.object.test, 2);
     }
 
     public void testEmptyObject() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object\" : {}}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareObject(StaticTestStruct::setObject, objectParser, new ParseField("object"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertNotNull(s.object);
     }
 
     public void testEmptyObjectInArray() throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"object_array\" : [{}]}");
-        ObjectParser<StaticTestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
+        ObjectParser<StaticTestStruct, Void> objectParser = new ObjectParser<>("foo", StaticTestStruct::new);
         objectParser.declareObjectArray(StaticTestStruct::setObjectArray, objectParser, new ParseField("object_array"));
-        StaticTestStruct s = objectParser.parse(parser, STRICT_PARSING);
+        StaticTestStruct s = objectParser.parse(parser, null);
         assertNotNull(s.objectArray);
     }
 
@@ -332,9 +313,9 @@ public class ObjectParserTests extends ESTestCase {
             }
         }
         XContentParser parser = createParser(JsonXContent.jsonXContent, "{ \"test\" : \"FOO\" }");
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         objectParser.declareString((struct, value) -> struct.set(TestEnum.valueOf(value)), new ParseField("test"));
-        TestStruct s = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
         assertEquals(s.test, TestEnum.FOO);
     }
 
@@ -428,7 +409,7 @@ public class ObjectParserTests extends ESTestCase {
                 this.string_or_null = string_or_null;
             }
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo");
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo");
         objectParser.declareInt(TestStruct::setInt_field, new ParseField("int_field"));
         objectParser.declareIntArray(TestStruct::setInt_array_field, new ParseField("int_array_field"));
         objectParser.declareLong(TestStruct::setLong_field, new ParseField("long_field"));
@@ -442,7 +423,7 @@ public class ObjectParserTests extends ESTestCase {
 
         objectParser.declareStringOrNull(TestStruct::setString_or_null, new ParseField("string_or_null"));
         objectParser.declareBoolean(TestStruct::setNull_value, new ParseField("boolean_field"));
-        TestStruct parse = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct parse = objectParser.parse(parser, new TestStruct(), null);
         assertArrayEquals(parse.double_array_field.toArray(), Arrays.asList(2.1d).toArray());
         assertEquals(parse.double_field, 2.1d, 0.0d);
 
@@ -471,7 +452,7 @@ public class ObjectParserTests extends ESTestCase {
                   "{\"named\": {\n"
                 + "  \"a\": {}"
                 + "}}");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertFalse(h.namedSuppliedInOrder);
@@ -482,7 +463,7 @@ public class ObjectParserTests extends ESTestCase {
                   "{\"named\": [\n"
                 + "  {\"a\": {}}"
                 + "]}");
-        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING);
+        NamedObjectHolder h = NamedObjectHolder.PARSER.apply(parser, null);
         assertThat(h.named, hasSize(1));
         assertEquals("a", h.named.get(0).name);
         assertTrue(h.namedSuppliedInOrder);
@@ -493,7 +474,7 @@ public class ObjectParserTests extends ESTestCase {
                   "{\"named\": [\n"
                 + "  {\"a\": {}, \"b\": {}}"
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -505,7 +486,7 @@ public class ObjectParserTests extends ESTestCase {
                   "{\"named\": [\n"
                 + "  {}"
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -517,7 +498,7 @@ public class ObjectParserTests extends ESTestCase {
                   "{\"named\": [\n"
                 + "  \"junk\""
                 + "]}");
-        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> NamedObjectHolder.PARSER.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals(
                 "[named] can be a single object with any number of fields or an array where each entry is an object with a single field",
@@ -531,12 +512,12 @@ public class ObjectParserTests extends ESTestCase {
                 + "]}");
 
         // Create our own parser for this test so we can disable support for the "ordered" mode specified by the array above
-        ObjectParser<NamedObjectHolder, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("named_object_holder",
+        ObjectParser<NamedObjectHolder, Void> objectParser = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         objectParser.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, new ParseField("named"));
 
         // Now firing the xml through it fails
-        ParsingException e = expectThrows(ParsingException.class, () -> objectParser.apply(parser, STRICT_PARSING));
+        ParsingException e = expectThrows(ParsingException.class, () -> objectParser.apply(parser, null));
         assertEquals("[named_object_holder] failed to parse field [named]", e.getMessage());
         assertEquals("[named] doesn't support arrays. Use a single object with multiple fields.", e.getCause().getMessage());
     }
@@ -555,9 +536,9 @@ public class ObjectParserTests extends ESTestCase {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", true, null);
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
-        TestStruct s = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
         assertEquals(s.test, "foo");
     }
 
@@ -579,9 +560,9 @@ public class ObjectParserTests extends ESTestCase {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", true, null);
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
-        TestStruct s = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
         assertEquals(s.test, "foo");
     }
 
@@ -606,14 +587,14 @@ public class ObjectParserTests extends ESTestCase {
         class TestStruct {
             public String test;
         }
-        ObjectParser<TestStruct, ParseFieldMatcherSupplier> objectParser = new ObjectParser<>("foo", true, null);
+        ObjectParser<TestStruct, Void> objectParser = new ObjectParser<>("foo", true, null);
         objectParser.declareField((i, c, x) -> c.test = i.text(), new ParseField("test"), ObjectParser.ValueType.STRING);
-        TestStruct s = objectParser.parse(parser, new TestStruct(), STRICT_PARSING);
+        TestStruct s = objectParser.parse(parser, new TestStruct(), null);
         assertEquals(s.test, "foo");
     }
 
     static class NamedObjectHolder {
-        public static final ObjectParser<NamedObjectHolder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("named_object_holder",
+        public static final ObjectParser<NamedObjectHolder, Void> PARSER = new ObjectParser<>("named_object_holder",
                 NamedObjectHolder::new);
         static {
             PARSER.declareNamedObjects(NamedObjectHolder::setNamed, NamedObject.PARSER, NamedObjectHolder::keepNamedInOrder,
@@ -633,11 +614,11 @@ public class ObjectParserTests extends ESTestCase {
     }
 
     public static class NamedObject {
-        public static final NamedObjectParser<NamedObject, ParseFieldMatcherSupplier> PARSER;
+        public static final NamedObjectParser<NamedObject, Void> PARSER;
         static {
-            ObjectParser<NamedObject, ParseFieldMatcherSupplier> parser = new ObjectParser<>("named");
+            ObjectParser<NamedObject, Void> parser = new ObjectParser<>("named");
             parser.declareInt(NamedObject::setFoo, new ParseField("foo"));
-            PARSER = (XContentParser p, ParseFieldMatcherSupplier v, String name) -> parser.parse(p, new NamedObject(name), STRICT_PARSING);
+            PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new NamedObject(name), null);
         }
 
         final String name;
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java
new file mode 100644
index 00000000000..5ed3b0cf8c2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/UnknownNamedObjectExceptionTests.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.xcontent;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class UnknownNamedObjectExceptionTests extends ESTestCase {
+    public void testRoundTrip() throws IOException {
+        XContentLocation location = new XContentLocation(between(1, 1000), between(1, 1000));
+        UnknownNamedObjectException created = new UnknownNamedObjectException(location, UnknownNamedObjectExceptionTests.class,
+                randomAsciiOfLength(5));
+        UnknownNamedObjectException roundTripped;
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            created.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                roundTripped = new UnknownNamedObjectException(in);
+            }
+        }
+        assertEquals(created.getMessage(), roundTripped.getMessage());
+        assertEquals(created.getLineNumber(), roundTripped.getLineNumber());
+        assertEquals(created.getColumnNumber(), roundTripped.getColumnNumber());
+        assertEquals(created.getCategoryClass(), roundTripped.getCategoryClass());
+        assertEquals(created.getName(), roundTripped.getName());
+    }
+
+    public void testStatusCode() {
+        XContentLocation location = new XContentLocation(between(1, 1000), between(1, 1000));
+        UnknownNamedObjectException e = new UnknownNamedObjectException(location, UnknownNamedObjectExceptionTests.class,
+                randomAsciiOfLength(5));
+        assertEquals(RestStatus.BAD_REQUEST, e.status());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
index a7257fbca8b..02fd68890a6 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserTests.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.common.xcontent;
 
+import com.fasterxml.jackson.core.JsonParseException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -32,6 +34,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.isIn;
 import static org.hamcrest.Matchers.nullValue;
 
 public class XContentParserTests extends ESTestCase {
@@ -102,4 +105,92 @@ public class XContentParserTests extends ESTestCase {
             return randomBoolean() ? parser.mapStringsOrdered() : parser.mapStrings();
         }
     }
+
+    @SuppressWarnings("deprecation") // #isBooleanValueLenient() and #booleanValueLenient() are the test subjects
+    public void testReadLenientBooleans() throws IOException {
+        // allow String, boolean and int representations of lenient booleans
+        String falsy = randomFrom("\"off\"", "\"no\"", "\"0\"", "0", "\"false\"", "false");
+        String truthy = randomFrom("\"on\"", "\"yes\"", "\"1\"", "1", "\"true\"", "true");
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"foo\": " + falsy + ", \"bar\": " + truthy + "}")) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            token = parser.nextToken();
+            assertThat(token, isIn(
+                Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValueLenient());
+            assertFalse(parser.booleanValueLenient());
+
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("bar"));
+            token = parser.nextToken();
+            assertThat(token, isIn(
+                Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_NUMBER, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValueLenient());
+            assertTrue(parser.booleanValueLenient());
+        }
+    }
+
+    public void testReadBooleansFailsForLenientBooleans() throws IOException {
+        String falsy = randomFrom("\"off\"", "\"no\"", "\"0\"", "0");
+        String truthy = randomFrom("\"on\"", "\"yes\"", "\"1\"", "1");
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"foo\": " + falsy + ", \"bar\": " + truthy + "}")) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_NUMBER)));
+            assertFalse(parser.isBooleanValue());
+            if (token.equals(XContentParser.Token.VALUE_STRING)) {
+                expectThrows(IllegalArgumentException.class, parser::booleanValue);
+            } else {
+                expectThrows(JsonParseException.class, parser::booleanValue);
+            }
+
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("bar"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_NUMBER)));
+            assertFalse(parser.isBooleanValue());
+            if (token.equals(XContentParser.Token.VALUE_STRING)) {
+                expectThrows(IllegalArgumentException.class, parser::booleanValue);
+            } else {
+                expectThrows(JsonParseException.class, parser::booleanValue);
+            }
+        }
+    }
+
+    public void testReadBooleans() throws IOException {
+        String falsy = randomFrom("\"false\"", "false");
+        String truthy = randomFrom("\"true\"", "true");
+
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"foo\": " + falsy + ", \"bar\": " + truthy + "}")) {
+            XContentParser.Token token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.START_OBJECT));
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("foo"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValue());
+            assertFalse(parser.booleanValue());
+
+            token = parser.nextToken();
+            assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
+            assertThat(parser.currentName(), equalTo("bar"));
+            token = parser.nextToken();
+            assertThat(token, isIn(Arrays.asList(XContentParser.Token.VALUE_STRING, XContentParser.Token.VALUE_BOOLEAN)));
+            assertTrue(parser.isBooleanValue());
+            assertTrue(parser.booleanValue());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
index bec1a0e4fb5..00edb25d5c3 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentParserUtilsTests.java
@@ -20,62 +20,24 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 public class XContentParserUtilsTests extends ESTestCase {
-
-    private XContentType xContentType;
-
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        xContentType = randomFrom(XContentType.values());
-    }
-
-    public void testEnsureFieldName() throws IOException {
-        ParsingException e = expectThrows(ParsingException.class, () -> {
-            XContentParser parser = createParser(createBuilder().startObject().endObject().bytes());
+    public void testEnsureExpectedToken() throws IOException {
+        final XContentParser.Token randomToken = randomFrom(XContentParser.Token.values());
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{}")) {
             // Parser current token is null
             assertNull(parser.currentToken());
-            XContentParserUtils.ensureFieldName(parser.currentToken(), parser::getTokenLocation);
-        });
-        assertThat(e.getMessage(), equalTo("Failed to parse object: expecting token of type [FIELD_NAME] but found [null]"));
-
-        e = expectThrows(ParsingException.class, () -> {
-            XContentParser parser = createParser(createBuilder().startObject().field("foo", "bar").endObject().bytes());
-            // Parser next token is a start object
-            XContentParserUtils.ensureFieldName(parser.nextToken(), parser::getTokenLocation);
-        });
-        assertThat(e.getMessage(), equalTo("Failed to parse object: expecting token of type [FIELD_NAME] but found [START_OBJECT]"));
-
-        e = expectThrows(ParsingException.class, () -> {
-            XContentParser parser = createParser(createBuilder().startObject().field("foo", "bar").endObject().bytes());
-            // Moves to start object
-            assertThat(parser.nextToken(), is(XContentParser.Token.START_OBJECT));
-            // Expected field name is "foo", not "test"
-            XContentParserUtils.ensureFieldName(parser, parser.nextToken(), "test");
-        });
-        assertThat(e.getMessage(), equalTo("Failed to parse object: expecting field with name [test] but found [foo]"));
-
-        // Everything is fine
-        final String randomFieldName = randomAsciiOfLength(5);
-        XContentParser parser = createParser(createBuilder().startObject().field(randomFieldName, 0).endObject().bytes());
-        assertThat(parser.nextToken(), is(XContentParser.Token.START_OBJECT));
-        XContentParserUtils.ensureFieldName(parser, parser.nextToken(), randomFieldName);
-    }
-
-    private XContentBuilder createBuilder() throws IOException {
-        return XContentBuilder.builder(xContentType.xContent());
-    }
-
-    private XContentParser createParser(BytesReference bytes) throws IOException {
-        return xContentType.xContent().createParser(bytes);
+            ParsingException e = expectThrows(ParsingException.class,
+                    () -> ensureExpectedToken(randomToken, parser.currentToken(), parser::getTokenLocation));
+            assertEquals("Failed to parse object: expecting token of type [" + randomToken + "] but found [null]", e.getMessage());
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
index a8091fc1122..219bd12daea 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentHelperTests.java
@@ -19,10 +19,17 @@
 
 package org.elasticsearch.common.xcontent.support;
 
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -61,5 +68,39 @@ public class XContentHelperTests extends ESTestCase {
         assertThat(content, Matchers.equalTo(expected));
     }
 
-
+    public void testToXContent() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final ToXContent toXContent;
+        final boolean error;
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (builder, params) -> builder.field("field", "value");
+            } else {
+                error = true;
+                toXContent = (builder, params) -> builder.startObject().field("field", "value").endObject();
+            }
+        } else {
+            if (randomBoolean()) {
+                error = false;
+                toXContent = (ToXContentObject) (builder, params) -> builder.startObject().field("field", "value").endObject();
+            } else {
+                error = true;
+                toXContent = (ToXContentObject) (builder, params) -> builder.field("field", "value");
+            }
+        }
+        if (error) {
+            expectThrows(IOException.class, () -> XContentHelper.toXContent(toXContent, xContentType, randomBoolean()));
+        } else {
+            BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, randomBoolean());
+            try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes)) {
+                assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+                assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+                assertTrue(parser.nextToken().isValue());
+                assertEquals("value", parser.text());
+                assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+                assertNull(parser.nextToken());
+            }
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
index 58c4f630bb6..3a1144afe93 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java
@@ -41,6 +41,7 @@ import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.core.IsEqual.equalTo;
@@ -148,8 +149,8 @@ public class XContentMapValuesTests extends ESTestCase {
         extValue = XContentMapValues.extractValue("path1.test", map);
         assertThat(extValue, instanceOf(List.class));
 
-        List extListValue = (List) extValue;
-        assertThat(extListValue.size(), equalTo(2));
+        List<?> extListValue = (List) extValue;
+        assertThat(extListValue, hasSize(2));
 
         builder = XContentFactory.jsonBuilder().startObject()
                 .startObject("path1")
@@ -168,7 +169,7 @@ public class XContentMapValuesTests extends ESTestCase {
         assertThat(extValue, instanceOf(List.class));
 
         extListValue = (List) extValue;
-        assertThat(extListValue.size(), equalTo(2));
+        assertThat(extListValue, hasSize(2));
         assertThat(extListValue.get(0).toString(), equalTo("value1"));
         assertThat(extListValue.get(1).toString(), equalTo("value2"));
 
@@ -234,9 +235,9 @@ public class XContentMapValuesTests extends ESTestCase {
         Map<String, Object> map = new HashMap<>();
         map.put("obj", "value");
         map.put("obj_name", "value_name");
-        Map<String, Object> filterdMap = XContentMapValues.filter(map, new String[]{"obj_name"}, Strings.EMPTY_ARRAY);
-        assertThat(filterdMap.size(), equalTo(1));
-        assertThat((String) filterdMap.get("obj_name"), equalTo("value_name"));
+        Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[]{"obj_name"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat((String) filteredMap.get("obj_name"), equalTo("value_name"));
     }
 
 
@@ -251,19 +252,17 @@ public class XContentMapValuesTests extends ESTestCase {
                             put("nested", 2);
                             put("nested_2", 3);
                         }}));
-        Map<String, Object> falteredMap = XContentMapValues.filter(map, new String[]{"array.nested"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
+        Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[]{"array.nested"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
 
-        // Selecting members of objects within arrays (ex. [ 1, { nested: "value"} ])  always returns all values in the array (1 in the ex)
-        // this is expected behavior as this types of objects are not supported in ES
-        assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
-        assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(1));
-        assertThat((Integer) ((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).get("nested"), equalTo(2));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(1));
+        assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).size(), equalTo(1));
+        assertThat((Integer) ((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).get("nested"), equalTo(2));
 
-        falteredMap = XContentMapValues.filter(map, new String[]{"array.*"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
-        assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(2));
+        filteredMap = XContentMapValues.filter(map, new String[]{"array.*"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(1));
+        assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(0)).size(), equalTo(2));
 
         map.clear();
         map.put("field", "value");
@@ -272,16 +271,16 @@ public class XContentMapValuesTests extends ESTestCase {
                     put("field", "value");
                     put("field2", "value2");
                 }});
-        falteredMap = XContentMapValues.filter(map, new String[]{"obj.field"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(1));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
+        filteredMap = XContentMapValues.filter(map, new String[]{"obj.field"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field"), equalTo("value"));
 
-        falteredMap = XContentMapValues.filter(map, new String[]{"obj.*"}, Strings.EMPTY_ARRAY);
-        assertThat(falteredMap.size(), equalTo(1));
-        assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(2));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
-        assertThat((String) ((Map<String, Object>) falteredMap.get("obj")).get("field2"), equalTo("value2"));
+        filteredMap = XContentMapValues.filter(map, new String[]{"obj.*"}, Strings.EMPTY_ARRAY);
+        assertThat(filteredMap.size(), equalTo(1));
+        assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field"), equalTo("value"));
+        assertThat((String) ((Map<String, Object>) filteredMap.get("obj")).get("field2"), equalTo("value2"));
 
     }
 
@@ -323,7 +322,7 @@ public class XContentMapValuesTests extends ESTestCase {
 
         filteredMap = XContentMapValues.filter(map, new String[]{"array"}, new String[]{"*.field2"});
         assertThat(filteredMap.size(), equalTo(1));
-        assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
+        assertThat(((List<?>) filteredMap.get("array")), hasSize(2));
         assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
         assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(1));
         assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).get("field").toString(), equalTo("value"));
@@ -386,7 +385,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj"}, Strings.EMPTY_ARRAY);
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -398,7 +397,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"nonExistingField"});
 
         assertThat(mapTuple.v2(), equalTo(filteredSource));
@@ -411,7 +410,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"obj.f1"});
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -431,25 +430,25 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject();
 
         // implicit include
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[]{"*.obj2"});
 
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
-        assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
+        assertThat(((Map) filteredSource.get("obj1")).size(), equalTo(0));
 
         // explicit include
         filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"obj1"}, new String[]{"*.obj2"});
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
-        assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
+        assertThat(((Map) filteredSource.get("obj1")).size(), equalTo(0));
 
         // wild card include
         filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, new String[]{"*.obj3"});
         assertThat(filteredSource.size(), equalTo(1));
         assertThat(filteredSource, hasKey("obj1"));
         assertThat(((Map<String, Object>) filteredSource.get("obj1")), hasKey("obj2"));
-        assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), Matchers.equalTo(0));
+        assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), equalTo(0));
     }
 
     @SuppressWarnings({"unchecked"})
@@ -461,7 +460,7 @@ public class XContentMapValuesTests extends ESTestCase {
                 .endObject()
                 .endObject();
 
-        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true);
+        Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true, builder.contentType());
         Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[]{"*.obj2"}, Strings.EMPTY_ARRAY);
 
         assertThat(filteredSource.size(), equalTo(1));
@@ -589,4 +588,15 @@ public class XContentMapValuesTests extends ESTestCase {
         assertEquals(Collections.singletonMap("foobar", 2), XContentMapValues.filter(map, new String[] {"foobar"}, new String[0]));
         assertEquals(Collections.singletonMap("foobaz", 3), XContentMapValues.filter(map, new String[0], new String[] {"foobar"}));
     }
+
+    public void testPrefix() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("photos", Arrays.asList(new String[] {"foo", "bar"}));
+        map.put("photosCount", 2);
+
+        Map<String, Object> filtered = XContentMapValues.filter(map, new String[] {"photosCount"}, new String[0]);
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("photosCount", 2);
+        assertEquals(expected, filtered);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
index 264e6ed0289..6d0ee8a9782 100644
--- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
@@ -47,7 +47,7 @@ public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase
         final Logger logger;
         final BlockingClusterStatePublishResponseHandler handler;
 
-        public PublishResponder(boolean fail, DiscoveryNode node, CyclicBarrier barrier, Logger logger, BlockingClusterStatePublishResponseHandler handler) {
+        PublishResponder(boolean fail, DiscoveryNode node, CyclicBarrier barrier, Logger logger, BlockingClusterStatePublishResponseHandler handler) {
             this.fail = fail;
 
             this.node = node;
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
index a2001504f19..378557b4047 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
@@ -31,6 +31,7 @@ import java.util.function.Supplier;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -55,6 +56,7 @@ import static org.mockito.Mockito.when;
 public class DiscoveryModuleTests extends ESTestCase {
 
     private TransportService transportService;
+    private NamedWriteableRegistry namedWriteableRegistry;
     private ClusterService clusterService;
     private ThreadPool threadPool;
 
@@ -71,6 +73,7 @@ public class DiscoveryModuleTests extends ESTestCase {
         Map<String, Supplier<Discovery>> impl();
         @Override
         default Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                                   NamedWriteableRegistry namedWriteableRegistry,
                                                                    ClusterService clusterService, UnicastHostsProvider hostsProvider) {
             return impl();
         }
@@ -80,6 +83,7 @@ public class DiscoveryModuleTests extends ESTestCase {
     public void setupDummyServices() {
         transportService = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, null, null);
         clusterService = mock(ClusterService.class);
+        namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         threadPool = mock(ThreadPool.class);
@@ -91,7 +95,7 @@ public class DiscoveryModuleTests extends ESTestCase {
     }
 
     private DiscoveryModule newModule(Settings settings, List<DiscoveryPlugin> plugins) {
-        return new DiscoveryModule(settings, threadPool, transportService, null, clusterService, plugins);
+        return new DiscoveryModule(settings, threadPool, transportService, namedWriteableRegistry, null, clusterService, plugins);
     }
 
     public void testDefaults() {
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 90d7e5e5a9e..5e2869a658e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -24,6 +24,7 @@ import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
@@ -47,6 +48,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.FaultDetection;
 import org.elasticsearch.discovery.zen.MembershipAction;
@@ -55,6 +57,7 @@ import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.discovery.zen.ZenPing;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.indices.store.IndicesStoreIntegrationIT;
 import org.elasticsearch.monitor.jvm.HotThreads;
 import org.elasticsearch.plugins.Plugin;
@@ -82,8 +85,6 @@ import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.TcpTransport;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -479,8 +480,9 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      * <p>
      * This test is a superset of tests run in the Jepsen test suite, with the exception of versioned updates
      */
-    @TestLogging("_root:DEBUG,org.elasticsearch.action.index:TRACE,org.elasticsearch.action.get:TRACE,discovery:TRACE,org.elasticsearch.cluster.service:TRACE,"
-        + "org.elasticsearch.indices.recovery:TRACE,org.elasticsearch.indices.cluster:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.get:TRACE,discovery:TRACE," +
+        "org.elasticsearch.cluster.service:TRACE,org.elasticsearch.indices.recovery:TRACE," +
+        "org.elasticsearch.indices.cluster:TRACE,org.elasticsearch.index.shard:TRACE")
     public void testAckedIndexing() throws Exception {
 
         final int seconds = !(TEST_NIGHTLY && rarely()) ? 1 : 5;
@@ -492,6 +494,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             .setSettings(Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1 + randomInt(2))
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
+                .put(IndexSettings.INDEX_SEQ_NO_CHECKPOINT_SYNC_INTERVAL.getKey(),  randomBoolean() ? "5s" : "200ms")
             ));
         ensureGreen();
 
@@ -528,7 +531,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                                 int shard = Math.floorMod(Murmur3HashFunction.hash(id), numPrimaries);
                                 logger.trace("[{}] indexing id [{}] through node [{}] targeting shard [{}]", name, id, node, shard);
                                 IndexResponse response =
-                                    client.prepareIndex("test", "type", id).setSource("{}").setTimeout(timeout).get(timeout);
+                                    client.prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).setTimeout(timeout).get(timeout);
                                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
                                 ackedDocs.put(id, node);
                                 logger.trace("[{}] indexed id [{}] through node [{}]", name, id, node);
@@ -587,17 +590,19 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 ensureGreen("test");
 
                 logger.info("validating successful docs");
-                for (String node : nodes) {
-                    try {
-                        logger.debug("validating through node [{}] ([{}] acked docs)", node, ackedDocs.size());
-                        for (String id : ackedDocs.keySet()) {
-                            assertTrue("doc [" + id + "] indexed via node [" + ackedDocs.get(id) + "] not found",
-                                client(node).prepareGet("test", "type", id).setPreference("_local").get().isExists());
+                assertBusy(() -> {
+                    for (String node : nodes) {
+                        try {
+                            logger.debug("validating through node [{}] ([{}] acked docs)", node, ackedDocs.size());
+                            for (String id : ackedDocs.keySet()) {
+                                assertTrue("doc [" + id + "] indexed via node [" + ackedDocs.get(id) + "] not found",
+                                    client(node).prepareGet("test", "type", id).setPreference("_local").get().isExists());
+                            }
+                        } catch (AssertionError | NoShardAvailableActionException e) {
+                            throw new AssertionError(e.getMessage() + " (checked via node [" + node + "]", e);
                         }
-                    } catch (AssertionError e) {
-                        throw new AssertionError(e.getMessage() + " (checked via node [" + node + "]", e);
                     }
-                }
+                }, 30, TimeUnit.SECONDS);
 
                 logger.info("done validating (iteration [{}])", iter);
             }
@@ -1118,7 +1123,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         final String node_2 = internalCluster().startDataOnlyNode();
         List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
         for (int i = 0; i < 100; i++) {
-            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc").setSource("{\"int_field\":1}"));
+            indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("doc")
+                .setSource("{\"int_field\":1}", XContentType.JSON));
         }
         indexRandom(true, indexRequestBuilderList);
         SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_2, random());
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index f089f76dcff..49af88f5331 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -304,7 +304,7 @@ public class ZenFaultDetectionTests extends ESTestCase {
         private final Set<Tuple<DiscoveryNode, Long>> completedPings = Collections.newSetFromMap(new ConcurrentHashMap<>());
         private final CountDownLatch waitForPings;
 
-        public PingProbe(int minCompletedPings) {
+        PingProbe(int minCompletedPings) {
             this.waitForPings = new CountDownLatch(minCompletedPings);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
index 0c4862bf32a..f59cc1cc6fb 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
@@ -66,7 +66,8 @@ public class ElectMasterServiceTests extends ESTestCase {
             roles.add(DiscoveryNode.Role.MASTER);
             DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, buildNewFakeTransportAddress(), Collections.emptyMap(),
                 roles, Version.CURRENT);
-            candidates.add(new MasterCandidate(node, randomBoolean() ? MasterCandidate.UNRECOVERED_CLUSTER_VERSION : randomPositiveLong()));
+            candidates.add(
+                new MasterCandidate(node, randomBoolean() ? MasterCandidate.UNRECOVERED_CLUSTER_VERSION : randomNonNegativeLong()));
         }
 
         Collections.shuffle(candidates, random());
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index bd9e3db88b4..c5c78189ee1 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -117,8 +117,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
         nodeJoinController = new NodeJoinController(clusterService, createAllocationService(Settings.EMPTY),
-            new ElectMasterService(Settings.EMPTY), new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
-            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), Settings.EMPTY);
+            new ElectMasterService(Settings.EMPTY), Settings.EMPTY);
     }
 
     @After
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
index f135bb5eaa1..70b291fee60 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeRemovalClusterStateTaskExecutorTests.java
@@ -32,7 +32,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -67,7 +66,7 @@ public class NodeRemovalClusterStateTaskExecutorTests extends ESTestCase {
                         .map(node -> new ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task(node, randomBoolean() ? "left" : "failed"))
                         .collect(Collectors.toList());
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result
                 = executor.execute(clusterState, tasks);
         assertThat(result.resultingState, equalTo(clusterState));
     }
@@ -106,7 +105,7 @@ public class NodeRemovalClusterStateTaskExecutorTests extends ESTestCase {
         }
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
                 executor.execute(clusterState, tasks);
         verify(electMasterService).hasEnoughMasterNodes(eq(remainingNodesClusterState.get().nodes()));
         verifyNoMoreInteractions(electMasterService);
@@ -156,7 +155,7 @@ public class NodeRemovalClusterStateTaskExecutorTests extends ESTestCase {
         }
         final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(builder).build();
 
-        final ClusterStateTaskExecutor.BatchResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
+        final ClusterStateTaskExecutor.ClusterTasksResult<ZenDiscovery.NodeRemovalClusterStateTaskExecutor.Task> result =
                 executor.execute(clusterState, tasks);
         verify(electMasterService).hasEnoughMasterNodes(eq(remainingNodesClusterState.get().nodes()));
         verifyNoMoreInteractions(electMasterService);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
index e1d2a226a02..5b6668c6f5e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/PublishClusterStateActionTests.java
@@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -36,6 +37,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -248,9 +250,11 @@ public class PublishClusterStateActionTests extends ESTestCase {
     ) {
         DiscoverySettings discoverySettings =
                 new DiscoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         return new MockPublishAction(
                 settings,
                 transportService,
+                namedWriteableRegistry,
                 clusterStateSupplier,
                 listener,
                 discoverySettings,
@@ -873,10 +877,10 @@ public class PublishClusterStateActionTests extends ESTestCase {
         AtomicBoolean timeoutOnCommit = new AtomicBoolean();
         AtomicBoolean errorOnCommit = new AtomicBoolean();
 
-        public MockPublishAction(Settings settings, TransportService transportService,
+        public MockPublishAction(Settings settings, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry,
                                  Supplier<ClusterState> clusterStateSupplier, NewPendingClusterStateListener listener,
                                  DiscoverySettings discoverySettings, ClusterName clusterName) {
-            super(settings, transportService, clusterStateSupplier, listener, discoverySettings, clusterName);
+            super(settings, transportService, namedWriteableRegistry, clusterStateSupplier, listener, discoverySettings, clusterName);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
index de8d1a562e8..40309a38f28 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNode.Role;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -34,18 +35,22 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.ConnectionProfile;
 import org.elasticsearch.transport.MockTcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportConnectionListener;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 import org.junit.After;
@@ -60,6 +65,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -73,6 +79,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -82,7 +89,6 @@ import static java.util.Collections.emptySet;
 import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
@@ -109,6 +115,7 @@ public class UnicastZenPingTests extends ESTestCase {
     @After
     public void tearDown() throws Exception {
         try {
+            logger.info("shutting down...");
             // JDK stack is broken, it does not iterate in the expected order (http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4475301)
             final List<Closeable> reverse = new ArrayList<>();
             while (!closeables.isEmpty()) {
@@ -124,8 +131,7 @@ public class UnicastZenPingTests extends ESTestCase {
 
     private static final UnicastHostsProvider EMPTY_HOSTS_PROVIDER = Collections::emptyList;
 
-    @TestLogging("org.elasticsearch.transport:TRACE,org.elasticsearch.discovery.zen.UnicastZenPing:TRACE")
-    public void testSimplePings() throws IOException, InterruptedException {
+    public void testSimplePings() throws IOException, InterruptedException, ExecutionException {
         // use ephemeral ports
         final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
         final Settings settingsMismatch =
@@ -140,7 +146,12 @@ public class UnicastZenPingTests extends ESTestCase {
             new NoneCircuitBreakerService(),
             new NamedWriteableRegistry(Collections.emptyList()),
             networkService,
-            v);
+            v) {
+            @Override
+            public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile) {
+                throw new AssertionError("zen pings should never connect to node (got [" + node + "])");
+            }
+        };
 
         NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier);
         closeables.push(handleA.transportService);
@@ -148,25 +159,30 @@ public class UnicastZenPingTests extends ESTestCase {
         closeables.push(handleB.transportService);
         NetworkHandle handleC = startServices(settingsMismatch, threadPool, "UZP_C", Version.CURRENT, supplier);
         closeables.push(handleC.transportService);
-        // just fake that no versions are compatible with this node
-        Version previousVersion = VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
-        Version versionD = VersionUtils.randomVersionBetween(random(), previousVersion.minimumCompatibilityVersion(), previousVersion);
+        final Version versionD;
+        if (randomBoolean()) {
+            versionD = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
+        } else {
+            versionD = Version.CURRENT;
+        }
+        logger.info("UZP_D version set to [{}]", versionD);
         NetworkHandle handleD = startServices(settingsMismatch, threadPool, "UZP_D", versionD, supplier);
         closeables.push(handleD.transportService);
 
-        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+        final ClusterState stateMismatch = ClusterState.builder(new ClusterName("mismatch")).version(randomNonNegativeLong()).build();
 
         Settings hostsSettings = Settings.builder()
-                .putArray("discovery.zen.ping.unicast.hosts",
+            .putArray("discovery.zen.ping.unicast.hosts",
                 NetworkAddress.format(new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleD.address.address().getAddress(), handleD.address.address().getPort())))
-                .put("cluster.name", "test")
-                .build();
+            .put("cluster.name", "test")
+            .build();
 
         Settings hostsSettingsMismatch = Settings.builder().put(hostsSettings).put(settingsMismatch).build();
-        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, EMPTY_HOSTS_PROVIDER);
+        TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
         zenPingA.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -180,7 +196,7 @@ public class UnicastZenPingTests extends ESTestCase {
         });
         closeables.push(zenPingA);
 
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, EMPTY_HOSTS_PROVIDER);
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
         zenPingB.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -194,7 +210,8 @@ public class UnicastZenPingTests extends ESTestCase {
         });
         closeables.push(zenPingB);
 
-        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleC.transportService, EMPTY_HOSTS_PROVIDER) {
+        TestUnicastZenPing zenPingC = new TestUnicastZenPing(hostsSettingsMismatch, threadPool, handleC,
+            EMPTY_HOSTS_PROVIDER) {
             @Override
             protected Version getVersion() {
                 return versionD;
@@ -208,12 +225,13 @@ public class UnicastZenPingTests extends ESTestCase {
 
             @Override
             public ClusterState clusterState() {
-                return state;
+                return stateMismatch;
             }
         });
         closeables.push(zenPingC);
 
-        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleD.transportService, EMPTY_HOSTS_PROVIDER);
+        TestUnicastZenPing zenPingD = new TestUnicastZenPing(hostsSettingsMismatch, threadPool, handleD,
+            EMPTY_HOSTS_PROVIDER);
         zenPingD.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -222,40 +240,48 @@ public class UnicastZenPingTests extends ESTestCase {
 
             @Override
             public ClusterState clusterState() {
-                return state;
+                return stateMismatch;
             }
         });
         closeables.push(zenPingD);
 
         logger.info("ping from UZP_A");
-        Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait(TimeValue.timeValueMillis(500));
+        Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
         assertThat(pingResponses.size(), equalTo(1));
         ZenPing.PingResponse ping = pingResponses.iterator().next();
         assertThat(ping.node().getId(), equalTo("UZP_B"));
         assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
-        assertCountersMoreThan(handleA, handleB, handleC, handleD);
+        assertPingCount(handleA, handleB, 3);
+        assertPingCount(handleA, handleC, 0); // mismatch, shouldn't ping
+        assertPingCount(handleA, handleD, 0); // mismatch, shouldn't ping
 
         // ping again, this time from B,
         logger.info("ping from UZP_B");
-        pingResponses = zenPingB.pingAndWait(TimeValue.timeValueMillis(500));
+        pingResponses = zenPingB.pingAndWait().toList();
         assertThat(pingResponses.size(), equalTo(1));
         ping = pingResponses.iterator().next();
         assertThat(ping.node().getId(), equalTo("UZP_A"));
         assertThat(ping.getClusterStateVersion(), equalTo(ElectMasterService.MasterCandidate.UNRECOVERED_CLUSTER_VERSION));
-        assertCountersMoreThan(handleB, handleA, handleC, handleD);
+        assertPingCount(handleB, handleA, 3);
+        assertPingCount(handleB, handleC, 0); // mismatch, shouldn't ping
+        assertPingCount(handleB, handleD, 0); // mismatch, shouldn't ping
 
         logger.info("ping from UZP_C");
-        pingResponses = zenPingC.pingAndWait(TimeValue.timeValueMillis(500));
-        assertThat(pingResponses.size(), equalTo(0));
-        assertCountersMoreThan(handleC, handleA, handleB, handleD);
+        pingResponses = zenPingC.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        assertPingCount(handleC, handleA, 0);
+        assertPingCount(handleC, handleB, 0);
+        assertPingCount(handleC, handleD, 3);
 
         logger.info("ping from UZP_D");
-        pingResponses = zenPingD.pingAndWait(TimeValue.timeValueMillis(500));
-        assertThat(pingResponses.size(), equalTo(0));
-        assertCountersMoreThan(handleD, handleA, handleB, handleC);
+        pingResponses = zenPingD.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        assertPingCount(handleD, handleA, 0);
+        assertPingCount(handleD, handleB, 0);
+        assertPingCount(handleD, handleC, 3);
     }
 
-    public void testUnknownHostNotCached() {
+    public void testUnknownHostNotCached() throws ExecutionException, InterruptedException {
         // use ephemeral ports
         final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
 
@@ -304,9 +330,9 @@ public class UnicastZenPingTests extends ESTestCase {
             .put("cluster.name", "test")
             .build();
 
-        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
 
-        final UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, EMPTY_HOSTS_PROVIDER);
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
         zenPingA.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -320,7 +346,7 @@ public class UnicastZenPingTests extends ESTestCase {
         });
         closeables.push(zenPingA);
 
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, EMPTY_HOSTS_PROVIDER);
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
         zenPingB.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -334,7 +360,7 @@ public class UnicastZenPingTests extends ESTestCase {
         });
         closeables.push(zenPingB);
 
-        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettings, threadPool, handleC.transportService, EMPTY_HOSTS_PROVIDER);
+        TestUnicastZenPing zenPingC = new TestUnicastZenPing(hostsSettings, threadPool, handleC, EMPTY_HOSTS_PROVIDER);
         zenPingC.start(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -350,12 +376,13 @@ public class UnicastZenPingTests extends ESTestCase {
 
         // the presence of an unresolvable host should not prevent resolvable hosts from being pinged
         {
-            final Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait(TimeValue.timeValueMillis(500));
+            final Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
             assertThat(pingResponses.size(), equalTo(1));
             ZenPing.PingResponse ping = pingResponses.iterator().next();
             assertThat(ping.node().getId(), equalTo("UZP_C"));
             assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
-            assertCountersMoreThan(handleA, handleC);
+            assertPingCount(handleA, handleB, 0);
+            assertPingCount(handleA, handleC, 3);
             assertNull(handleA.counters.get(handleB.address));
         }
 
@@ -373,11 +400,13 @@ public class UnicastZenPingTests extends ESTestCase {
 
         // now we should see pings to UZP_B; this establishes that host resolutions are not cached
         {
-            final Collection<ZenPing.PingResponse> secondPingResponses = zenPingA.pingAndWait(TimeValue.timeValueMillis(500));
+            handleA.counters.clear();
+            final Collection<ZenPing.PingResponse> secondPingResponses = zenPingA.pingAndWait().toList();
             assertThat(secondPingResponses.size(), equalTo(2));
             final Set<String> ids = new HashSet<>(secondPingResponses.stream().map(p -> p.node().getId()).collect(Collectors.toList()));
             assertThat(ids, equalTo(new HashSet<>(Arrays.asList("UZP_B", "UZP_C"))));
-            assertCountersMoreThan(moreThan, handleA, handleB, handleC);
+            assertPingCount(handleA, handleB, 3);
+            assertPingCount(handleA, handleC, 3);
         }
     }
 
@@ -393,17 +422,16 @@ public class UnicastZenPingTests extends ESTestCase {
             Version.CURRENT);
         closeables.push(transport);
         final TransportService transportService =
-            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
         closeables.push(transportService);
-        final AtomicInteger idGenerator = new AtomicInteger();
         final int limitPortCounts = randomIntBetween(1, 10);
-        final List<DiscoveryNode> discoveryNodes = UnicastZenPing.resolveDiscoveryNodes(
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
             executorService,
             logger,
             Collections.singletonList("127.0.0.1"),
             limitPortCounts,
             transportService,
-            () -> Integer.toString(idGenerator.incrementAndGet()),
+            "test_",
             TimeValue.timeValueSeconds(1));
         assertThat(discoveryNodes, hasSize(limitPortCounts));
         final Set<Integer> ports = new HashSet<>();
@@ -437,17 +465,16 @@ public class UnicastZenPingTests extends ESTestCase {
         closeables.push(transport);
 
         final TransportService transportService =
-            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
         closeables.push(transportService);
-        final AtomicInteger idGenerator = new AtomicInteger();
 
-        final List<DiscoveryNode> discoveryNodes = UnicastZenPing.resolveDiscoveryNodes(
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
             executorService,
             logger,
             Arrays.asList(hostname),
             1,
             transportService,
-            () -> Integer.toString(idGenerator.incrementAndGet()),
+            "test_",
             TimeValue.timeValueSeconds(1)
         );
 
@@ -488,18 +515,17 @@ public class UnicastZenPingTests extends ESTestCase {
         closeables.push(transport);
 
         final TransportService transportService =
-            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
         closeables.push(transportService);
-        final AtomicInteger idGenerator = new AtomicInteger();
         final TimeValue resolveTimeout = TimeValue.timeValueSeconds(randomIntBetween(1, 3));
         try {
-            final List<DiscoveryNode> discoveryNodes = UnicastZenPing.resolveDiscoveryNodes(
+            final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
                 executorService,
                 logger,
                 Arrays.asList("hostname1", "hostname2"),
                 1,
                 transportService,
-                () -> Integer.toString(idGenerator.incrementAndGet()),
+                "test+",
                 resolveTimeout);
 
             assertThat(discoveryNodes, hasSize(1));
@@ -513,6 +539,156 @@ public class UnicastZenPingTests extends ESTestCase {
         }
     }
 
+    public void testResolveReuseExistingNodeConnections() throws ExecutionException, InterruptedException {
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+
+        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v);
+
+        NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleA.transportService);
+        NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleB.transportService);
+
+        final boolean useHosts = randomBoolean();
+        final Settings.Builder hostsSettingsBuilder = Settings.builder().put("cluster.name", "test");
+        if (useHosts) {
+            hostsSettingsBuilder.putArray("discovery.zen.ping.unicast.hosts",
+                NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort()))
+            );
+        } else {
+            hostsSettingsBuilder.put("discovery.zen.ping.unicast.hosts", (String) null);
+        }
+        final Settings hostsSettings = hostsSettingsBuilder.build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+
+        // connection to reuse
+        handleA.transportService.connectToNode(handleB.node);
+
+        // install a listener to check that no new connections are made
+        handleA.transportService.addConnectionListener(new TransportConnectionListener() {
+            @Override
+            public void onConnectionOpened(DiscoveryNode node) {
+                fail("should not open any connections. got [" + node + "]");
+            }
+        });
+
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        zenPingA.start(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().add(handleA.node).add(handleB.node).localNodeId("UZP_A").build();
+            }
+
+            @Override
+            public ClusterState clusterState() {
+                return ClusterState.builder(state).blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)).build();
+            }
+        });
+        closeables.push(zenPingA);
+
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        zenPingB.start(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B").build();
+            }
+
+            @Override
+            public ClusterState clusterState() {
+                return state;
+            }
+        });
+        closeables.push(zenPingB);
+
+        Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+        assertThat(pingResponses.size(), equalTo(1));
+        ZenPing.PingResponse ping = pingResponses.iterator().next();
+        assertThat(ping.node().getId(), equalTo("UZP_B"));
+        assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+
+    }
+
+    public void testPingingTemporalPings() throws ExecutionException, InterruptedException {
+        final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build();
+
+        NetworkService networkService = new NetworkService(settings, Collections.emptyList());
+
+        final BiFunction<Settings, Version, Transport> supplier = (s, v) -> new MockTcpTransport(
+            s,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(),
+            new NamedWriteableRegistry(Collections.emptyList()),
+            networkService,
+            v);
+
+        NetworkHandle handleA = startServices(settings, threadPool, "UZP_A", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleA.transportService);
+        NetworkHandle handleB = startServices(settings, threadPool, "UZP_B", Version.CURRENT, supplier, EnumSet.allOf(Role.class));
+        closeables.push(handleB.transportService);
+
+        final Settings hostsSettings = Settings.builder()
+            .put("cluster.name", "test")
+            .put("discovery.zen.ping.unicast.hosts", (String) null) // use nodes for simplicity
+            .build();
+        final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build();
+
+        final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER);
+        zenPingA.start(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().add(handleA.node).add(handleB.node).localNodeId("UZP_A").build();
+            }
+
+            @Override
+            public ClusterState clusterState() {
+                return ClusterState.builder(state).blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)).build();
+            }
+        });
+        closeables.push(zenPingA);
+
+        // Node B doesn't know about A!
+        TestUnicastZenPing zenPingB = new TestUnicastZenPing(hostsSettings, threadPool, handleB, EMPTY_HOSTS_PROVIDER);
+        zenPingB.start(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().add(handleB.node).localNodeId("UZP_B").build();
+            }
+
+            @Override
+            public ClusterState clusterState() {
+                return state;
+            }
+        });
+        closeables.push(zenPingB);
+
+        {
+            logger.info("pinging from UZP_A so UZP_B will learn about it");
+            Collection<ZenPing.PingResponse> pingResponses = zenPingA.pingAndWait().toList();
+            assertThat(pingResponses.size(), equalTo(1));
+            ZenPing.PingResponse ping = pingResponses.iterator().next();
+            assertThat(ping.node().getId(), equalTo("UZP_B"));
+            assertThat(ping.getClusterStateVersion(), equalTo(state.version()));
+        }
+        {
+            logger.info("pinging from UZP_B");
+            Collection<ZenPing.PingResponse> pingResponses = zenPingB.pingAndWait().toList();
+            assertThat(pingResponses.size(), equalTo(1));
+            ZenPing.PingResponse ping = pingResponses.iterator().next();
+            assertThat(ping.node().getId(), equalTo("UZP_A"));
+            assertThat(ping.getClusterStateVersion(), equalTo(-1L)); // A has a block
+        }
+    }
+
     public void testInvalidHosts() throws InterruptedException {
         final Logger logger = mock(Logger.class);
         final NetworkService networkService = new NetworkService(Settings.EMPTY, Collections.emptyList());
@@ -527,16 +703,15 @@ public class UnicastZenPingTests extends ESTestCase {
         closeables.push(transport);
 
         final TransportService transportService =
-            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            new TransportService(Settings.EMPTY, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null);
         closeables.push(transportService);
-        final AtomicInteger idGenerator = new AtomicInteger();
-        final List<DiscoveryNode> discoveryNodes = UnicastZenPing.resolveDiscoveryNodes(
+        final List<DiscoveryNode> discoveryNodes = TestUnicastZenPing.resolveHostsLists(
             executorService,
             logger,
             Arrays.asList("127.0.0.1:9300:9300", "127.0.0.1:9301"),
             1,
             transportService,
-            () -> Integer.toString(idGenerator.incrementAndGet()),
+            "test_",
             TimeValue.timeValueSeconds(1));
         assertThat(discoveryNodes, hasSize(1)); // only one of the two is valid and will be used
         assertThat(discoveryNodes.get(0).getAddress().getAddress(), equalTo("127.0.0.1"));
@@ -544,24 +719,13 @@ public class UnicastZenPingTests extends ESTestCase {
         verify(logger).warn(eq("failed to resolve host [127.0.0.1:9300:9300]"), Matchers.any(ExecutionException.class));
     }
 
-    // assert that we tried to ping each of the configured nodes at least once
-    private void assertCountersMoreThan(final NetworkHandle that, final NetworkHandle...handles) {
-        final HashMap<TransportAddress, Integer> moreThan = new HashMap<>();
-        for (final NetworkHandle handle : handles) {
-            assert handle != that;
-            moreThan.put(handle.address, 0);
-        }
-        assertCountersMoreThan(moreThan, that, handles);
-    }
-
-    private void assertCountersMoreThan(
-            final Map<TransportAddress, Integer> moreThan,
-            final NetworkHandle that,
-            final NetworkHandle... handles) {
-        for (final NetworkHandle handle : handles) {
-            assert handle != that;
-            assertThat(that.counters.get(handle.address).get(), greaterThan(moreThan.get(handle.address)));
-        }
+    private void assertPingCount(final NetworkHandle fromNode, final NetworkHandle toNode, int expectedCount) {
+        final AtomicInteger counter = fromNode.counters.getOrDefault(toNode.address, new AtomicInteger());
+        final String onNodeName = fromNode.node.getName();
+        assertNotNull("handle for [" + onNodeName + "] has no 'expected' counter", counter);
+        final String forNodeName = toNode.node.getName();
+        assertThat("node [" + onNodeName + "] ping count to [" + forNodeName + "] is unexpected",
+            counter.get(), equalTo(expectedCount));
     }
 
     private NetworkHandle startServices(
@@ -570,33 +734,36 @@ public class UnicastZenPingTests extends ESTestCase {
         final String nodeId,
         final Version version,
         final BiFunction<Settings, Version, Transport> supplier) {
-        final Transport transport = supplier.apply(settings, version);
-        final TransportService transportService =
-            new MockTransportService(settings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+        return startServices(settings, threadPool, nodeId, version, supplier, emptySet());
+
+    }
+
+    private NetworkHandle startServices(
+        final Settings settings,
+        final ThreadPool threadPool,
+        final String nodeId,
+        final Version version,
+        final BiFunction<Settings, Version, Transport> supplier,
+        final Set<Role> nodeRoles) {
+        final Settings nodeSettings = Settings.builder().put(settings)
+            .put("node.name", nodeId)
+            .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "internal:discovery/zen/unicast")
+            .build();
+        final Transport transport = supplier.apply(nodeSettings, version);
+        final MockTransportService transportService =
+            new MockTransportService(nodeSettings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR,  boundAddress ->
+                new DiscoveryNode(nodeId, nodeId, boundAddress.publishAddress(), emptyMap(), nodeRoles, version), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         final ConcurrentMap<TransportAddress, AtomicInteger> counters = ConcurrentCollections.newConcurrentMap();
-        transportService.addConnectionListener(new TransportConnectionListener() {
-
+        transportService.addTracer(new MockTransportService.Tracer() {
             @Override
-            public void onNodeConnected(DiscoveryNode node) {
-            }
-
-            @Override
-            public void onConnectionOpened(DiscoveryNode node) {
+            public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
                 counters.computeIfAbsent(node.getAddress(), k -> new AtomicInteger());
                 counters.get(node.getAddress()).incrementAndGet();
             }
-
-            @Override
-            public void onNodeDisconnected(DiscoveryNode node) {
-            }
-
         });
-        final DiscoveryNode node =
-            new DiscoveryNode(nodeId, transportService.boundAddress().publishAddress(), emptyMap(), emptySet(), version);
-        transportService.setLocalNode(node);
-        return new NetworkHandle(transport.boundAddress().publishAddress(), transportService, node, counters);
+        return new NetworkHandle(transport.boundAddress().publishAddress(), transportService, transportService.getLocalNode(), counters);
     }
 
     private static class NetworkHandle {
@@ -606,7 +773,7 @@ public class UnicastZenPingTests extends ESTestCase {
         public final DiscoveryNode node;
         public final ConcurrentMap<TransportAddress, AtomicInteger> counters;
 
-        public NetworkHandle(
+        NetworkHandle(
             final TransportAddress address,
             final TransportService transportService,
             final DiscoveryNode discoveryNode,
@@ -616,7 +783,125 @@ public class UnicastZenPingTests extends ESTestCase {
             this.node = discoveryNode;
             this.counters = counters;
         }
+    }
 
+    private static class TestUnicastZenPing extends UnicastZenPing {
+
+        TestUnicastZenPing(Settings settings, ThreadPool threadPool, NetworkHandle networkHandle,
+                                  UnicastHostsProvider unicastHostsProvider) {
+            super(Settings.builder().put("node.name", networkHandle.node.getName()).put(settings).build(),
+                threadPool, networkHandle.transportService, unicastHostsProvider);
+        }
+
+        volatile CountDownLatch allTasksCompleted;
+        volatile AtomicInteger pendingTasks;
+        volatile CountDownLatch pingingRoundClosed;
+
+        PingCollection pingAndWait() throws ExecutionException, InterruptedException {
+            allTasksCompleted = new CountDownLatch(1);
+            pingingRoundClosed = new CountDownLatch(1);
+            pendingTasks = new AtomicInteger();
+            // mark the three sending rounds as ongoing
+            markTaskAsStarted("send pings");
+            markTaskAsStarted("send pings");
+            markTaskAsStarted("send pings");
+            final AtomicReference<PingCollection> response = new AtomicReference<>();
+            ping(response::set, TimeValue.timeValueMillis(1), TimeValue.timeValueSeconds(1));
+            pingingRoundClosed.await();
+            final PingCollection result = response.get();
+            assertNotNull("pinging didn't complete",  result);
+            return result;
+        }
+
+        @Override
+        protected void finishPingingRound(PingingRound pingingRound) {
+            // wait for all activity to finish before closing
+            try {
+                allTasksCompleted.await();
+            } catch (InterruptedException e) {
+                // ok, finish anyway
+            }
+            super.finishPingingRound(pingingRound);
+            pingingRoundClosed.countDown();
+        }
+
+        @Override
+        protected void sendPings(TimeValue timeout, PingingRound pingingRound) {
+            super.sendPings(timeout, pingingRound);
+            markTaskAsCompleted("send pings");
+        }
+
+        @Override
+        protected void submitToExecutor(AbstractRunnable abstractRunnable) {
+            markTaskAsStarted("executor runnable");
+            super.submitToExecutor(new AbstractRunnable() {
+                @Override
+                public void onRejection(Exception e) {
+                    try {
+                        super.onRejection(e);
+                    } finally {
+                        markTaskAsCompleted("executor runnable (rejected)");
+                    }
+                }
+
+                @Override
+                public void onAfter() {
+                    markTaskAsCompleted("executor runnable");
+                }
+
+                @Override
+                protected void doRun() throws Exception {
+                    abstractRunnable.run();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // we shouldn't really end up here.
+                    throw new AssertionError("unexpected error", e);
+                }
+            });
+        }
+
+        private void markTaskAsStarted(String task) {
+            logger.trace("task [{}] started. count [{}]", task, pendingTasks.incrementAndGet());
+        }
+
+        private void markTaskAsCompleted(String task) {
+            final int left = pendingTasks.decrementAndGet();
+            logger.trace("task [{}] completed. count [{}]", task, left);
+            if (left == 0) {
+                allTasksCompleted.countDown();
+            }
+        }
+
+        @Override
+        protected TransportResponseHandler<UnicastPingResponse> getPingResponseHandler(PingingRound pingingRound, DiscoveryNode node) {
+            markTaskAsStarted("ping [" + node + "]");
+            TransportResponseHandler<UnicastPingResponse> original = super.getPingResponseHandler(pingingRound, node);
+            return new TransportResponseHandler<UnicastPingResponse>() {
+                @Override
+                public UnicastPingResponse newInstance() {
+                    return original.newInstance();
+                }
+
+                @Override
+                public void handleResponse(UnicastPingResponse response) {
+                    original.handleResponse(response);
+                    markTaskAsCompleted("ping [" + node + "]");
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    original.handleException(exp);
+                    markTaskAsCompleted("ping [" + node + "] (error)");
+                }
+
+                @Override
+                public String executor() {
+                    return original.executor();
+                }
+            };
+        }
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index cc8d43cc79e..7670e8122f3 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -236,12 +236,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
index b46df47afbc..5bbd0f19d2f 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
@@ -19,23 +19,11 @@
 
 package org.elasticsearch.discovery.zen;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -50,7 +38,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.zen.PublishClusterStateActionTests.AssertingAckListener;
@@ -65,6 +53,20 @@ import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
@@ -171,38 +173,36 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         Settings settings = Settings.builder()
                                 .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)).build();
 
-        ArrayList<Closeable> toClose = new ArrayList<>();
+        ArrayDeque<Closeable> toClose = new ArrayDeque<>();
         try {
             Set<DiscoveryNode> expectedFDNodes = null;
 
             final MockTransportService masterTransport = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
             masterTransport.start();
-            DiscoveryNode masterNode = new DiscoveryNode("master",  masterTransport.boundAddress().publishAddress(), Version.CURRENT);
-            toClose.add(masterTransport);
-            masterTransport.setLocalNode(masterNode);
+            DiscoveryNode masterNode = masterTransport.getLocalNode();
+            toClose.addFirst(masterTransport);
             ClusterState state = ClusterStateCreationUtils.state(masterNode, masterNode, masterNode);
             // build the zen discovery and cluster service
             ClusterService masterClusterService = createClusterService(threadPool, masterNode);
-            toClose.add(masterClusterService);
+            toClose.addFirst(masterClusterService);
             // TODO: clustername shouldn't be stored twice in cluster service, but for now, work around it
             state = ClusterState.builder(masterClusterService.getClusterName()).nodes(state.nodes()).build();
             setState(masterClusterService, state);
             ZenDiscovery masterZen = buildZenDiscovery(settings, masterTransport, masterClusterService, threadPool);
-            toClose.add(masterZen);
+            toClose.addFirst(masterZen);
             masterTransport.acceptIncomingRequests();
 
             final MockTransportService otherTransport = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
             otherTransport.start();
-            toClose.add(otherTransport);
-            DiscoveryNode otherNode = new DiscoveryNode("other", otherTransport.boundAddress().publishAddress(), Version.CURRENT);
-            otherTransport.setLocalNode(otherNode);
+            toClose.addFirst(otherTransport);
+            DiscoveryNode otherNode = otherTransport.getLocalNode();
             final ClusterState otherState = ClusterState.builder(masterClusterService.getClusterName())
                 .nodes(DiscoveryNodes.builder().add(otherNode).localNodeId(otherNode.getId())).build();
             ClusterService otherClusterService = createClusterService(threadPool, masterNode);
-            toClose.add(otherClusterService);
+            toClose.addFirst(otherClusterService);
             setState(otherClusterService, otherState);
             ZenDiscovery otherZen = buildZenDiscovery(settings, otherTransport, otherClusterService, threadPool);
-            toClose.add(otherZen);
+            toClose.addFirst(otherZen);
             otherTransport.acceptIncomingRequests();
 
             masterTransport.connectToNode(otherNode);
@@ -242,21 +242,20 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)).build();
 
-        ArrayList<Closeable> toClose = new ArrayList<>();
+        ArrayDeque<Closeable> toClose = new ArrayDeque<>();
         try {
             final MockTransportService masterTransport = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
             masterTransport.start();
-            DiscoveryNode masterNode = new DiscoveryNode("master",  masterTransport.boundAddress().publishAddress(), Version.CURRENT);
-            toClose.add(masterTransport);
-            masterTransport.setLocalNode(masterNode);
+            DiscoveryNode masterNode = masterTransport.getLocalNode();
+            toClose.addFirst(masterTransport);
             ClusterState state = ClusterStateCreationUtils.state(masterNode, null, masterNode);
             // build the zen discovery and cluster service
             ClusterService masterClusterService = createClusterService(threadPool, masterNode);
-            toClose.add(masterClusterService);
+            toClose.addFirst(masterClusterService);
             state = ClusterState.builder(masterClusterService.getClusterName()).nodes(state.nodes()).build();
             setState(masterClusterService, state);
             ZenDiscovery masterZen = buildZenDiscovery(settings, masterTransport, masterClusterService, threadPool);
-            toClose.add(masterZen);
+            toClose.addFirst(masterZen);
             masterTransport.acceptIncomingRequests();
 
             // inject a pending cluster state
@@ -289,7 +288,8 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
     }
 
     private ZenDiscovery buildZenDiscovery(Settings settings, TransportService service, ClusterService clusterService, ThreadPool threadPool) {
-        ZenDiscovery zenDiscovery = new ZenDiscovery(settings, threadPool, service, clusterService, Collections::emptyList);
+        ZenDiscovery zenDiscovery = new ZenDiscovery(settings, threadPool, service, new NamedWriteableRegistry(ClusterModule.getNamedWriteables()),
+            clusterService, Collections::emptyList);
         zenDiscovery.start();
         return zenDiscovery;
     }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
index 9c70587d0e5..9fa680fc200 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
@@ -68,7 +68,7 @@ public class ZenPingTests extends ESTestCase {
         Collections.shuffle(pings, random());
 
         ZenPing.PingCollection collection = new ZenPing.PingCollection();
-        collection.addPings(pings);
+        pings.forEach(collection::addPing);
 
         List<ZenPing.PingResponse> aggregate = collection.toList();
 
diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
index e3556c8cc7c..34b4bc05727 100644
--- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
+++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -32,6 +32,7 @@ import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
@@ -185,7 +186,7 @@ public class DocumentActionsIT extends ESIntegTestCase {
                 .add(client().prepareIndex().setIndex("test").setType("type1").setId("2").setSource(source("2", "test")).setCreate(true))
                 .add(client().prepareIndex().setIndex("test").setType("type1").setSource(source("3", "test")))
                 .add(client().prepareDelete().setIndex("test").setType("type1").setId("1"))
-                .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }")) // failure
+                .add(client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }", XContentType.JSON)) // failure
                 .execute().actionGet();
 
         assertThat(bulkResponse.hasFailures(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
index 814a861139e..84166bb3f96 100644
--- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -45,7 +46,7 @@ public class ShardInfoIT extends ESIntegTestCase {
 
     public void testIndexAndDelete() throws Exception {
         prepareIndex(1);
-        IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get();
+        IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON).get();
         assertShardInfo(indexResponse);
         DeleteResponse deleteResponse = client().prepareDelete("idx", "type", indexResponse.getId()).get();
         assertShardInfo(deleteResponse);
@@ -53,7 +54,8 @@ public class ShardInfoIT extends ESIntegTestCase {
 
     public void testUpdate() throws Exception {
         prepareIndex(1);
-        UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}").setDocAsUpsert(true).get();
+        UpdateResponse updateResponse = client().prepareUpdate("idx", "type", "1").setDoc("{}", XContentType.JSON).setDocAsUpsert(true)
+            .get();
         assertShardInfo(updateResponse);
     }
 
@@ -61,7 +63,7 @@ public class ShardInfoIT extends ESIntegTestCase {
         prepareIndex(1);
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         for (int i = 0; i < 10; i++) {
-            bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}"));
+            bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
         }
 
         BulkResponse bulkResponse = bulkRequestBuilder.get();
@@ -83,7 +85,8 @@ public class ShardInfoIT extends ESIntegTestCase {
         prepareIndex(1);
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
         for (int i = 0; i < 10; i++) {
-            bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}").setDocAsUpsert(true));
+            bulkRequestBuilder.add(client().prepareUpdate("idx", "type", Integer.toString(i)).setDoc("{}", XContentType.JSON)
+                .setDocAsUpsert(true));
         }
 
         BulkResponse bulkResponse = bulkRequestBuilder.get();
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index 6240a13a0bd..5a62c9ee82c 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -27,8 +27,10 @@ import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.fieldstats.IndexConstraint;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.cache.request.RequestCacheStats;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -40,6 +42,7 @@ import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX;
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.nullValue;
@@ -60,7 +63,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
                     "long", "type=long",
                     "integer", "type=integer",
                     "short", "type=short",
-                    "byte", "type=byte"));
+                    "byte", "type=byte",
+                    "location", "type=geo_point"));
         ensureGreen("test");
 
         // index=false
@@ -74,7 +78,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
             "short", "type=short,index=false",
-            "byte", "type=byte,index=false"
+            "byte", "type=byte,index=false",
+            "location", "type=geo_point,index=false"
         ));
         ensureGreen("test1");
 
@@ -89,7 +94,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
             "short", "type=short,index=false",
-            "byte", "type=byte,index=false"
+            "byte", "type=byte,index=false",
+            "location", "type=geo_point,index=false"
         ));
         ensureGreen("test3");
 
@@ -107,6 +113,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         double maxFloat = Double.NEGATIVE_INFINITY;
         double minDouble = Double.POSITIVE_INFINITY;
         double maxDouble = Double.NEGATIVE_INFINITY;
+        GeoPoint minLoc = new GeoPoint(90, 180);
+        GeoPoint maxLoc = new GeoPoint(-90, -180);
         String minString = new String(Character.toChars(1114111));
         String maxString = "0";
 
@@ -135,6 +143,9 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             double d = randomDouble();
             minDouble = Math.min(minDouble, d);
             maxDouble = Math.max(maxDouble, d);
+            GeoPoint loc = RandomGeoGenerator.randomPoint(random());
+            minLoc.reset(Math.min(loc.lat(), minLoc.lat()), Math.min(loc.lon(), minLoc.lon()));
+            maxLoc.reset(Math.max(loc.lat(), maxLoc.lat()), Math.max(loc.lon(), maxLoc.lon()));
             String str = randomRealisticUnicodeOfLength(3);
             if (str.compareTo(minString) < 0) {
                 minString = str;
@@ -151,6 +162,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
                                 "half_float", hf,
                                 "float", f,
                                 "double", d,
+                                "location", loc,
                                 "string", str)
             );
         }
@@ -158,7 +170,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
         FieldStatsResponse response = client()
             .prepareFieldStats()
-            .setFields("byte", "short", "integer", "long", "half_float", "float", "double", "string").get();
+            .setFields("byte", "short", "integer", "long", "half_float", "float", "double", "location", "string").get();
         assertAllSuccessful(response);
 
         for (FieldStats<?> stats : response.getAllFieldStats().values()) {
@@ -188,6 +200,11 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
         assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(maxDouble));
         assertThat(response.getAllFieldStats().get("double").getDisplayType(), equalTo("float"));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMinValue()).lat(), closeTo(minLoc.lat(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMinValue()).lon(), closeTo(minLoc.lon(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMaxValue()).lat(), closeTo(maxLoc.lat(), 1E-5));
+        assertThat(((GeoPoint)response.getAllFieldStats().get("location").getMaxValue()).lon(), closeTo(maxLoc.lon(), 1E-5));
+        assertThat(response.getAllFieldStats().get("location").getDisplayType(), equalTo("geo_point"));
     }
 
     public void testFieldStatsIndexLevel() throws Exception {
@@ -522,6 +539,24 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertEquals(oldHitCount, indexStats.getHitCount());
     }
 
+    public void testGeoPointNotIndexed() throws Exception {
+        assertAcked(prepareCreate("test").addMapping("test", "value", "type=long", "location", "type=geo_point,index=false"));
+        ensureGreen("test");
+        client().prepareIndex("test", "test").setSource("value", 1L, "location", new GeoPoint(32, -132)).get();
+        client().prepareIndex("test", "test").setSource("value", 2L).get();
+        client().prepareIndex("test", "test").setSource("value", 3L).get();
+        client().prepareIndex("test", "test").setSource("value", 4L).get();
+        refresh();
+
+        FieldStatsResponse response = client().prepareFieldStats().setFields("value", "location").get();
+        assertAllSuccessful(response);
+        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
+        assertThat(response.getAllFieldStats().get("location").getMinValue(), equalTo(null));
+        assertThat(response.getAllFieldStats().get("location").getMaxValue(), equalTo(null));
+        assertThat(response.getAllFieldStats().get("location").isAggregatable(), equalTo(true));
+        assertThat(response.getAllFieldStats().get("location").isSearchable(), equalTo(false));
+    }
+
     private void indexRange(String index, long from, long to) throws Exception {
         indexRange(index, "value", from, to);
     }
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
index b58502bffe9..d514cce2223 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
@@ -614,59 +614,59 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
     /**
      * creates a random field stats which does not guarantee that {@link FieldStats#maxValue} is greater than {@link FieldStats#minValue}
      **/
-    private FieldStats randomFieldStats(boolean withNullMinMax) throws UnknownHostException {
+    public static FieldStats randomFieldStats(boolean withNullMinMax) throws UnknownHostException {
         int type = randomInt(5);
         switch (type) {
             case 0:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Long(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Long(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(), randomLong(), randomLong());
+                    return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomLong(), randomLong());
                 }
             case 1:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Double(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Double(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(), randomDouble(), randomDouble());
+                    return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomDouble(), randomDouble());
                 }
             case 2:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Date(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Date(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(), Joda.forPattern("basicDate"),
+                    return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(), Joda.forPattern("basicDate"),
                         new Date().getTime(), new Date().getTime());
                 }
             case 3:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Text(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Text(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(),
+                    return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
                         new BytesRef(randomAsciiOfLength(10)), new BytesRef(randomAsciiOfLength(20)));
                 }
             case 4:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(),
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
                         InetAddress.getByName("::1"), InetAddress.getByName("::1"));
                 }
             case 5:
                 if (withNullMinMax && randomBoolean()) {
-                    return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean());
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean());
                 } else {
-                    return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                        randomPositiveLong(), randomBoolean(), randomBoolean(),
+                    return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
+                        randomNonNegativeLong(), randomBoolean(), randomBoolean(),
                         InetAddress.getByName("1.2.3.4"), InetAddress.getByName("1.2.3.4"));
                 }
             default:
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 13e39a342dc..1cdaa27815a 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -232,7 +232,7 @@ public class AsyncShardFetchTests extends ESTestCase {
             private final CountDownLatch executeLatch = new CountDownLatch(1);
             private final CountDownLatch waitLatch = new CountDownLatch(1);
 
-            public Entry(Response response, Throwable failure) {
+            Entry(Response response, Throwable failure) {
                 this.response = response;
                 this.failure = failure;
             }
@@ -242,7 +242,7 @@ public class AsyncShardFetchTests extends ESTestCase {
         private final Map<String, Entry> simulations = new ConcurrentHashMap<>();
         private AtomicInteger reroute = new AtomicInteger();
 
-        public TestFetch(ThreadPool threadPool) {
+        TestFetch(ThreadPool threadPool) {
             super(Loggers.getLogger(TestFetch.class), "test", new ShardId("test", "_na_", 1), null);
             this.threadPool = threadPool;
         }
@@ -305,7 +305,7 @@ public class AsyncShardFetchTests extends ESTestCase {
 
     static class Response extends BaseNodeResponse {
 
-        public Response(DiscoveryNode node) {
+        Response(DiscoveryNode node) {
             super(node);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
index 91d6d3f08fe..4502e32e6d3 100644
--- a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
@@ -47,7 +47,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testCleanupWhenEmpty() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
@@ -58,7 +58,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
     }
     public void testDanglingIndicesDiscovery() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             assertTrue(danglingState.getDanglingIndices().isEmpty());
@@ -76,7 +76,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testInvalidIndexFolder() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
@@ -100,7 +100,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testDanglingProcessing() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             MetaData metaData = MetaData.builder().build();
@@ -144,7 +144,7 @@ public class DanglingIndicesStateTests extends ESTestCase {
 
     public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             DanglingIndicesState danglingState = createDanglingIndicesState(env, metaStateService);
 
             final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 22f06b9098d..25153576b6b 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -58,6 +58,7 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDI
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -434,7 +435,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
         assertNotNull(ex.getCause());
         assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
-        assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]");
+        assertEquals(ex.getCause().getMessage(), "Unknown filter type [icu_collation] for [myCollator]");
     }
 
     /**
@@ -491,7 +492,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         assertEquals(ex.getMessage(), "Failed to verify index " + metaData.getIndex());
         assertNotNull(ex.getCause());
         assertEquals(MapperParsingException.class, ex.getCause().getClass());
-        assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]");
+        assertThat(ex.getCause().getMessage(), containsString("analyzer [test] not found for field [field1]"));
     }
 
     public void testArchiveBrokenClusterSettings() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index b106ca64227..bbf95795337 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -61,12 +61,6 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class GatewayMetaStateTests extends ESAllocationTestCase {
 
-    @Before
-    public void setup() {
-        MetaData.registerPrototype(CustomMetaData1.TYPE, new CustomMetaData1(""));
-        MetaData.registerPrototype(CustomMetaData2.TYPE, new CustomMetaData2(""));
-    }
-
     ClusterChangedEvent generateEvent(boolean initializing, boolean versionChanged, boolean masterEligible) {
         //ridiculous settings to make sure we don't run into uninitialized because fo default
         AllocationService strategy = createAllocationService(Settings.builder()
@@ -391,8 +385,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
     private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService {
         private final boolean upgrade;
 
-        public MockMetaDataIndexUpgradeService(boolean upgrade) {
-            super(Settings.EMPTY, null, null);
+        MockMetaDataIndexUpgradeService(boolean upgrade) {
+            super(Settings.EMPTY, null, null, null);
             this.upgrade = upgrade;
         }
         @Override
@@ -409,12 +403,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData1(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
@@ -432,12 +421,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new CustomMetaData2(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
@@ -450,7 +434,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
     private static MetaData randomMetaData(TestCustomMetaData... customMetaDatas) {
         MetaData.Builder builder = MetaData.builder();
         for (TestCustomMetaData customMetaData : customMetaDatas) {
-            builder.putCustom(customMetaData.type(), customMetaData);
+            builder.putCustom(customMetaData.getWriteableName(), customMetaData);
         }
         for (int i = 0; i < randomIntBetween(1, 5); i++) {
             builder.put(
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 2bec3d5eded..a4e3aad8bec 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -19,7 +19,10 @@
 
 package org.elasticsearch.gateway;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -34,7 +37,7 @@ public class GatewayServiceTests extends ESTestCase {
     private GatewayService createService(Settings.Builder settings) {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                null);
+                null, () -> new DiscoveryNode(UUIDs.randomBase64UUID(), buildNewFakeTransportAddress(), Version.CURRENT));
         return new GatewayService(settings.build(),
                 null, clusterService, null, null, null, new NoopDiscovery(), null);
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
index 0f0e69b2643..5cb6b2d9f98 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -89,7 +90,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         assertThat(resource, notNullValue());
         Path dst = tmp.resolve("global-3.st");
         Files.copy(resource, dst);
-        MetaData read = format.read(dst);
+        MetaData read = format.read(xContentRegistry(), dst);
         assertThat(read, notNullValue());
         assertThat(read.clusterUUID(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg"));
         // indices are empty since they are serialized separately
@@ -114,7 +115,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
         }
         final int version2 = between(version, Integer.MAX_VALUE);
@@ -130,7 +131,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             list = content("foo-*", stateDir);
             assertEquals(list.length,1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-"+ (id+1) + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state2));
 
         }
@@ -156,7 +157,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
         }
     }
@@ -180,12 +181,12 @@ public class MetaDataStateFormatTests extends ESTestCase {
             list = content("foo-*", stateDir);
             assertEquals(list.length, 1);
             assertThat(list[0].getFileName().toString(), equalTo("foo-" + id + ".st"));
-            DummyState read = format.read(list[0]);
+            DummyState read = format.read(NamedXContentRegistry.EMPTY, list[0]);
             assertThat(read, equalTo(state));
             // now corrupt it
             corruptFile(list[0], logger);
             try {
-                format.read(list[0]);
+                format.read(NamedXContentRegistry.EMPTY, list[0]);
                 fail("corrupted file");
             } catch (CorruptStateException ex) {
                 // expected
@@ -195,7 +196,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
 
     public static void corruptFile(Path file, Logger logger) throws IOException {
         Path fileToCorrupt = file;
-        try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
+        try (SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
             long checksumBeforeCorruption;
             try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
                 checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
@@ -272,7 +273,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         }
         List<Path> dirList = Arrays.asList(dirs);
         Collections.shuffle(dirList, random());
-        MetaData loadedMetaData = format.loadLatestState(logger, dirList.toArray(new Path[0]));
+        MetaData loadedMetaData = format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
         MetaData latestMetaData = meta.get(numStates-1);
         assertThat(loadedMetaData.clusterUUID(), not(equalTo("_na_")));
         assertThat(loadedMetaData.clusterUUID(), equalTo(latestMetaData.clusterUUID()));
@@ -299,7 +300,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
                 MetaDataStateFormatTests.corruptFile(file, logger);
             }
             try {
-                format.loadLatestState(logger, dirList.toArray(new Path[0]));
+                format.loadLatestState(logger, xContentRegistry(), dirList.toArray(new Path[0]));
                 fail("latest version can not be read");
             } catch (ElasticsearchException ex) {
                 assertThat(ExceptionsHelper.unwrap(ex, CorruptStateException.class), notNullValue());
@@ -385,7 +386,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
                     '}';
         }
 
-        public DummyState(String string, int aInt, long aLong, double aDouble, boolean aBoolean) {
+        DummyState(String string, int aInt, long aLong, double aDouble, boolean aBoolean) {
             this.string = string;
             this.aInt = aInt;
             this.aLong = aLong;
@@ -393,7 +394,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
             this.aBoolean = aBoolean;
         }
 
-        public DummyState() {
+        DummyState() {
 
         }
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
index 62f040c0163..ec7607085c0 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
@@ -39,7 +39,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteLoadIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             metaStateService.writeIndex("test_write", index);
@@ -49,14 +49,14 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testLoadMissingIndex() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
             assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
         }
     }
 
     public void testWriteLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -68,7 +68,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             MetaData metaData = MetaData.builder()
                     .persistentSettings(Settings.builder().put("test1", "value1").build())
@@ -84,7 +84,7 @@ public class MetaStateServiceTests extends ESTestCase {
 
     public void testLoadGlobal() throws Exception {
         try (NodeEnvironment env = newNodeEnvironment()) {
-            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env, xContentRegistry());
 
             IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
             MetaData metaData = MetaData.builder()
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index 08d806fa790..cb4bd91288f 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -74,6 +74,16 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
     private final DiscoveryNode node3 = newNode("node3");
     private TestAllocator testAllocator;
 
+
+    /**
+     * needed due to random usage of {@link IndexMetaData#INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING}. removed once
+     * shadow replicas are removed.
+     */
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
+
     @Before
     public void buildTestAllocator() {
         this.testAllocator = new TestAllocator();
@@ -754,7 +764,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
 
         private Map<DiscoveryNode, TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> data;
 
-        public TestAllocator() {
+        TestAllocator() {
             super(Settings.EMPTY);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index 431b592fac9..c97c0982ebb 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -26,8 +26,6 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
-import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -45,7 +43,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
-import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
 
 import java.nio.file.DirectoryStream;
@@ -55,7 +52,9 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.IntStream;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@@ -384,105 +383,92 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         assertThat(state.metaData().index("test").getAliases().get("test_alias").filter(), notNullValue());
     }
 
-    public void testReusePeerRecovery() throws Exception {
-        final Settings settings = Settings.builder()
-            .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false)
-            .put("gateway.recover_after_nodes", 4)
-            .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 4)
-            .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 4)
-            .put(MockFSDirectoryService.CRASH_INDEX_SETTING.getKey(), false).build();
+    public void testReuseInFileBasedPeerRecovery() throws Exception {
+        internalCluster().startMasterOnlyNode();
+        final String primaryNode = internalCluster().startDataOnlyNode(nodeSettings(0));
 
-        internalCluster().startNodes(4, settings);
-        // prevent any rebalance actions during the peer recovery
-        // if we run into a relocation the reuse count will be 0 and this fails the test. We are testing here if
-        // we reuse the files on disk after full restarts for replicas.
-        assertAcked(prepareCreate("test").setSettings(Settings.builder()
-            .put(indexSettings())
-            .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
-        ensureGreen();
-        logger.info("--> indexing docs");
-        for (int i = 0; i < 1000; i++) {
-            client().prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
-            if ((i % 200) == 0) {
-                client().admin().indices().prepareFlush().execute().actionGet();
-            }
-        }
-        if (randomBoolean()) {
-            client().admin().indices().prepareFlush().execute().actionGet();
-        }
-        logger.info("Running Cluster Health");
-        ensureGreen();
-        client().admin().indices().prepareForceMerge("test").setMaxNumSegments(100).get(); // just wait for merges
-        client().admin().indices().prepareFlush().setForce(true).get();
-
-        boolean useSyncIds = randomBoolean();
-        if (useSyncIds == false) {
-            logger.info("--> disabling allocation while the cluster is shut down");
-
-            // Disable allocations while we are closing nodes
-            client().admin().cluster().prepareUpdateSettings()
-                .setTransientSettings(Settings.builder()
-                    .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE))
-                .get();
-            logger.info("--> full cluster restart");
-            internalCluster().fullRestart();
-
-            logger.info("--> waiting for cluster to return to green after first shutdown");
-            ensureGreen();
-        } else {
-            logger.info("--> trying to sync flush");
-            assertEquals(client().admin().indices().prepareSyncedFlush("test").get().failedShards(), 0);
-            assertSyncIdsNotNull();
-        }
-
-        logger.info("--> disabling allocation while the cluster is shut down{}", useSyncIds ? "" : " a second time");
-        // Disable allocations while we are closing nodes
-        client().admin().cluster().prepareUpdateSettings()
-            .setTransientSettings(Settings.builder()
-                .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE))
+        // create the index with our mapping
+        client(primaryNode)
+            .admin()
+            .indices()
+            .prepareCreate("test")
+            .setSettings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 1))
             .get();
 
-        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
-
-        logger.info("--> full cluster restart");
-        internalCluster().fullRestart();
-
-        logger.info("--> waiting for cluster to return to green after {}shutdown", useSyncIds ? "" : "second ");
-        ensureGreen();
-        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
-
-        if (useSyncIds) {
-            assertSyncIdsNotNull();
+        logger.info("--> indexing docs");
+        for (int i = 0; i < randomIntBetween(1, 1024); i++) {
+            client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
         }
-        RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
-        for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
+
+        client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
+
+        // start the replica node; we do this after indexing so a file-based recovery is triggered to ensure the files are identical
+        final String replicaNode = internalCluster().startDataOnlyNode(nodeSettings(1));
+        ensureGreen();
+
+        final RecoveryResponse initialRecoveryReponse = client().admin().indices().prepareRecoveries("test").get();
+        final Set<String> files = new HashSet<>();
+        for (final RecoveryState recoveryState : initialRecoveryReponse.shardRecoveryStates().get("test")) {
+            if (recoveryState.getTargetNode().getName().equals(replicaNode)) {
+                for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
+                    files.add(file.name());
+                }
+                break;
+            }
+        }
+
+        logger.info("--> restart replica node");
+
+        internalCluster().restartNode(replicaNode, new RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                // index some more documents; we expect to reuse the files that already exist on the replica
+                for (int i = 0; i < randomIntBetween(1, 1024); i++) {
+                    client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
+                }
+
+                // prevent a sequence-number-based recovery from being possible
+                client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
+                return super.onNodeStopped(nodeName);
+            }
+        });
+
+        ensureGreen();
+
+        final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
+        for (final RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
             long recovered = 0;
-            for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
-                if (file.name().startsWith("segments")) {
+            long reused = 0;
+            int filesRecovered = 0;
+            int filesReused = 0;
+            for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
+                if (files.contains(file.name()) == false) {
                     recovered += file.length();
+                    filesRecovered++;
+                } else {
+                    reused += file.length();
+                    filesReused++;
                 }
             }
-            if (!recoveryState.getPrimary() && (useSyncIds == false)) {
-                logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}",
-                    recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(),
-                    recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
-                assertThat("no bytes should be recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
-                assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0L));
-                // we have to recover the segments file since we commit the translog ID on engine startup
-                assertThat("all bytes should be reused except of the segments file", recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes() - recovered));
-                assertThat("no files should be recovered except of the segments file", recoveryState.getIndex().recoveredFileCount(), equalTo(1));
-                assertThat("all files should be reused except of the segments file", recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount() - 1));
-                assertThat("> 0 files should be reused", recoveryState.getIndex().reusedFileCount(), greaterThan(0));
-            } else {
-                if (useSyncIds && !recoveryState.getPrimary()) {
-                    logger.info("--> replica shard {} recovered from {} to {} using sync id, recovered {}, reuse {}",
-                        recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(),
-                        recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
-                }
+            if (recoveryState.getPrimary()) {
                 assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L));
                 assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes()));
                 assertThat(recoveryState.getIndex().recoveredFileCount(), equalTo(0));
                 assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount()));
+            } else {
+                logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}",
+                    recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(),
+                    recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
+                assertThat("bytes should have been recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
+                assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0L));
+                // we have to recover the segments file since we commit the translog ID on engine startup
+                assertThat("all existing files should be reused, byte count mismatch", recoveryState.getIndex().reusedBytes(), equalTo(reused));
+                assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes() - recovered));
+                assertThat("the segment from the last round of indexing should be recovered", recoveryState.getIndex().recoveredFileCount(), equalTo(filesRecovered));
+                assertThat("all existing files should be reused, file count mismatch", recoveryState.getIndex().reusedFileCount(), equalTo(filesReused));
+                assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount() - filesRecovered));
+                assertThat("> 0 files should be reused", recoveryState.getIndex().reusedFileCount(), greaterThan(0));
+                assertThat("no translog ops should be recovered", recoveryState.getTranslog().recoveredOperations(), equalTo(0));
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 0e977f8c122..d78b849917d 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -339,7 +339,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
         private Map<DiscoveryNode, TransportNodesListShardStoreMetaData.StoreFilesMetaData> data = null;
         private AtomicBoolean fetchDataCalled = new AtomicBoolean(false);
 
-        public TestAllocator() {
+        TestAllocator() {
             super(Settings.EMPTY);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index 2a0505b273b..577087ba952 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -629,8 +630,8 @@ public class GetActionIT extends ESIntegTestCase {
 
         logger.info("indexing documents");
 
-        client().prepareIndex("my-index", "my-type1", "1").setSource(source).get();
-        client().prepareIndex("my-index", "my-type2", "1").setSource(source).get();
+        client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get();
+        client().prepareIndex("my-index", "my-type2", "1").setSource(source, XContentType.JSON).get();
 
         logger.info("checking real time retrieval");
 
@@ -682,30 +683,6 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField(field).getValues().get(1).toString(), equalTo("value2"));
     }
 
-    public void testGetAllField() throws Exception {
-        assertAcked(prepareCreate("test")
-                .addAlias(new Alias("alias"))
-                .addMapping("my-type1", jsonBuilder()
-                        .startObject()
-                        .startObject("my-type1")
-                        .startObject("_all")
-                        .field("store", true)
-                        .endObject()
-                        .startObject("properties")
-                        .startObject("some_field")
-                        .field("type", "text")
-                        .endObject()
-                        .endObject()
-                        .endObject()
-                        .endObject()));
-        index("test", "my-type1", "1", "some_field", "some text");
-        refresh();
-
-        GetResponse getResponse = client().prepareGet(indexOrAlias(), "my-type1", "1").setStoredFields("_all").get();
-        assertNotNull(getResponse.getField("_all").getValue());
-        assertThat(getResponse.getField("_all").getValue().toString(), equalTo("some text"));
-    }
-
     public void testUngeneratedFieldsThatAreNeverStored() throws IOException {
         String createIndexSource = "{\n" +
                 "  \"settings\": {\n" +
@@ -722,7 +699,7 @@ public class GetActionIT extends ESIntegTestCase {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"suggest\": {\n" +
@@ -761,10 +738,10 @@ public class GetActionIT extends ESIntegTestCase {
                 "    }\n" +
                 "  }\n" +
                 "}";
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
 
-        client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").get();
+        client().prepareIndex("test", "doc").setId("1").setSource("{}", XContentType.JSON).setParent("1").get();
 
         String[] fieldsList = {"_parent"};
         // before refresh - document is only in translog
@@ -785,12 +762,12 @@ public class GetActionIT extends ESIntegTestCase {
             "  }\n" +
             "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
             "  \"text\": \"some text.\"\n" +
             "}\n";
-        client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
+        client().prepareIndex("test", "doc").setId("1").setSource(doc, XContentType.JSON).setRouting("1").get();
         String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
@@ -804,7 +781,7 @@ public class GetActionIT extends ESIntegTestCase {
 
     public void testGeneratedStringFieldsUnstored() throws IOException {
         indexSingleDocumentWithStringFieldsGeneratedFromText(false, randomBoolean());
-        String[] fieldsList = {"_all", "_field_names"};
+        String[] fieldsList = {"_field_names"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList);
         refresh();
@@ -817,7 +794,7 @@ public class GetActionIT extends ESIntegTestCase {
 
     public void testGeneratedStringFieldsStored() throws IOException {
         indexSingleDocumentWithStringFieldsGeneratedFromText(true, randomBoolean());
-        String[] fieldsList = {"_all"};
+        String[] fieldsList = {"text1", "text2"};
         String[] alwaysNotStoredFieldsList = {"_field_names"};
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList);
         assertGetFieldsNull(indexOrAlias(), "doc", "1", alwaysNotStoredFieldsList);
@@ -838,12 +815,21 @@ public class GetActionIT extends ESIntegTestCase {
                 "  \"mappings\": {\n" +
                 "    \"doc\": {\n" +
                 "      \"_source\" : {\"enabled\" : " + sourceEnabled + "}," +
-                "      \"_all\" : {\"enabled\" : true, \"store\":\"" + storedString + "\" }" +
+                "      \"properties\": {\n" +
+                "        \"text1\": {\n" +
+                "          \"type\": \"text\",\n" +
+                "          \"store\": \"" + storedString + "\"" +
+                "        },\n" +
+                "        \"text2\": {\n" +
+                "          \"type\": \"text\",\n" +
+                "          \"store\": \"" + storedString + "\"" +
+                "        }" +
+                "      }\n" +
                 "    }\n" +
                 "  }\n" +
                 "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"text1\": \"some text.\"\n," +
@@ -905,7 +891,7 @@ public class GetActionIT extends ESIntegTestCase {
                 "  }\n" +
                 "}";
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource, XContentType.JSON));
         ensureGreen();
         String doc = "{\n" +
                 "  \"token_count\": \"A text with five words.\",\n" +
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
deleted file mode 100644
index 2d827ebb6e4..00000000000
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.http;
-
-import java.util.Collections;
-import java.util.Map;
-
-import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
-import org.elasticsearch.rest.AbstractRestChannel;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-public class HttpServerTests extends ESTestCase {
-    private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20);
-    private HttpServer httpServer;
-    private CircuitBreaker inFlightRequestsBreaker;
-
-    @Before
-    public void setup() {
-        Settings settings = Settings.EMPTY;
-        CircuitBreakerService circuitBreakerService = new HierarchyCircuitBreakerService(
-            Settings.builder()
-                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), BREAKER_LIMIT)
-                .build(),
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
-        inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-
-        HttpServerTransport httpServerTransport = new TestHttpServerTransport();
-        RestController restController = new RestController(settings, Collections.emptySet(), null);
-        restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel, client) -> channel.sendResponse(
-                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
-        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
-                throw new IllegalArgumentException("test error");
-            });
-
-        httpServer = new HttpServer(settings, httpServerTransport, restController, null, circuitBreakerService);
-        httpServer.start();
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnSuccess() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnError() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/error", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt();
-        String content = randomAsciiOfLength(contentLength);
-        // we will produce an error in the rest handler and one more when sending the error response
-        TestRestRequest request = new TestRestRequest("/error", content);
-        ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    public void testDispatchRequestLimitsBytes() {
-        int contentLength = BREAKER_LIMIT.bytesAsInt() + 1;
-        String content = randomAsciiOfLength(contentLength);
-        TestRestRequest request = new TestRestRequest("/", content);
-        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE);
-
-        httpServer.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
-
-        assertEquals(1, inFlightRequestsBreaker.getTrippedCount());
-        assertEquals(0, inFlightRequestsBreaker.getUsed());
-    }
-
-    private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
-        HttpServerTransport {
-
-        public TestHttpServerTransport() {
-            super(Settings.EMPTY);
-        }
-
-        @Override
-        protected void doStart() {
-        }
-
-        @Override
-        protected void doStop() {
-        }
-
-        @Override
-        protected void doClose() {
-        }
-
-        @Override
-        public BoundTransportAddress boundAddress() {
-            TransportAddress transportAddress = buildNewFakeTransportAddress();
-            return new BoundTransportAddress(new TransportAddress[] {transportAddress} ,transportAddress);
-        }
-
-        @Override
-        public HttpInfo info() {
-            return null;
-        }
-
-        @Override
-        public HttpStats stats() {
-            return null;
-        }
-
-        @Override
-        public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
-
-        }
-    }
-
-    private static final class AssertingChannel extends AbstractRestChannel {
-        private final RestStatus expectedStatus;
-
-        protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
-            super(request, detailedErrorsEnabled);
-            this.expectedStatus = expectedStatus;
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            assertEquals(expectedStatus, response.status());
-        }
-    }
-
-    private static final class ExceptionThrowingChannel extends AbstractRestChannel {
-
-        protected ExceptionThrowingChannel(RestRequest request, boolean detailedErrorsEnabled) {
-            super(request, detailedErrorsEnabled);
-        }
-
-        @Override
-        public void sendResponse(RestResponse response) {
-            throw new IllegalStateException("always throwing an exception for testing");
-        }
-    }
-
-    private static final class TestRestRequest extends RestRequest {
-
-        private final BytesReference content;
-
-        private TestRestRequest(String path, String content) {
-            super(Collections.emptyMap(), path);
-            this.content = new BytesArray(content);
-        }
-
-        @Override
-        public Method method() {
-            return Method.GET;
-        }
-
-        @Override
-        public String uri() {
-            return null;
-        }
-
-        @Override
-        public boolean hasContent() {
-            return true;
-        }
-
-        @Override
-        public BytesReference content() {
-            return content;
-        }
-
-        @Override
-        public String header(String name) {
-            return null;
-        }
-
-        @Override
-        public Iterable<Map.Entry<String, String>> headers() {
-            return null;
-        }
-
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 6b24c2e356c..106dc9274da 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -48,7 +48,9 @@ import org.elasticsearch.index.cache.query.IndexQueryCache;
 import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.engine.InternalEngineTests;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexingOperationListener;
@@ -64,7 +66,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptService;
@@ -110,7 +111,6 @@ public class IndexModuleTests extends ESTestCase {
     private CircuitBreakerService circuitBreakerService;
     private BigArrays bigArrays;
     private ScriptService scriptService;
-    private IndicesQueriesRegistry indicesQueriesRegistry;
     private ClusterService clusterService;
 
     @Override
@@ -130,7 +130,6 @@ public class IndexModuleTests extends ESTestCase {
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         scriptService = new ScriptService(settings, environment, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry,
                 scriptContextRegistry, scriptSettings);
-        indicesQueriesRegistry = new IndicesQueriesRegistry();
         clusterService = ClusterServiceUtils.createClusterService(threadPool);
         nodeEnvironment = new NodeEnvironment(settings, environment);
         mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
@@ -144,14 +143,14 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     private IndexService newIndexService(IndexModule module) throws IOException {
-        return module.newIndexService(nodeEnvironment, deleter, circuitBreakerService, bigArrays, threadPool, scriptService,
-                indicesQueriesRegistry, clusterService, null, indicesQueryCache, mapperRegistry, shardId -> {},
+        return module.newIndexService(nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool,
+                scriptService, clusterService, null, indicesQueryCache, mapperRegistry, shardId -> {},
                 new IndicesFieldDataCache(settings, listener));
     }
 
     public void testWrapperIsBound() throws IOException {
         IndexModule module = new IndexModule(indexSettings,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.setSearcherWrapper((s) -> new Wrapper());
         module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class));
 
@@ -171,7 +170,7 @@ public class IndexModuleTests extends ESTestCase {
             .build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
         IndexModule module = new IndexModule(indexSettings,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addIndexStore("foo_store", FooStore::new);
         try {
             module.addIndexStore("foo_store", FooStore::new);
@@ -196,7 +195,7 @@ public class IndexModuleTests extends ESTestCase {
         };
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
         IndexModule module = new IndexModule(indexSettings,
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addIndexEventListener(eventListener);
         IndexService indexService = newIndexService(module);
         IndexSettings x = indexService.getIndexSettings();
@@ -211,7 +210,7 @@ public class IndexModuleTests extends ESTestCase {
     public void testListener() throws IOException {
         Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
         AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
@@ -231,11 +230,11 @@ public class IndexModuleTests extends ESTestCase {
 
     public void testAddIndexOperationListener() throws IOException {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 executed.set(true);
                 return operation;
             }
@@ -250,9 +249,11 @@ public class IndexModuleTests extends ESTestCase {
         assertEquals(IndexingSlowLog.class, indexService.getIndexOperationListeners().get(0).getClass());
         assertSame(listener, indexService.getIndexOperationListeners().get(1));
 
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), null);
+        ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null);
+        Engine.Index index = new Engine.Index(new Term("_uid",  doc.uid()), doc);
+        ShardId shardId = new ShardId(new Index("foo", "bar"), 0);
         for (IndexingOperationListener l : indexService.getIndexOperationListeners()) {
-            l.preIndex(index);
+            l.preIndex(shardId, index);
         }
         assertTrue(executed.get());
         indexService.close("simon says", false);
@@ -260,7 +261,7 @@ public class IndexModuleTests extends ESTestCase {
 
     public void testAddSearchOperationListener() throws IOException {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
 
@@ -294,8 +295,8 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
-        module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() {
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+        module.addSimilarity("test_similarity", (string, providerSettings, indexLevelSettings) -> new SimilarityProvider() {
             @Override
             public String name() {
                 return string;
@@ -303,7 +304,7 @@ public class IndexModuleTests extends ESTestCase {
 
             @Override
             public Similarity get() {
-                return new TestSimilarity(settings.get("key"));
+                return new TestSimilarity(providerSettings.get("key"));
             }
         });
 
@@ -318,7 +319,7 @@ public class IndexModuleTests extends ESTestCase {
 
     public void testFrozen() {
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.freeze();
         String msg = "Can't modify IndexModule once the index service has been created";
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage());
@@ -337,7 +338,7 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
         assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage());
     }
@@ -349,7 +350,7 @@ public class IndexModuleTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Exception ex = expectThrows(IllegalArgumentException.class, () -> newIndexService(module));
         assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage());
     }
@@ -359,7 +360,7 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
         IndexService indexService = newIndexService(module);
@@ -372,7 +373,7 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof IndexQueryCache);
         indexService.close("simon says", false);
@@ -384,7 +385,7 @@ public class IndexModuleTests extends ESTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings),
-                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         IndexService indexService = newIndexService(module);
         assertTrue(indexService.cache().query() instanceof DisabledQueryCache);
@@ -417,7 +418,7 @@ public class IndexModuleTests extends ESTestCase {
         private final String key;
 
 
-        public TestSimilarity(String key) {
+        TestSimilarity(String key) {
             if (key == null) {
                 throw new AssertionError("key is null");
             }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
index 84a9fe2970b..7fa769b2613 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
@@ -40,10 +41,11 @@ public class IndexRequestBuilderIT extends ESIntegTestCase {
         map.put("test_field", "foobar");
         IndexRequestBuilder[] builders = new IndexRequestBuilder[] {
                 client().prepareIndex("test", "test").setSource((Object)"test_field", (Object)"foobar"),
-                client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"),
-                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
-                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
-                client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))),
+                client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}", XContentType.JSON),
+                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON),
+                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}"), XContentType.JSON),
+                client().prepareIndex("test", "test")
+                    .setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}")), XContentType.JSON),
                 client().prepareIndex("test", "test").setSource(map)
         };
         indexRandom(true, builders);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index a601f238d24..80e453d665e 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -21,20 +21,19 @@ package org.elasticsearch.index;
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TopDocs;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -45,24 +44,32 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 
 /** Unit test(s) for IndexService */
 public class IndexServiceTests extends ESSingleNodeTestCase {
     public void testDetermineShadowEngineShouldBeUsed() {
-        Settings regularSettings = Settings.builder()
-                .put(SETTING_NUMBER_OF_SHARDS, 2)
-                .put(SETTING_NUMBER_OF_REPLICAS, 1)
-                .build();
+        IndexSettings regularSettings = new IndexSettings(
+            IndexMetaData
+                .builder("regular")
+                .settings(Settings.builder()
+                    .put(SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 1)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .build())
+                .build(),
+            Settings.EMPTY);
 
-        Settings shadowSettings = Settings.builder()
-                .put(SETTING_NUMBER_OF_SHARDS, 2)
-                .put(SETTING_NUMBER_OF_REPLICAS, 1)
-                .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
-                .build();
+        IndexSettings shadowSettings = new IndexSettings(
+            IndexMetaData
+                .builder("shadow")
+                .settings(Settings.builder()
+                    .put(SETTING_NUMBER_OF_SHARDS, 2)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 1)
+                    .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                    .build())
+                .build(),
+            Settings.EMPTY);
 
         assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(true, regularSettings));
         assertFalse("no shadow replicas for normal settings", IndexService.useShadowEngine(false, regularSettings));
@@ -226,7 +233,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
         // now disable
         IndexMetaData metaData = IndexMetaData.builder(indexService.getMetaData()).settings(Settings.builder().put(indexService.getMetaData().getSettings()).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)).build();
         indexService.updateMetaData(metaData);
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             TopDocs search = searcher.searcher().search(new MatchAllDocsQuery(), 10);
@@ -253,7 +260,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
         IndexService indexService = createIndex("test", settings);
         ensureGreen("test");
         assertTrue(indexService.getRefreshTask().mustReschedule());
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         assertBusy(() -> {
             assertFalse(shard.getTranslog().syncNeeded());
@@ -272,7 +279,7 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
         indexService.updateMetaData(metaData);
         assertNotNull(indexService.getFsyncTask());
         assertTrue(indexService.getRefreshTask().mustReschedule());
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}").get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\": \"bar\"}", XContentType.JSON).get();
         IndexShard shard = indexService.getShard(0);
         assertBusy(() -> {
             assertFalse(shard.getTranslog().syncNeeded());
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index b6ea5e719de..a32d076272b 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -288,6 +288,29 @@ public class IndexSettingsTests extends ESTestCase {
         settings = new IndexSettings(metaData, Settings.EMPTY);
         assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
     }
+    
+    public void testMaxAdjacencyMatrixFiltersSetting() {
+        IndexMetaData metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), 15)
+            .build());
+        IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(15, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index",
+            Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(),
+            42).build()));
+        assertEquals(42, settings.getMaxAdjacencyMatrixFilters());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+
+        metaData = newIndexMeta("index", Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .build());
+        settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), 
+                settings.getMaxAdjacencyMatrixFilters());
+    }    
 
     public void testGCDeletesSetting() {
         TimeValue gcDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index e0f5d2a4377..e713f14be14 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.common.Priority;
@@ -57,9 +56,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.ConnectionProfile;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -362,6 +358,9 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         client().admin().indices().prepareUpdateSettings(IDX).setSettings(build).execute().actionGet();
 
         ensureGreen(IDX);
+        // check if primary has relocated to node3
+        assertEquals(internalCluster().clusterService(node3).localNode().getId(),
+            client().admin().cluster().prepareState().get().getState().routingTable().index(IDX).shard(0).primaryShard().currentNodeId());
         logger.info("--> performing query");
         SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
         assertHitCount(resp, 2);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index 16903a01d91..c815b2b55f9 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexingSlowLog.SlowLogParsedDocumentPrinter;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -41,7 +42,7 @@ public class IndexingSlowLogTests extends ESTestCase {
     public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes();
         ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), SeqNoFieldMapper.SequenceID.emptySeqID(), "id",
-                "test", null, null, source, null);
+                "test", null, null, source, XContentType.JSON, null);
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
         SlowLogParsedDocumentPrinter p = new SlowLogParsedDocumentPrinter(index, pd, 10, true, 0);
@@ -88,7 +89,7 @@ public class IndexingSlowLogTests extends ESTestCase {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING.getKey(), "NOT A BOOLEAN").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse value [NOT A BOOLEAN] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]");
+            assertEquals(ex.getMessage(), "Failed to parse value [NOT A BOOLEAN] as only [true] or [false] are allowed.");
         }
         assertTrue(log.isReformat());
     }
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
index 0f72e72f6a2..ca089e6eb83 100644
--- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -95,14 +96,15 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
 
         // Update with RefreshPolicy.WAIT_UNTIL
-        UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
+        UpdateResponse update = client().prepareUpdate("test", "test", "1")
+            .setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
                 .get();
         assertEquals(2, update.getVersion());
         assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
 
         // Upsert with RefreshPolicy.WAIT_UNTIL
-        update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat")
+        update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "cat")
                 .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
         assertEquals(1, update.getVersion());
         assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
@@ -125,7 +127,7 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
 
         // Update by bulk with RefreshPolicy.WAIT_UNTIL
         bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
-        bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz"));
+        bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz"));
         assertBulkSuccess(bulk.get());
         assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
index 4a5a0b95672..9d9631e1b00 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisRegistryTests.java
@@ -65,7 +65,7 @@ public class AnalysisRegistryTests extends ESTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         registry = new AnalysisRegistry(new Environment(settings),
-            emptyMap(), emptyMap(), emptyMap(), emptyMap());
+            emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap());
     }
 
     public void testDefaultAnalyzers() throws IOException {
@@ -76,7 +76,8 @@ public class AnalysisRegistryTests extends ESTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
             .build(idxSettings);
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
@@ -88,7 +89,7 @@ public class AnalysisRegistryTests extends ESTestCase {
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
             singletonMap("default", analyzerProvider("default"))
-                , emptyMap(), emptyMap(), emptyMap());
+                , emptyMap(), emptyMap(), emptyMap(), emptyMap());
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
@@ -100,7 +101,7 @@ public class AnalysisRegistryTests extends ESTestCase {
         AnalyzerProvider<?> defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> registry.build(IndexSettingsModule.newIndexSettings("index", settings),
-                        singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap()));
+                        singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
     }
 
@@ -109,17 +110,19 @@ public class AnalysisRegistryTests extends ESTestCase {
                 VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
-                singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap());
+                singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap(), emptyMap());
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
     }
 
     public void testOverrideDefaultSearchAnalyzer() {
         Version version = VersionUtils.randomVersion(random());
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
-                singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap());
+                singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap(), emptyMap());
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
@@ -133,10 +136,12 @@ public class AnalysisRegistryTests extends ESTestCase {
         analyzers.put("default_index", analyzerProvider("default_index"));
         analyzers.put("default_search", analyzerProvider("default_search"));
         IndexAnalyzers indexAnalyzers = registry.build(IndexSettingsModule.newIndexSettings("index", settings),
-                analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+                analyzers, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
         assertThat(indexAnalyzers.getDefaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(indexAnalyzers.getDefaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
+        assertWarnings("setting [index.analysis.analyzer.default_index] is deprecated, use [index.analysis.analyzer.default] " +
+                "instead for index [index]");
     }
 
     public void testConfigureCamelCaseTokenFilter() throws IOException {
@@ -192,10 +197,11 @@ public class AnalysisRegistryTests extends ESTestCase {
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
                 .build(idxSettings);
         IndexAnalyzers otherIndexAnalyzers = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(),
-                emptyMap()).build(idxSettings);
+                emptyMap(), emptyMap()).build(idxSettings);
         final int numIters = randomIntBetween(5, 20);
         for (int i = 0; i < numIters; i++) {
             PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
@@ -215,7 +221,8 @@ public class AnalysisRegistryTests extends ESTestCase {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
+                () -> new AnalysisRegistry(new Environment(settings),
+                        emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
         assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
     }
 
@@ -224,7 +231,8 @@ public class AnalysisRegistryTests extends ESTestCase {
         Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+        IndexAnalyzers indexAnalyzers = new AnalysisRegistry(new Environment(settings),
+                emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap())
             .build(idxSettings);
         indexAnalyzers.close();
         indexAnalyzers.close();
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java
new file mode 100644
index 00000000000..3e71a609737
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CustomNormalizerTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+
+import java.io.IOException;
+
+public class CustomNormalizerTests extends ESTokenStreamTestCase {
+
+    public void testBasics() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase", "asciifolding")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "Cet été-là"), new String[] {"cet ete-la"});
+        assertEquals(new BytesRef("cet ete-la"), normalizer.normalize("foo", "Cet été-là"));
+    }
+
+    public void testUnknownType() {
+        Settings settings = Settings.builder()
+                .put("index.analysis.normalizer.my_normalizer.type", "foobar")
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase", "asciifolding")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Unknown normalizer type [foobar] for [my_normalizer]", e.getMessage());
+    }
+
+    public void testTokenizer() throws IOException {
+        Settings settings = Settings.builder()
+                .put("index.analysis.normalizer.my_normalizer.tokenizer", "keyword")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] cannot configure a tokenizer", e.getMessage());
+    }
+
+    public void testCharFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .put("index.analysis.char_filter.my_mapping.type", "mapping")
+                .putArray("index.analysis.char_filter.my_mapping.mappings", "a => z")
+                .putArray("index.analysis.normalizer.my_normalizer.char_filter", "my_mapping")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        ESTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(settings);
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertTokenStreamContents(normalizer.tokenStream("foo", "abc"), new String[] {"zbc"});
+        assertEquals(new BytesRef("zbc"), normalizer.normalize("foo", "abc"));
+    }
+
+    public void testIllegalFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.filter", "porter_stem")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] may not use filter [porter_stem]", e.getMessage());
+    }
+
+    public void testIllegalCharFilters() throws IOException {
+        Settings settings = Settings.builder()
+                .putArray("index.analysis.normalizer.my_normalizer.char_filter", "html_strip")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+                .build();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> AnalysisTestsHelper.createTestAnalysisFromSettings(settings));
+        assertEquals("Custom normalizer [my_normalizer] may not use char filter [html_strip]", e.getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java
new file mode 100644
index 00000000000..259da010daa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/FlattenGraphTokenFilterFactoryTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESTokenStreamTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+
+public class FlattenGraphTokenFilterFactoryTests extends ESTokenStreamTestCase {
+
+    public void testBasic() throws IOException {
+
+        Index index = new Index("test", "_na_");
+        String name = "ngr";
+        Settings indexSettings = newAnalysisSettingsBuilder().build();
+        IndexSettings indexProperties = IndexSettingsModule.newIndexSettings(index, indexSettings);
+        Settings settings = newAnalysisSettingsBuilder().build();
+
+        // "wow that's funny" and "what the fudge" are separate side paths, in parallel with "wtf", on input:
+        TokenStream in = new CannedTokenStream(0, 12, new Token[] {
+                    token("wtf", 1, 5, 0, 3),
+                    token("what", 0, 1, 0, 3),
+                    token("wow", 0, 3, 0, 3),
+                    token("the", 1, 1, 0, 3),
+                    token("fudge", 1, 3, 0, 3),
+                    token("that's", 1, 1, 0, 3),
+                    token("funny", 1, 1, 0, 3),
+                    token("happened", 1, 1, 4, 12)
+                });
+
+        TokenStream tokens = new FlattenGraphTokenFilterFactory(indexProperties, null, name, settings).create(in);
+
+        // ... but on output, it's flattened to wtf/what/wow that's/the fudge/funny happened:
+        assertTokenStreamContents(tokens,
+                new String[] {"wtf", "what", "wow", "the", "that's", "fudge", "funny", "happened"},
+                new int[] {0, 0, 0, 0, 0, 0, 0, 4},
+                new int[] {3, 3, 3, 3, 3, 3, 3, 12},
+                new int[] {1, 0, 0, 1, 0, 1, 0, 1},
+                new int[] {3, 1, 1, 1, 1, 1, 1, 1},
+                12);
+    }
+
+    private static Token token(String term, int posInc, int posLength, int startOffset, int endOffset) {
+        final Token t = new Token(term, startOffset, endOffset);
+        t.setPositionIncrement(posInc);
+        t.setPositionLength(posLength);
+        return t;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index ea89acbd8f8..e49e21bd2ff 100644
--- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -89,7 +89,7 @@ public class CodecTests extends ESTestCase {
         dir.close();
     }
 
-    private static CodecService createCodecService() throws IOException {
+    private CodecService createCodecService() throws IOException {
         Settings nodeSettings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
@@ -97,7 +97,8 @@ public class CodecTests extends ESTestCase {
         SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap());
         IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers;
         MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap());
-        MapperService service = new MapperService(settings, indexAnalyzers, similarityService, mapperRegistry, () -> null);
+        MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry,
+                () -> null);
         return new CodecService(service, ESLoggerFactory.getLogger("test"));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 523efb6829c..8de796a7612 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -32,6 +32,7 @@ import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.DirectoryReader;
@@ -63,6 +64,7 @@ import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
@@ -85,6 +87,8 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
@@ -105,9 +109,10 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.RootObjectMapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -124,6 +129,7 @@ import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.OldIndexUtils;
+import org.elasticsearch.test.rest.yaml.section.Assertion;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.MatcherAssert;
@@ -132,6 +138,8 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Reader;
+import java.io.UncheckedIOException;
 import java.nio.charset.Charset;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -144,6 +152,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
@@ -259,19 +268,22 @@ public class InternalEngineTests extends ESTestCase {
     }
 
 
-    private Document testDocumentWithTextField() {
+    private static Document testDocumentWithTextField() {
         Document document = testDocument();
         document.add(new TextField("value", "test", Field.Store.YES));
         return document;
     }
 
-    private Document testDocument() {
+    private static Document testDocument() {
         return new Document();
     }
 
+    public static ParsedDocument createParsedDoc(String id, String type, String routing) {
+        return testParsedDocument(id, type, routing, testDocumentWithTextField(), new BytesArray("{ \"value\" : \"test\" }"), null);
+    }
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, Document document, BytesReference source, Mapping mappingUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    private static ParsedDocument testParsedDocument(String id, String type, String routing, Document document, BytesReference source, Mapping mappingUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
         document.add(uidField);
@@ -279,7 +291,8 @@ public class InternalEngineTests extends ESTestCase {
         document.add(seqID.seqNo);
         document.add(seqID.seqNoDocValue);
         document.add(seqID.primaryTerm);
-        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate);
+        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON,
+            mappingUpdate);
     }
 
     protected Store createStore() throws IOException {
@@ -378,7 +391,7 @@ public class InternalEngineTests extends ESTestCase {
         };
         EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, deletionPolicy,
                 mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
-                new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(),
+                new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(),
                 IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), refreshListener,
             maxUnsafeAutoIdTimestamp);
 
@@ -399,11 +412,11 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
 
             // create two docs and refresh
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            Engine.Index first = new Engine.Index(newUid("1"), doc);
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            Engine.Index first = indexForDoc(doc);
             Engine.IndexResult firstResult = engine.index(first);
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null);
-            Engine.Index second = new Engine.Index(newUid("2"), doc2);
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null);
+            Engine.Index second = indexForDoc(doc2);
             Engine.IndexResult secondResult = engine.index(second);
             assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation()));
             engine.refresh("test");
@@ -435,8 +448,8 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
             assertThat(segments.get(0).isCompound(), equalTo(true));
 
-            ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("3"), doc3));
+            ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc3));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -462,7 +475,7 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(segments.get(1).isCompound(), equalTo(true));
 
 
-            engine.delete(new Engine.Delete("test", "1", newUid("1")));
+            engine.delete(new Engine.Delete("test", "1", newUid(doc)));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -482,8 +495,8 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(segments.get(1).isCompound(), equalTo(true));
 
             engine.onSettingsChanged();
-            ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("4"), doc4));
+            ParsedDocument doc4 = testParsedDocument("4", "test", null, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc4));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -516,19 +529,19 @@ public class InternalEngineTests extends ESTestCase {
             List<Segment> segments = engine.segments(true);
             assertThat(segments.isEmpty(), equalTo(true));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.refresh("test");
 
             segments = engine.segments(true);
             assertThat(segments.size(), equalTo(1));
             assertThat(segments.get(0).ramTree, notNullValue());
 
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null);
-            engine.index(new Engine.Index(newUid("2"), doc2));
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null);
+            engine.index(indexForDoc(doc2));
             engine.refresh("test");
-            ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null);
-            engine.index(new Engine.Index(newUid("3"), doc3));
+            ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null);
+            engine.index(indexForDoc(doc3));
             engine.refresh("test");
 
             segments = engine.segments(true);
@@ -542,12 +555,12 @@ public class InternalEngineTests extends ESTestCase {
     public void testSegmentsWithMergeFlag() throws Exception {
         try (Store store = createStore();
             Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-            Engine.Index index = new Engine.Index(newUid("1"), doc);
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
             engine.index(index);
             engine.flush();
             assertThat(engine.segments(false).size(), equalTo(1));
-            index = new Engine.Index(newUid("2"), doc);
+            index = indexForDoc(testParsedDocument("2", "test", null, testDocument(), B_1, null));
             engine.index(index);
             engine.flush();
             List<Segment> segments = engine.segments(false);
@@ -555,7 +568,7 @@ public class InternalEngineTests extends ESTestCase {
             for (Segment segment : segments) {
                 assertThat(segment.getMergeId(), nullValue());
             }
-            index = new Engine.Index(newUid("3"), doc);
+            index = indexForDoc(testParsedDocument("3", "test", null, testDocument(), B_1, null));
             engine.index(index);
             engine.flush();
             segments = engine.segments(false);
@@ -564,7 +577,7 @@ public class InternalEngineTests extends ESTestCase {
                 assertThat(segment.getMergeId(), nullValue());
             }
 
-            index = new Engine.Index(newUid("4"), doc);
+            index = indexForDoc(doc);
             engine.index(index);
             engine.flush();
             final long gen1 = store.readLastCommittedSegmentsInfo().getGeneration();
@@ -596,8 +609,8 @@ public class InternalEngineTests extends ESTestCase {
             Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
             assertThat(engine.segmentsStats(true).getFileSizes().size(), equalTo(0));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.refresh("test");
 
             SegmentsStats stats = engine.segmentsStats(true);
@@ -606,8 +619,8 @@ public class InternalEngineTests extends ESTestCase {
 
             ObjectObjectCursor<String, Long> firstEntry = stats.getFileSizes().iterator().next();
 
-            ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null);
-            engine.index(new Engine.Index(newUid("2"), doc2));
+            ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null);
+            engine.index(indexForDoc(doc2));
             engine.refresh("test");
 
             assertThat(engine.segmentsStats(true).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value));
@@ -638,14 +651,14 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(stats1.getGeneration(), greaterThan(0L));
             assertThat(stats1.getId(), notNullValue());
             assertThat(stats1.getUserData(), hasKey(Translog.TRANSLOG_GENERATION_KEY));
-            assertThat(stats1.getUserData(), hasKey(InternalEngine.LOCAL_CHECKPOINT_KEY));
+            assertThat(stats1.getUserData(), hasKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
             assertThat(
-                Long.parseLong(stats1.getUserData().get(InternalEngine.LOCAL_CHECKPOINT_KEY)),
+                Long.parseLong(stats1.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)),
                 equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
 
-            assertThat(stats1.getUserData(), hasKey(InternalEngine.MAX_SEQ_NO));
+            assertThat(stats1.getUserData(), hasKey(SequenceNumbers.MAX_SEQ_NO));
             assertThat(
-                Long.parseLong(stats1.getUserData().get(InternalEngine.MAX_SEQ_NO)),
+                Long.parseLong(stats1.getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
                 equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
 
             maxSeqNo.set(rarely() ? SequenceNumbersService.NO_OPS_PERFORMED : randomIntBetween(0, 1024));
@@ -667,9 +680,9 @@ public class InternalEngineTests extends ESTestCase {
                 stats2.getUserData().get(Translog.TRANSLOG_GENERATION_KEY),
                 not(equalTo(stats1.getUserData().get(Translog.TRANSLOG_GENERATION_KEY))));
             assertThat(stats2.getUserData().get(Translog.TRANSLOG_UUID_KEY), equalTo(stats1.getUserData().get(Translog.TRANSLOG_UUID_KEY)));
-            assertThat(Long.parseLong(stats2.getUserData().get(InternalEngine.LOCAL_CHECKPOINT_KEY)), equalTo(localCheckpoint.get()));
-            assertThat(stats2.getUserData(), hasKey(InternalEngine.MAX_SEQ_NO));
-            assertThat(Long.parseLong(stats2.getUserData().get(InternalEngine.MAX_SEQ_NO)), equalTo(maxSeqNo.get()));
+            assertThat(Long.parseLong(stats2.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(localCheckpoint.get()));
+            assertThat(stats2.getUserData(), hasKey(SequenceNumbers.MAX_SEQ_NO));
+            assertThat(Long.parseLong(stats2.getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(maxSeqNo.get()));
         } finally {
             IOUtils.close(engine);
         }
@@ -707,8 +720,8 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testFlushIsDisabledDuringTranslogRecovery() throws IOException {
         assertFalse(engine.isRecovering());
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), SOURCE, null);
+        engine.index(indexForDoc(doc));
         engine.close();
 
         engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
@@ -716,8 +729,8 @@ public class InternalEngineTests extends ESTestCase {
         assertTrue(engine.isRecovering());
         engine.recoverFromTranslog();
         assertFalse(engine.isRecovering());
-        doc = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("2"), doc));
+        doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), SOURCE, null);
+        engine.index(indexForDoc(doc));
         engine.flush();
     }
 
@@ -728,13 +741,13 @@ public class InternalEngineTests extends ESTestCase {
         try {
             initialEngine = engine;
             for (int i = 0; i < ops; i++) {
-                final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), SOURCE, null);
+                final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), SOURCE, null);
                 if (randomBoolean()) {
-                    final Engine.Index operation = new Engine.Index(newUid("test#1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
+                    final Engine.Index operation = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
                     operations.add(operation);
                     initialEngine.index(operation);
                 } else {
-                    final Engine.Delete operation = new Engine.Delete("test", "1", newUid("test#1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime());
+                    final Engine.Delete operation = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime());
                     operations.add(operation);
                     initialEngine.delete(operation);
                 }
@@ -764,8 +777,8 @@ public class InternalEngineTests extends ESTestCase {
             initialEngine = engine;
             for (int i = 0; i < docs; i++) {
                 final String id = Integer.toString(i);
-                final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null);
-                initialEngine.index(new Engine.Index(newUid(id), doc));
+                final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                initialEngine.index(indexForDoc(doc));
             }
         } finally {
             IOUtils.close(initialEngine);
@@ -793,33 +806,30 @@ public class InternalEngineTests extends ESTestCase {
     }
 
     public void testConcurrentGetAndFlush() throws Exception {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
 
         final AtomicReference<Engine.GetResult> latestGetResult = new AtomicReference<>();
-        latestGetResult.set(engine.get(new Engine.Get(true, newUid("1"))));
+        latestGetResult.set(engine.get(new Engine.Get(true, newUid(doc))));
         final AtomicBoolean flushFinished = new AtomicBoolean(false);
         final CyclicBarrier barrier = new CyclicBarrier(2);
-        Thread getThread = new Thread() {
-            @Override
-            public void run() {
-                try {
-                    barrier.await();
-                } catch (InterruptedException | BrokenBarrierException e) {
-                    throw new RuntimeException(e);
+        Thread getThread = new Thread(() -> {
+            try {
+                barrier.await();
+            } catch (InterruptedException | BrokenBarrierException e) {
+                throw new RuntimeException(e);
+            }
+            while (flushFinished.get() == false) {
+                Engine.GetResult previousGetResult = latestGetResult.get();
+                if (previousGetResult != null) {
+                    previousGetResult.release();
                 }
-                while (flushFinished.get() == false) {
-                    Engine.GetResult previousGetResult = latestGetResult.get();
-                    if (previousGetResult != null) {
-                        previousGetResult.release();
-                    }
-                    latestGetResult.set(engine.get(new Engine.Get(true, newUid("1"))));
-                    if (latestGetResult.get().exists() == false) {
-                        break;
-                    }
+                latestGetResult.set(engine.get(new Engine.Get(true, newUid(doc))));
+                if (latestGetResult.get().exists() == false) {
+                    break;
                 }
             }
-        };
+        });
         getThread.start();
         barrier.await();
         engine.flush();
@@ -837,8 +847,8 @@ public class InternalEngineTests extends ESTestCase {
         // create a document
         Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -847,12 +857,12 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, not there non realtime
-        Engine.GetResult getResult = engine.get(new Engine.Get(false, newUid("1")));
+        Engine.GetResult getResult = engine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // but, we can still get it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -867,7 +877,7 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // also in non realtime
-        getResult = engine.get(new Engine.Get(false, newUid("1")));
+        getResult = engine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -876,8 +886,8 @@ public class InternalEngineTests extends ESTestCase {
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_2, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_2, null);
+        engine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = engine.acquireSearcher("test");
@@ -887,7 +897,7 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -902,7 +912,7 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // now delete
-        engine.delete(new Engine.Delete("test", "1", newUid("1")));
+        engine.delete(new Engine.Delete("test", "1", newUid(doc)));
 
         // its not deleted yet
         searchResult = engine.acquireSearcher("test");
@@ -912,7 +922,7 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, get should not see it (in realtime)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
@@ -928,8 +938,8 @@ public class InternalEngineTests extends ESTestCase {
         // add it back
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -952,7 +962,7 @@ public class InternalEngineTests extends ESTestCase {
         engine.flush();
 
         // and, verify get (in real time)
-        getResult = engine.get(new Engine.Get(true, newUid("1")));
+        getResult = engine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -961,8 +971,8 @@ public class InternalEngineTests extends ESTestCase {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = engine.acquireSearcher("test");
@@ -987,8 +997,8 @@ public class InternalEngineTests extends ESTestCase {
         searchResult.close();
 
         // create a document
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = engine.acquireSearcher("test");
@@ -1006,7 +1016,7 @@ public class InternalEngineTests extends ESTestCase {
         // don't release the search result yet...
 
         // delete, refresh and do a new search, it should not be there
-        engine.delete(new Engine.Delete("test", "1", newUid("1")));
+        engine.delete(new Engine.Delete("test", "1", newUid(doc)));
         engine.refresh("test");
         Engine.Searcher updateSearchResult = engine.acquireSearcher("test");
         MatcherAssert.assertThat(updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
@@ -1023,8 +1033,8 @@ public class InternalEngineTests extends ESTestCase {
             Engine engine = new InternalEngine(config(defaultSettings, store, createTempDir(),
                      new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
             final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             Engine.CommitId commitID = engine.flush();
             assertThat(commitID, equalTo(new Engine.CommitId(store.readLastCommittedSegmentsInfo().getId())));
             byte[] wrongBytes = Base64.getDecoder().decode(commitID.toString());
@@ -1032,7 +1042,7 @@ public class InternalEngineTests extends ESTestCase {
             Engine.CommitId wrongId = new Engine.CommitId(wrongBytes);
             assertEquals("should fail to sync flush with wrong id (but no docs)", engine.syncFlush(syncId + "1", wrongId),
                     Engine.SyncedFlushResult.COMMIT_MISMATCH);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             assertEquals("should fail to sync flush with right id but pending doc", engine.syncFlush(syncId + "2", commitID),
                     Engine.SyncedFlushResult.PENDING_OPERATIONS);
             commitID = engine.flush();
@@ -1050,20 +1060,20 @@ public class InternalEngineTests extends ESTestCase {
                  InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(),
                          new LogDocMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
                 final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-                ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-                Engine.Index doc1 = new Engine.Index(newUid("1"), doc);
+                Engine.Index doc1 = indexForDoc(testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null));
                 engine.index(doc1);
                 assertEquals(engine.getLastWriteNanos(), doc1.startTime());
                 engine.flush();
-                Engine.Index doc2 = new Engine.Index(newUid("2"), doc);
+                Engine.Index doc2 = indexForDoc(testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null));
                 engine.index(doc2);
                 assertEquals(engine.getLastWriteNanos(), doc2.startTime());
                 engine.flush();
                 final boolean forceMergeFlushes = randomBoolean();
+                final ParsedDocument parsedDoc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_1, null);
                 if (forceMergeFlushes) {
-                    engine.index(new Engine.Index(newUid("3"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false));
+                    engine.index(new Engine.Index(newUid(parsedDoc3), parsedDoc3, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime() - engine.engineConfig.getFlushMergesAfter().nanos(), -1, false));
                 } else {
-                    engine.index(new Engine.Index(newUid("3"), doc));
+                    engine.index(indexForDoc(parsedDoc3));
                 }
                 Engine.CommitId commitID = engine.flush();
                 assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
@@ -1085,7 +1095,7 @@ public class InternalEngineTests extends ESTestCase {
                 assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
 
                 if (randomBoolean()) {
-                    Engine.Index doc4 = new Engine.Index(newUid("4"), doc);
+                    Engine.Index doc4 = indexForDoc(testParsedDocument("4", "test", null, testDocumentWithTextField(), B_1, null));
                     engine.index(doc4);
                     assertEquals(engine.getLastWriteNanos(), doc4.startTime());
                 } else {
@@ -1103,8 +1113,8 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testSyncedFlushSurvivesEngineRestart() throws IOException {
         final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
         final Engine.CommitId commitID = engine.flush();
         assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
                 Engine.SyncedFlushResult.SUCCESS);
@@ -1126,15 +1136,15 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testSyncedFlushVanishesOnReplay() throws IOException {
         final String syncId = randomUnicodeOfCodepointLengthBetween(10, 20);
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        engine.index(indexForDoc(doc));
         final Engine.CommitId commitID = engine.flush();
         assertEquals("should succeed to flush commit with right id and no pending doc", engine.syncFlush(syncId, commitID),
                 Engine.SyncedFlushResult.SUCCESS);
         assertEquals(store.readLastCommittedSegmentsInfo().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
         assertEquals(engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID), syncId);
-        doc = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), new BytesArray("{}"), null);
-        engine.index(new Engine.Index(newUid("2"), doc));
+        doc = testParsedDocument("2", "test", null, testDocumentWithTextField(), new BytesArray("{}"), null);
+        engine.index(indexForDoc(doc));
         EngineConfig config = engine.config();
         engine.close();
         engine = new InternalEngine(copy(config, EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
@@ -1142,80 +1152,80 @@ public class InternalEngineTests extends ESTestCase {
         assertNull("Sync ID must be gone since we have a document to replay", engine.getLastCommittedSegmentInfos().getUserData().get(Engine.SYNC_COMMIT_ID));
     }
 
-    public void testVersioningNewCreate() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED);
+    public void testVersioningNewCreate() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index create = new Engine.Index(newUid(doc), doc, Versions.MATCH_DELETED);
         Engine.IndexResult indexResult = engine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        create = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        create = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), create.primaryTerm(), indexResult.getVersion(), create.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
     }
 
-    public void testVersioningNewIndex() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testVersioningNewIndex() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
     }
 
-    public void testExternalVersioningNewIndex() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+    public void testExternalVersioningNewIndex() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(12L));
 
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getVersion(), equalTo(12L));
     }
 
-    public void testVersioningIndexConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testVersioningIndexConflict() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(2L));
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
         // future versions should not work as well
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testExternalVersioningIndexConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+    public void testExternalVersioningIndexConflict() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(12L));
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(14L));
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
     public void testForceVersioningNotAllowedExceptForOlderIndices() throws Exception {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 42, VersionType.FORCE, PRIMARY, 0, -1, false);
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 42, VersionType.FORCE, PRIMARY, 0, -1, false);
 
         Engine.IndexResult indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
@@ -1227,56 +1237,56 @@ public class InternalEngineTests extends ESTestCase {
                 .build());
         try (Store store = createStore();
                 Engine engine = createEngine(oldIndexSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
-            index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, PRIMARY, 0, -1, false);
+            index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE, PRIMARY, 0, -1, false);
             Engine.IndexResult result = engine.index(index);
             assertTrue(result.hasFailure());
             assertThat(result.getFailure(), instanceOf(IllegalArgumentException.class));
             assertThat(result.getFailure().getMessage(), containsString("version type [FORCE] may not be used for non-translog operations"));
 
-            index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE,
+            index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 84, VersionType.FORCE,
                     Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY, 0, -1, false);
             result = engine.index(index);
             assertThat(result.getVersion(), equalTo(84L));
         }
     }
 
-    public void testVersioningIndexConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testVersioningIndexConflictWithFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(2L));
 
         engine.flush();
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
         // future versions should not work as well
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testExternalVersioningIndexConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+    public void testExternalVersioningIndexConflictWithFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 12, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(12L));
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 14, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(14L));
 
         engine.flush();
 
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 13, VersionType.EXTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
@@ -1288,8 +1298,8 @@ public class InternalEngineTests extends ESTestCase {
                      new LogByteSizeMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) { // use log MP here we test some behavior in ESMP
             int numDocs = randomIntBetween(10, 100);
             for (int i = 0; i < numDocs; i++) {
-                ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), B_1, null);
-                Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc);
+                ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), B_1, null);
+                Engine.Index index = indexForDoc(doc);
                 engine.index(index);
                 engine.refresh("test");
             }
@@ -1299,8 +1309,8 @@ public class InternalEngineTests extends ESTestCase {
             engine.forceMerge(true, 1, false, false, false);
             assertEquals(engine.segments(true).size(), 1);
 
-            ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, testDocument(), B_1, null);
-            Engine.Index index = new Engine.Index(newUid(Integer.toString(0)), doc);
+            ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
             engine.delete(new Engine.Delete(index.type(), index.id(), index.uid()));
             engine.forceMerge(true, 10, true, false, false); //expunge deletes
 
@@ -1310,8 +1320,8 @@ public class InternalEngineTests extends ESTestCase {
                 assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.reader().maxDoc());
             }
 
-            doc = testParsedDocument(Integer.toString(1), Integer.toString(1), "test", null, testDocument(), B_1, null);
-            index = new Engine.Index(newUid(Integer.toString(1)), doc);
+            doc = testParsedDocument(Integer.toString(1), "test", null, testDocument(), B_1, null);
+            index = indexForDoc(doc);
             engine.delete(new Engine.Delete(index.type(), index.id(), index.uid()));
             engine.forceMerge(true, 10, false, false, false); //expunge deletes
 
@@ -1345,8 +1355,8 @@ public class InternalEngineTests extends ESTestCase {
                                 int numDocs = randomIntBetween(1, 20);
                                 for (int j = 0; j < numDocs; j++) {
                                     i++;
-                                    ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), B_1, null);
-                                    Engine.Index index = new Engine.Index(newUid(Integer.toString(i)), doc);
+                                    ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), B_1, null);
+                                    Engine.Index index = indexForDoc(doc);
                                     engine.index(index);
                                 }
                                 engine.refresh("test");
@@ -1357,8 +1367,10 @@ public class InternalEngineTests extends ESTestCase {
                                     return;
                                 }
                             }
-                        } catch (AlreadyClosedException | EngineClosedException ex) {
+                        } catch (AlreadyClosedException ex) {
                             // fine
+                        } catch (IOException e) {
+                            throw new AssertionError(e);
                         }
                     }
                 };
@@ -1377,58 +1389,58 @@ public class InternalEngineTests extends ESTestCase {
 
     }
 
-    public void testVersioningDeleteConflict() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testVersioningDeleteConflict() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(2L));
 
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0);
+        Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0);
         Engine.DeleteResult result = engine.delete(delete);
         assertTrue(result.hasFailure());
         assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
 
         // future versions should not work as well
-        delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0);
+        delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0);
         result = engine.delete(delete);
         assertTrue(result.hasFailure());
         assertThat(result.getFailure(), instanceOf(VersionConflictEngineException.class));
 
         // now actually delete
-        delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0);
+        delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0);
         result = engine.delete(delete);
         assertThat(result.getVersion(), equalTo(3L));
 
         // now check if we can index to a delete doc with version
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testVersioningDeleteConflictWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testVersioningDeleteConflictWithFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(2L));
 
         engine.flush();
 
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0);
+        Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1L, VersionType.INTERNAL, PRIMARY, 0);
         Engine.DeleteResult deleteResult = engine.delete(delete);
         assertTrue(deleteResult.hasFailure());
         assertThat(deleteResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
         // future versions should not work as well
-        delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0);
+        delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 3L, VersionType.INTERNAL, PRIMARY, 0);
         deleteResult = engine.delete(delete);
         assertTrue(deleteResult.hasFailure());
         assertThat(deleteResult.getFailure(), instanceOf(VersionConflictEngineException.class));
@@ -1436,146 +1448,191 @@ public class InternalEngineTests extends ESTestCase {
         engine.flush();
 
         // now actually delete
-        delete = new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0);
+        delete = new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0);
         deleteResult = engine.delete(delete);
         assertThat(deleteResult.getVersion(), equalTo(3L));
 
         engine.flush();
 
         // now check if we can index to a delete doc with version
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2L, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(index);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testVersioningCreateExistsException() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+    public void testVersioningCreateExistsException() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         Engine.IndexResult indexResult = engine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(create);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testVersioningCreateExistsExceptionWithFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+    public void testVersioningCreateExistsExceptionWithFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         Engine.IndexResult indexResult = engine.index(create);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
         engine.flush();
 
-        create = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
+        create = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, 0, -1, false);
         indexResult = engine.index(create);
         assertTrue(indexResult.hasFailure());
         assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
     }
 
-    public void testVersioningReplicaConflict1() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        Engine.IndexResult indexResult = engine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(1L));
+    public void testVersioningReplicaConflict1() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        final Engine.Index v1Index = indexForDoc(doc);
+        final Engine.IndexResult v1Result = engine.index(v1Index);
+        assertThat(v1Result.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc);
-        indexResult = engine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(2L));
+        final Engine.Index v2Index = indexForDoc(doc);
+        final Engine.IndexResult v2Result = engine.index(v2Index);
+        assertThat(v2Result.getVersion(), equalTo(2L));
 
         // apply the second index to the replica, should work fine
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA,  0, -1, false);
-        indexResult = replicaEngine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(2L));
+        final Engine.Index replicaV2Index = new Engine.Index(
+            newUid(doc),
+            doc,
+            v2Result.getSeqNo(),
+            v2Index.primaryTerm(),
+            v2Result.getVersion(),
+            VersionType.INTERNAL.versionTypeForReplicationAndRecovery(),
+            REPLICA,
+            0,
+            -1,
+            false);
+        final Engine.IndexResult replicaV2Result = replicaEngine.index(replicaV2Index);
+        assertThat(replicaV2Result.getVersion(), equalTo(2L));
 
-        long seqNo = indexResult.getSeqNo();
-        long primaryTerm = index.primaryTerm();
-        // now, the old one should not work
-        index = new Engine.Index(newUid("1"), doc, seqNo, primaryTerm, 1L, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        indexResult = replicaEngine.index(index);
-        assertTrue(indexResult.hasFailure());
-        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
+        // now, the old one should produce an indexing result
+        final Engine.Index replicaV1Index = new Engine.Index(
+            newUid(doc),
+            doc,
+            v1Result.getSeqNo(),
+            v1Index.primaryTerm(),
+            v1Result.getVersion(),
+            VersionType.INTERNAL.versionTypeForReplicationAndRecovery(),
+            REPLICA,
+            0,
+            -1,
+            false);
+        final Engine.IndexResult replicaV1Result = replicaEngine.index(replicaV1Index);
+        assertFalse(replicaV1Result.hasFailure());
+        assertFalse(replicaV1Result.isCreated());
+        assertThat(replicaV1Result.getVersion(), equalTo(2L));
 
         // second version on replica should fail as well
-        index = new Engine.Index(newUid("1"), doc, seqNo, primaryTerm, 2L
-                , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        indexResult = replicaEngine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(2L));
-        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
+        final Engine.IndexResult replicaV2ReplayResult = replicaEngine.index(replicaV2Index);
+        assertFalse(replicaV2Result.hasFailure());
+        assertFalse(replicaV1Result.isCreated());
+        assertThat(replicaV2ReplayResult.getVersion(), equalTo(2L));
     }
 
-    public void testVersioningReplicaConflict2() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
-        Engine.IndexResult indexResult = engine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(1L));
+    public void testVersioningReplicaConflict2() throws IOException {
+        final ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        final Engine.Index v1Index = indexForDoc(doc);
+        final Engine.IndexResult v1Result = engine.index(v1Index);
+        assertThat(v1Result.getVersion(), equalTo(1L));
 
         // apply the first index to the replica, should work fine
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), 1L,
-                VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        indexResult = replicaEngine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(1L));
+        final Engine.Index replicaV1Index = new Engine.Index(
+            newUid(doc),
+            doc,
+            v1Result.getSeqNo(),
+            v1Index.primaryTerm(),
+            v1Result.getVersion(),
+            VersionType.INTERNAL.versionTypeForReplicationAndRecovery(),
+            REPLICA,
+            0,
+            -1,
+            false);
+        Engine.IndexResult replicaV1Result = replicaEngine.index(replicaV1Index);
+        assertThat(replicaV1Result.getVersion(), equalTo(1L));
 
         // index it again
-        index = new Engine.Index(newUid("1"), doc);
-        indexResult = engine.index(index);
-        assertThat(indexResult.getVersion(), equalTo(2L));
+        final Engine.Index v2Index = indexForDoc(doc);
+        final Engine.IndexResult v2Result = engine.index(v2Index);
+        assertThat(v2Result.getVersion(), equalTo(2L));
 
         // now delete it
-        Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
-        Engine.DeleteResult deleteResult = engine.delete(delete);
+        final Engine.Delete delete = new Engine.Delete("test", "1", newUid(doc));
+        final Engine.DeleteResult deleteResult = engine.delete(delete);
         assertThat(deleteResult.getVersion(), equalTo(3L));
 
         // apply the delete on the replica (skipping the second index)
-        delete = new Engine.Delete("test", "1", newUid("1"), deleteResult.getSeqNo(), delete.primaryTerm(), 3L
-                , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0);
-        deleteResult = replicaEngine.delete(delete);
-        assertThat(deleteResult.getVersion(), equalTo(3L));
+        final Engine.Delete replicaDelete = new Engine.Delete(
+            "test",
+            "1",
+            newUid(doc),
+            deleteResult.getSeqNo(),
+            delete.primaryTerm(),
+            deleteResult.getVersion(),
+            VersionType.INTERNAL.versionTypeForReplicationAndRecovery(),
+            REPLICA,
+            0);
+        final Engine.DeleteResult replicaDeleteResult = replicaEngine.delete(replicaDelete);
+        assertThat(replicaDeleteResult.getVersion(), equalTo(3L));
 
-        // second time delete with same version should fail
-        delete = new Engine.Delete("test", "1", newUid("1"), deleteResult.getSeqNo(), delete.primaryTerm(), 3L
-                , VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0);
-        deleteResult = replicaEngine.delete(delete);
-        assertTrue(deleteResult.hasFailure());
-        assertThat(deleteResult.getFailure(), instanceOf(VersionConflictEngineException.class));
+        // second time delete with same version should just produce the same version
+        final Engine.DeleteResult deleteReplayResult = replicaEngine.delete(replicaDelete);
+        assertFalse(deleteReplayResult.hasFailure());
+        assertTrue(deleteReplayResult.isFound());
+        assertThat(deleteReplayResult.getVersion(), equalTo(3L));
 
-        // now do the second index on the replica, it should fail
-        index = new Engine.Index(newUid("1"), doc, deleteResult.getSeqNo(), delete.primaryTerm(), 2L, VersionType.INTERNAL.versionTypeForReplicationAndRecovery(), REPLICA, 0, -1, false);
-        indexResult = replicaEngine.index(index);
-        assertTrue(indexResult.hasFailure());
-        assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
+        // now do the second index on the replica, it should result in the current version
+        final Engine.Index replicaV2Index = new Engine.Index(
+            newUid(doc),
+            doc,
+            v2Result.getSeqNo(),
+            v2Index.primaryTerm(),
+            v2Result.getVersion(),
+            VersionType.INTERNAL.versionTypeForReplicationAndRecovery(),
+            REPLICA,
+            0,
+            -1,
+            false);
+        final Engine.IndexResult replicaV2Result = replicaEngine.index(replicaV2Index);
+        assertFalse(replicaV2Result.hasFailure());
+        assertFalse(replicaV2Result.isCreated());
+        assertThat(replicaV2Result.getVersion(), equalTo(3L));
     }
 
-    public void testBasicCreatedFlag() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testBasicCreatedFlag() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertTrue(indexResult.isCreated());
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertFalse(indexResult.isCreated());
 
-        engine.delete(new Engine.Delete(null, "1", newUid("1")));
+        engine.delete(new Engine.Delete(null, "1", newUid(doc)));
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertTrue(indexResult.isCreated());
     }
 
-    public void testCreatedFlagAfterFlush() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocument(), B_1, null);
-        Engine.Index index = new Engine.Index(newUid("1"), doc);
+    public void testCreatedFlagAfterFlush() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocument(), B_1, null);
+        Engine.Index index = indexForDoc(doc);
         Engine.IndexResult indexResult = engine.index(index);
         assertTrue(indexResult.isCreated());
 
-        engine.delete(new Engine.Delete(null, "1", newUid("1")));
+        engine.delete(new Engine.Delete(null, "1", newUid(doc)));
 
         engine.flush();
 
-        index = new Engine.Index(newUid("1"), doc);
+        index = indexForDoc(doc);
         indexResult = engine.index(index);
         assertTrue(indexResult.isCreated());
     }
@@ -1585,7 +1642,7 @@ public class InternalEngineTests extends ESTestCase {
 
         public boolean sawIndexWriterIFDMessage;
 
-        public MockAppender(final String name) throws IllegalAccessException {
+        MockAppender(final String name) throws IllegalAccessException {
             super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null);
         }
 
@@ -1607,7 +1664,7 @@ public class InternalEngineTests extends ESTestCase {
     // #5891: make sure IndexWriter's infoStream output is
     // sent to lucene.iw with log level TRACE:
 
-    public void testIndexWriterInfoStream() throws IllegalAccessException {
+    public void testIndexWriterInfoStream() throws IllegalAccessException, IOException {
         assumeFalse("who tests the tester?", VERBOSE);
         MockAppender mockAppender = new MockAppender("testIndexWriterInfoStream");
         mockAppender.start();
@@ -1620,14 +1677,14 @@ public class InternalEngineTests extends ESTestCase {
 
         try {
             // First, with DEBUG, which should NOT log IndexWriter output:
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
 
             // Again, with TRACE, which should log IndexWriter output:
             Loggers.setLevel(rootLogger, Level.TRACE);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertTrue(mockAppender.sawIndexWriterMessage);
 
@@ -1676,8 +1733,8 @@ public class InternalEngineTests extends ESTestCase {
                 } else {
                     // index a document
                     id = randomFrom(ids);
-                    ParsedDocument doc = testParsedDocument("test#" + id, id, "test", null, testDocumentWithTextField(), SOURCE, null);
-                    final Engine.Index index = new Engine.Index(newUid("test#" + id), doc,
+                    ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                    final Engine.Index index = new Engine.Index(newUid(doc), doc,
                         SequenceNumbersService.UNASSIGNED_SEQ_NO, 0,
                         rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL,
                         PRIMARY, 0, -1, false);
@@ -1718,14 +1775,14 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(initialEngine.seqNoService().stats().getGlobalCheckpoint(), equalTo(replicaLocalCheckpoint));
 
             assertThat(
-                Long.parseLong(initialEngine.commitStats().getUserData().get(InternalEngine.LOCAL_CHECKPOINT_KEY)),
+                Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)),
                 equalTo(localCheckpoint));
             initialEngine.getTranslog().sync(); // to guarantee the global checkpoint is written to the translog checkpoint
             assertThat(
                 initialEngine.getTranslog().getLastSyncedGlobalCheckpoint(),
                 equalTo(globalCheckpoint));
             assertThat(
-                Long.parseLong(initialEngine.commitStats().getUserData().get(InternalEngine.MAX_SEQ_NO)),
+                Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
                 equalTo(maxSeqNo));
 
         } finally {
@@ -1739,13 +1796,13 @@ public class InternalEngineTests extends ESTestCase {
 
             assertEquals(primarySeqNo, recoveringEngine.seqNoService().getMaxSeqNo());
             assertThat(
-                Long.parseLong(recoveringEngine.commitStats().getUserData().get(InternalEngine.LOCAL_CHECKPOINT_KEY)),
+                Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)),
                 equalTo(primarySeqNo));
             assertThat(
                 recoveringEngine.getTranslog().getLastSyncedGlobalCheckpoint(),
                 equalTo(globalCheckpoint));
             assertThat(
-                Long.parseLong(recoveringEngine.commitStats().getUserData().get(InternalEngine.MAX_SEQ_NO)),
+                Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)),
                 // after recovering from translog, all docs have been flushed to Lucene segments, so here we will assert
                 // that the committed max seq no is equivalent to what the current primary seq no is, as all data
                 // we have assigned sequence numbers to should be in the commit
@@ -1761,8 +1818,8 @@ public class InternalEngineTests extends ESTestCase {
     // this test writes documents to the engine while concurrently flushing/commit
     // and ensuring that the commit points contain the correct sequence number data
     public void testConcurrentWritesAndCommits() throws Exception {
-        try (final Store store = createStore();
-             final InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(),
+        try (Store store = createStore();
+             InternalEngine engine = new InternalEngine(config(defaultSettings, store, createTempDir(), newMergePolicy(),
                                                                      new SnapshotDeletionPolicy(NoDeletionPolicy.INSTANCE),
                                                                      IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, null))) {
 
@@ -1773,22 +1830,19 @@ public class InternalEngineTests extends ESTestCase {
             // create N indexing threads to index documents simultaneously
             for (int threadNum = 0; threadNum < numIndexingThreads; threadNum++) {
                 final int threadIdx = threadNum;
-                Thread indexingThread = new Thread() {
-                    @Override
-                    public void run() {
-                        try {
-                            barrier.await(); // wait for all threads to start at the same time
-                            // index random number of docs
-                            for (int i = 0; i < numDocsPerThread; i++) {
-                                final String id = "thread" + threadIdx + "#" + i;
-                                ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocument(), B_1, null);
-                                engine.index(new Engine.Index(newUid(id), doc));
-                            }
-                        } catch (Exception e) {
-                            throw new RuntimeException(e);
+                Thread indexingThread = new Thread(() -> {
+                    try {
+                        barrier.await(); // wait for all threads to start at the same time
+                        // index random number of docs
+                        for (int i = 0; i < numDocsPerThread; i++) {
+                            final String id = "thread" + threadIdx + "#" + i;
+                            ParsedDocument doc = testParsedDocument(id, "test", null, testDocument(), B_1, null);
+                            engine.index(indexForDoc(doc));
                         }
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
                     }
-                };
+                });
                 indexingThreads.add(indexingThread);
             }
 
@@ -1810,11 +1864,11 @@ public class InternalEngineTests extends ESTestCase {
             long prevMaxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
             for (IndexCommit commit : DirectoryReader.listCommits(store.directory())) {
                 Map<String, String> userData = commit.getUserData();
-                long localCheckpoint = userData.containsKey(InternalEngine.LOCAL_CHECKPOINT_KEY) ?
-                                           Long.parseLong(userData.get(InternalEngine.LOCAL_CHECKPOINT_KEY)) :
+                long localCheckpoint = userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) ?
+                                           Long.parseLong(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)) :
                                            SequenceNumbersService.NO_OPS_PERFORMED;
-                long maxSeqNo = userData.containsKey(InternalEngine.MAX_SEQ_NO) ?
-                                    Long.parseLong(userData.get(InternalEngine.MAX_SEQ_NO)) :
+                long maxSeqNo = userData.containsKey(SequenceNumbers.MAX_SEQ_NO) ?
+                                    Long.parseLong(userData.get(SequenceNumbers.MAX_SEQ_NO)) :
                                     SequenceNumbersService.UNASSIGNED_SEQ_NO;
                 // local checkpoint and max seq no shouldn't go backwards
                 assertThat(localCheckpoint, greaterThanOrEqualTo(prevLocalCheckpoint));
@@ -1871,7 +1925,7 @@ public class InternalEngineTests extends ESTestCase {
     }
 
     // #8603: make sure we can separately log IFD's messages
-    public void testIndexWriterIFDInfoStream() throws IllegalAccessException {
+    public void testIndexWriterIFDInfoStream() throws IllegalAccessException, IOException {
         assumeFalse("who tests the tester?", VERBOSE);
         MockAppender mockAppender = new MockAppender("testIndexWriterIFDInfoStream");
         mockAppender.start();
@@ -1883,15 +1937,15 @@ public class InternalEngineTests extends ESTestCase {
 
         try {
             // First, with DEBUG, which should NOT log IndexWriter output:
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
+            ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
             assertFalse(mockAppender.sawIndexWriterIFDMessage);
 
             // Again, with TRACE, which should only log IndexWriter IFD output:
             Loggers.setLevel(iwIFDLogger, Level.TRACE);
-            engine.index(new Engine.Index(newUid("2"), doc));
+            engine.index(indexForDoc(doc));
             engine.flush();
             assertFalse(mockAppender.sawIndexWriterMessage);
             assertTrue(mockAppender.sawIndexWriterIFDMessage);
@@ -1912,14 +1966,14 @@ public class InternalEngineTests extends ESTestCase {
             Document document = testDocument();
             document.add(new TextField("value", "test1", Field.Store.YES));
 
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_2, null);
-            engine.index(new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
+            ParsedDocument doc = testParsedDocument("1", "test", null, document, B_2, null);
+            engine.index(new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false));
 
             // Delete document we just added:
-            engine.delete(new Engine.Delete("test", "1", newUid("1"), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()));
+            engine.delete(new Engine.Delete("test", "1", newUid(doc), SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()));
 
             // Get should not find the document
-            Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
+            Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid(doc)));
             assertThat(getResult.exists(), equalTo(false));
 
             // Give the gc pruning logic a chance to kick in
@@ -1937,23 +1991,23 @@ public class InternalEngineTests extends ESTestCase {
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=1 with a too-old version, should fail:
-            Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
+            Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
             Engine.IndexResult indexResult = engine.index(index);
             assertTrue(indexResult.hasFailure());
             assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should still not find the document
-            getResult = engine.get(new Engine.Get(true, newUid("1")));
+            getResult = engine.get(new Engine.Get(true, newUid(doc)));
             assertThat(getResult.exists(), equalTo(false));
 
             // Try to index uid=2 with a too-old version, should fail:
-            Engine.Index index1 = new Engine.Index(newUid("2"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
+            Engine.Index index1 = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), -1, false);
             indexResult = engine.index(index1);
             assertTrue(indexResult.hasFailure());
             assertThat(indexResult.getFailure(), instanceOf(VersionConflictEngineException.class));
 
             // Get should not find the document
-            getResult = engine.get(new Engine.Get(true, newUid("2")));
+            getResult = engine.get(new Engine.Get(true, newUid(doc)));
             assertThat(getResult.exists(), equalTo(false));
         }
     }
@@ -1962,6 +2016,14 @@ public class InternalEngineTests extends ESTestCase {
         return new Term("_uid", id);
     }
 
+    protected Term newUid(ParsedDocument doc) {
+        return new Term("_uid", doc.uid());
+    }
+
+    private Engine.Index indexForDoc(ParsedDocument doc) {
+        return new Engine.Index(newUid(doc), doc);
+    }
+
     public void testExtractShardId() {
         try (Engine.Searcher test = this.engine.acquireSearcher("test")) {
             ShardId shardId = ShardUtils.extractShardId(test.getDirectoryReader());
@@ -2044,8 +2106,8 @@ public class InternalEngineTests extends ESTestCase {
     public void testTranslogReplayWithFailure() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
             Engine.IndexResult indexResult = engine.index(firstIndexRequest);
             assertThat(indexResult.getVersion(), equalTo(1L));
         }
@@ -2094,8 +2156,8 @@ public class InternalEngineTests extends ESTestCase {
     public void testSkipTranslogReplay() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
             Engine.IndexResult indexResult = engine.index(firstIndexRequest);
             assertThat(indexResult.getVersion(), equalTo(1L));
         }
@@ -2156,7 +2218,7 @@ public class InternalEngineTests extends ESTestCase {
             final long size = Files.size(tlogFile);
             logger.debug("upgrading index {} file: {} size: {}", indexName, tlogFiles[0].getFileName(), size);
             Directory directory = newFSDirectory(src.resolve("0").resolve("index"));
-            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, src);
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, xContentRegistry(), src);
             final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
             final Store store = createStore(indexSettings, directory);
             final int iters = randomIntBetween(0, 2);
@@ -2187,8 +2249,8 @@ public class InternalEngineTests extends ESTestCase {
                 }
                 final int numExtraDocs = randomIntBetween(1, 10);
                 for (int i = 0; i < numExtraDocs; i++) {
-                    ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "extra" + Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-                    Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+                    ParsedDocument doc = testParsedDocument("extra" + Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
+                    Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
                     Engine.IndexResult indexResult = engine.index(firstIndexRequest);
                     assertThat(indexResult.getVersion(), equalTo(1L));
                 }
@@ -2216,8 +2278,8 @@ public class InternalEngineTests extends ESTestCase {
     public void testTranslogReplay() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
             Engine.IndexResult indexResult = engine.index(firstIndexRequest);
             assertThat(indexResult.getVersion(), equalTo(1L));
         }
@@ -2258,17 +2320,16 @@ public class InternalEngineTests extends ESTestCase {
 
         final boolean flush = randomBoolean();
         int randomId = randomIntBetween(numDocs + 1, numDocs + 10);
-        String uuidValue = "test#" + Integer.toString(randomId);
-        ParsedDocument doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
-        Engine.Index firstIndexRequest = new Engine.Index(newUid(uuidValue), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
+        ParsedDocument doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
+        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 1, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
         Engine.IndexResult indexResult = engine.index(firstIndexRequest);
         assertThat(indexResult.getVersion(), equalTo(1L));
         if (flush) {
             engine.flush();
         }
 
-        doc = testParsedDocument(uuidValue, Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
-        Engine.Index idxRequest = new Engine.Index(newUid(uuidValue), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
+        doc = testParsedDocument(Integer.toString(randomId), "test", null, testDocument(), new BytesArray("{}"), null);
+        Engine.Index idxRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, 2, VersionType.EXTERNAL, PRIMARY, System.nanoTime(), -1, false);
         Engine.IndexResult result = engine.index(idxRequest);
         engine.refresh("test");
         assertThat(result.getVersion(), equalTo(2L));
@@ -2285,7 +2346,7 @@ public class InternalEngineTests extends ESTestCase {
         }
         parser = (TranslogHandler) engine.config().getTranslogRecoveryPerformer();
         assertEquals(flush ? 1 : 2, parser.recoveredOps.get());
-        engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(uuidValue)));
+        engine.delete(new Engine.Delete("test", Integer.toString(randomId), newUid(doc)));
         if (randomBoolean()) {
             engine.refresh("test");
         } else {
@@ -2305,16 +2366,17 @@ public class InternalEngineTests extends ESTestCase {
 
         public final AtomicInteger recoveredOps = new AtomicInteger(0);
 
-        public TranslogHandler(String indexName, Logger logger) {
+        public TranslogHandler(NamedXContentRegistry xContentRegistry, String indexName, Logger logger) {
             super(new ShardId("test", "_na_", 0), null, logger);
             Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
             Index index = new Index(indexName, "_na_");
             IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
             NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
-            IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, Collections.emptyMap());
+            IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, Collections.emptyMap(), Collections.emptyMap());
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
             MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
-            mapperService = new MapperService(indexSettings, indexAnalyzers, similarityService, mapperRegistry, () -> null);
+            mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
+                    () -> null);
         }
 
         @Override
@@ -2333,8 +2395,8 @@ public class InternalEngineTests extends ESTestCase {
     public void testRecoverFromForeignTranslog() throws IOException {
         final int numDocs = randomIntBetween(1, 10);
         for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
             Engine.IndexResult index = engine.index(firstIndexRequest);
             assertThat(index.getVersion(), equalTo(1L));
         }
@@ -2421,8 +2483,8 @@ public class InternalEngineTests extends ESTestCase {
 
             // create
             {
-                ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, testDocument(), new BytesArray("{}"), null);
-                Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
+                ParsedDocument doc = testParsedDocument(Integer.toString(0), "test", null, testDocument(), new BytesArray("{}"), null);
+                Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
 
                 try (InternalEngine engine = new InternalEngine(copy(config, EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG))){
                     assertFalse(engine.isRecovering());
@@ -2480,102 +2542,119 @@ public class InternalEngineTests extends ESTestCase {
         }
     }
 
-    public void testCheckDocumentFailure() throws Exception {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        Exception documentFailure = engine.checkIfDocumentFailureOrThrow(new Engine.Index(newUid("1"), doc), new IOException("simulated document failure"));
-        assertThat(documentFailure, instanceOf(IOException.class));
-        try {
-            engine.checkIfDocumentFailureOrThrow(new Engine.Index(newUid("1"), doc), new CorruptIndexException("simulated environment failure", ""));
-            fail("expected exception to be thrown");
-        } catch (Exception envirnomentException) {
-            assertThat(envirnomentException.getMessage(), containsString("simulated environment failure"));
-        }
-    }
-
     private static class ThrowingIndexWriter extends IndexWriter {
-        private boolean throwDocumentFailure;
+        private AtomicReference<Supplier<Exception>> failureToThrow = new AtomicReference<>();
 
-        public ThrowingIndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
+        ThrowingIndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
             super(d, conf);
         }
 
         @Override
         public long addDocument(Iterable<? extends IndexableField> doc) throws IOException {
-            if (throwDocumentFailure) {
-                throw new IOException("simulated");
-            } else {
-                return super.addDocument(doc);
+            maybeThrowFailure();
+            return super.addDocument(doc);
+        }
+
+        private void maybeThrowFailure() throws IOException {
+            if (failureToThrow.get() != null) {
+                Exception failure = failureToThrow.get().get();
+                if (failure instanceof RuntimeException) {
+                    throw (RuntimeException) failure;
+                } else if (failure instanceof IOException) {
+                    throw (IOException) failure;
+                } else {
+                    assert false: "unsupported failure class: " + failure.getClass().getCanonicalName();
+                }
             }
         }
 
         @Override
         public long deleteDocuments(Term... terms) throws IOException {
-            if (throwDocumentFailure) {
-                throw new IOException("simulated");
-            } else {
-                return super.deleteDocuments(terms);
-            }
+            maybeThrowFailure();
+            return super.deleteDocuments(terms);
         }
 
-        public void setThrowDocumentFailure(boolean throwDocumentFailure) {
-            this.throwDocumentFailure = throwDocumentFailure;
+        public void setThrowFailure(Supplier<Exception> failureSupplier) {
+            failureToThrow.set(failureSupplier);
+        }
+
+        public void clearFailure() {
+            failureToThrow.set(null);
         }
     }
 
     public void testHandleDocumentFailure() throws Exception {
         try (Store store = createStore()) {
-            ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
+            final ParsedDocument doc1 = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+            final ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_1, null);
+            final ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_1, null);
+
             ThrowingIndexWriter throwingIndexWriter = new ThrowingIndexWriter(store.directory(), new IndexWriterConfig());
             try (Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, () -> throwingIndexWriter)) {
                 // test document failure while indexing
-                throwingIndexWriter.setThrowDocumentFailure(true);
-                Engine.IndexResult indexResult = engine.index(randomAppendOnly(1, doc, false));
+                if (randomBoolean()) {
+                    throwingIndexWriter.setThrowFailure(() -> new IOException("simulated"));
+                } else {
+                    throwingIndexWriter.setThrowFailure(() -> new IllegalArgumentException("simulated max token length"));
+                }
+                Engine.IndexResult indexResult = engine.index(indexForDoc(doc1));
                 assertNotNull(indexResult.getFailure());
 
-                throwingIndexWriter.setThrowDocumentFailure(false);
-                indexResult = engine.index(randomAppendOnly(1, doc, false));
+                throwingIndexWriter.clearFailure();
+                indexResult = engine.index(indexForDoc(doc1));
                 assertNull(indexResult.getFailure());
+                engine.index(indexForDoc(doc2));
 
-                // test document failure while deleting
-                throwingIndexWriter.setThrowDocumentFailure(true);
-                Engine.DeleteResult deleteResult = engine.delete(new Engine.Delete("test", "", newUid("1")));
-                assertNotNull(deleteResult.getFailure());
+                // test failure while deleting
+                // all these simulated exceptions are not fatal to the IW so we treat them as document failures
+                if (randomBoolean()) {
+                    throwingIndexWriter.setThrowFailure(() -> new IOException("simulated"));
+                    expectThrows(IOException.class, () -> engine.delete(new Engine.Delete("test", "1", newUid(doc1))));
+                } else {
+                    throwingIndexWriter.setThrowFailure(() -> new IllegalArgumentException("simulated max token length"));
+                    expectThrows(IllegalArgumentException.class, () -> engine.delete(new Engine.Delete("test", "1", newUid(doc1))));
+                }
+
+                // test non document level failure is thrown
+                if (randomBoolean()) {
+                    // simulate close by corruption
+                    throwingIndexWriter.setThrowFailure(null);
+                    UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> {
+                        Engine.Index index = indexForDoc(doc3);
+                        index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") {
+                            // this is a hack to add a failure during store document which triggers a tragic event
+                            // and in turn fails the engine
+                            @Override
+                            public BytesRef binaryValue() {
+                                throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
+                            }
+                        });
+                        engine.index(index);
+                    });
+                    assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException);
+                } else {
+                    // normal close
+                    engine.close();
+                }
+                // now the engine is closed check we respond correctly
+                try {
+                    if (randomBoolean()) {
+                        engine.index(indexForDoc(doc1));
+                    } else {
+                        engine.delete(new Engine.Delete("test", "", newUid(doc1)));
+                    }
+                    fail("engine should be closed");
+                } catch (Exception e) {
+                    assertThat(e, instanceOf(AlreadyClosedException.class));
+                }
             }
         }
-
-    }
-
-    public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
-        for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            Engine.IndexResult indexResult = engine.index(firstIndexRequest);
-            assertThat(indexResult.getVersion(), equalTo(1L));
-        }
-        DocsStats docStats = engine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        engine.forceMerge(randomBoolean(), 1, false, false, false);
-
-        ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, testDocument(), new BytesArray("{}"), null);
-        Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-        Engine.IndexResult index = engine.index(firstIndexRequest);
-        assertThat(index.getVersion(), equalTo(2L));
-        engine.flush(); // flush - buffered deletes are not counted
-        docStats = engine.getDocStats();
-        assertEquals(1, docStats.getDeleted());
-        assertEquals(numDocs, docStats.getCount());
-        engine.forceMerge(randomBoolean(), 1, false, false, false);
-        docStats = engine.getDocStats();
-        assertEquals(0, docStats.getDeleted());
-        assertEquals(numDocs, docStats.getCount());
     }
 
     public void testDoubleDelivery() throws IOException {
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-        Engine.Index operation = randomAppendOnly(1, doc, false);
-        Engine.Index retry = randomAppendOnly(1, doc, true);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        Engine.Index operation = randomAppendOnly(doc, false, 1);
+        Engine.Index retry = randomAppendOnly(doc, true, 1);
         if (randomBoolean()) {
             Engine.IndexResult indexResult = engine.index(operation);
             assertFalse(engine.indexWriterHasDeletions());
@@ -2603,8 +2682,8 @@ public class InternalEngineTests extends ESTestCase {
             TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), 10);
             assertEquals(1, topDocs.totalHits);
         }
-        operation = randomAppendOnly(1, doc, false);
-        retry = randomAppendOnly(1, doc, true);
+        operation = randomAppendOnly(doc, false, 1);
+        retry = randomAppendOnly(doc, true, 1);
         if (randomBoolean()) {
             Engine.IndexResult indexResult = engine.index(operation);
             assertNotNull(indexResult.getTranslogLocation());
@@ -2629,20 +2708,20 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testRetryWithAutogeneratedIdWorksAndNoDuplicateDocs() throws IOException {
 
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
         boolean isRetry = false;
         long autoGeneratedIdTimestamp = 0;
 
-        Engine.Index index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         Engine.IndexResult indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
 
         isRetry = true;
-        index = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         indexResult = engine.index(index);
         assertThat(indexResult.getVersion(), equalTo(1L));
         engine.refresh("test");
@@ -2651,7 +2730,7 @@ public class InternalEngineTests extends ESTestCase {
             assertEquals(1, topDocs.totalHits);
         }
 
-        index = new Engine.Index(newUid("1"), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        index = new Engine.Index(newUid(doc), doc, indexResult.getSeqNo(), index.primaryTerm(), indexResult.getVersion(), index.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         indexResult = replicaEngine.index(index);
         assertThat(indexResult.hasFailure(), equalTo(false));
         replicaEngine.refresh("test");
@@ -2663,20 +2742,20 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testRetryWithAutogeneratedIdsAndWrongOrderWorksAndNoDuplicateDocs() throws IOException {
 
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
         boolean isRetry = true;
         long autoGeneratedIdTimestamp = 0;
 
-        Engine.Index firstIndexRequest = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         Engine.IndexResult result = engine.index(firstIndexRequest);
         assertThat(result.getVersion(), equalTo(1L));
 
-        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid("1"), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index firstIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), firstIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         Engine.IndexResult indexReplicaResult = replicaEngine.index(firstIndexRequestReplica);
         assertThat(indexReplicaResult.getVersion(), equalTo(1L));
 
         isRetry = false;
-        Engine.Index secondIndexRequest = new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index secondIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         Engine.IndexResult indexResult = engine.index(secondIndexRequest);
         assertTrue(indexResult.isCreated());
         engine.refresh("test");
@@ -2685,7 +2764,7 @@ public class InternalEngineTests extends ESTestCase {
             assertEquals(1, topDocs.totalHits);
         }
 
-        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid("1"), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
+        Engine.Index secondIndexRequestReplica = new Engine.Index(newUid(doc), doc, result.getSeqNo(), secondIndexRequest.primaryTerm(), result.getVersion(), firstIndexRequest.versionType().versionTypeForReplicationAndRecovery(), REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, isRetry);
         replicaEngine.index(secondIndexRequestReplica);
         replicaEngine.refresh("test");
         try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) {
@@ -2694,11 +2773,13 @@ public class InternalEngineTests extends ESTestCase {
         }
     }
 
-    public Engine.Index randomAppendOnly(int docId, ParsedDocument doc, boolean retry) {
+    public Engine.Index randomAppendOnly(ParsedDocument doc, boolean retry, final long autoGeneratedIdTimestamp) {
         if (randomBoolean()) {
-            return new Engine.Index(newUid(Integer.toString(docId)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), docId, retry);
+            return new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY,
+                VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), autoGeneratedIdTimestamp, retry);
         }
-        return new Engine.Index(newUid(Integer.toString(docId)), doc, 0, 0, 1, VersionType.EXTERNAL, Engine.Operation.Origin.REPLICA, System.nanoTime(), docId, retry);
+        return new Engine.Index(newUid(doc), doc, 0, 0, 1, VersionType.EXTERNAL,
+            Engine.Operation.Origin.REPLICA, System.nanoTime(), autoGeneratedIdTimestamp, retry);
     }
 
     public void testRetryConcurrently() throws InterruptedException, IOException {
@@ -2706,9 +2787,9 @@ public class InternalEngineTests extends ESTestCase {
         int numDocs = randomIntBetween(1000, 10000);
         List<Engine.Index> docs = new ArrayList<>();
         for (int i = 0; i < numDocs; i++) {
-            final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-            Engine.Index originalIndex = randomAppendOnly(i, doc, false);
-            Engine.Index retryIndex = randomAppendOnly(i, doc, true);
+            final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+            Engine.Index originalIndex = randomAppendOnly(doc, false, i);
+            Engine.Index retryIndex = randomAppendOnly(doc, true, i);
             docs.add(originalIndex);
             docs.add(retryIndex);
         }
@@ -2716,21 +2797,22 @@ public class InternalEngineTests extends ESTestCase {
         CountDownLatch startGun = new CountDownLatch(thread.length);
         AtomicInteger offset = new AtomicInteger(-1);
         for (int i = 0; i < thread.length; i++) {
-            thread[i] = new Thread() {
-                @Override
-                public void run() {
-                    startGun.countDown();
+            thread[i] = new Thread(() -> {
+                startGun.countDown();
+                try {
+                    startGun.await();
+                } catch (InterruptedException e) {
+                    throw new AssertionError(e);
+                }
+                int docOffset;
+                while ((docOffset = offset.incrementAndGet()) < docs.size()) {
                     try {
-                        startGun.await();
-                    } catch (InterruptedException e) {
+                        engine.index(docs.get(docOffset));
+                    } catch (IOException e) {
                         throw new AssertionError(e);
                     }
-                    int docOffset;
-                    while ((docOffset = offset.incrementAndGet()) < docs.size()) {
-                        engine.index(docs.get(docOffset));
-                    }
                 }
-            };
+            });
             thread[i].start();
         }
         for (int i = 0; i < thread.length; i++) {
@@ -2769,8 +2851,8 @@ public class InternalEngineTests extends ESTestCase {
         assertEquals(0, engine.getNumIndexVersionsLookups());
         List<Engine.Index> docs = new ArrayList<>();
         for (int i = 0; i < numDocs; i++) {
-            final ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
-            Engine.Index index = randomAppendOnly(i, doc, false);
+            final ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+            Engine.Index index = randomAppendOnly(doc, false, i);
             docs.add(index);
         }
         Collections.shuffle(docs, random());
@@ -2788,7 +2870,11 @@ public class InternalEngineTests extends ESTestCase {
                     }
                     int docOffset;
                     while ((docOffset = offset.incrementAndGet()) < docs.size()) {
-                        engine.index(docs.get(docOffset));
+                        try {
+                            engine.index(docs.get(docOffset));
+                        } catch (IOException e) {
+                            throw new AssertionError(e);
+                        }
                     }
                 }
             };
@@ -2841,16 +2927,16 @@ public class InternalEngineTests extends ESTestCase {
                             } catch (InterruptedException e) {
                                 throw new AssertionError(e);
                             }
-                            throw new AlreadyClosedException("boom");
+                            throw new ElasticsearchException("something completely different");
                         }
                     }
                 });
             InternalEngine internalEngine = new InternalEngine(config);
             int docId = 0;
-            final ParsedDocument doc = testParsedDocument(Integer.toString(docId), Integer.toString(docId), "test", null,
+            final ParsedDocument doc = testParsedDocument(Integer.toString(docId), "test", null,
                 testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
 
-            Engine.Index index = randomAppendOnly(docId, doc, false);
+            Engine.Index index = randomBoolean() ? indexForDoc(doc) : randomAppendOnly(doc, false, docId);
             internalEngine.index(index);
             Runnable r = () ->  {
                 try {
@@ -2861,11 +2947,11 @@ public class InternalEngineTests extends ESTestCase {
                 try {
                     internalEngine.refresh("test");
                     fail();
-                } catch (EngineClosedException ex) {
-                    // we can't guarantee that we are entering the refresh call before it's fully
-                    // closed so we also expecting ECE here
-                    assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException);
-                } catch (RefreshFailedEngineException | AlreadyClosedException  ex) {
+                } catch (AlreadyClosedException ex) {
+                    if (ex.getCause() != null) {
+                        assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException);
+                    }
+                } catch (RefreshFailedEngineException ex) {
                     // fine
                 } finally {
                     start.countDown();
@@ -2892,11 +2978,11 @@ public class InternalEngineTests extends ESTestCase {
         // create a document
         Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(indexForDoc(doc));
         engine.refresh("test");
 
-        seqID = getSequenceID(engine, new Engine.Get(false, newUid("1")));
+        seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc)));
         logger.info("--> got seqID: {}", seqID);
         assertThat(seqID.v1(), equalTo(0L));
         assertThat(seqID.v2(), equalTo(0L));
@@ -2904,11 +2990,11 @@ public class InternalEngineTests extends ESTestCase {
         // Index the same document again
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(indexForDoc(doc));
         engine.refresh("test");
 
-        seqID = getSequenceID(engine, new Engine.Get(false, newUid("1")));
+        seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc)));
         logger.info("--> got seqID: {}", seqID);
         assertThat(seqID.v1(), equalTo(1L));
         assertThat(seqID.v2(), equalTo(0L));
@@ -2916,13 +3002,13 @@ public class InternalEngineTests extends ESTestCase {
         // Index the same document for the third time, this time changing the primary term
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        engine.index(new Engine.Index(newUid("1"), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 1,
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        engine.index(new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 1,
                         Versions.MATCH_ANY, VersionType.INTERNAL, Engine.Operation.Origin.PRIMARY,
                         System.nanoTime(), -1, false));
         engine.refresh("test");
 
-        seqID = getSequenceID(engine, new Engine.Get(false, newUid("1")));
+        seqID = getSequenceID(engine, new Engine.Get(false, newUid(doc)));
         logger.info("--> got seqID: {}", seqID);
         assertThat(seqID.v1(), equalTo(2L));
         assertThat(seqID.v2(), equalTo(1L));
@@ -2973,11 +3059,16 @@ public class InternalEngineTests extends ESTestCase {
             final InternalEngine finalInitialEngine = initialEngine;
             for (int i = 0; i < docs; i++) {
                 final String id = Integer.toString(i);
-                final Term uid = newUid(id);
-                final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
 
                 skip.set(randomBoolean());
-                final Thread thread = new Thread(() -> finalInitialEngine.index(new Engine.Index(uid, doc)));
+                final Thread thread = new Thread(() -> {
+                    try {
+                        finalInitialEngine.index(indexForDoc(doc));
+                    } catch (IOException e) {
+                        throw new AssertionError(e);
+                    }
+                });
                 thread.start();
                 if (skip.get()) {
                     threads.add(thread);
@@ -2999,7 +3090,7 @@ public class InternalEngineTests extends ESTestCase {
             IOUtils.close(initialEngine);
         }
 
-        try (final Engine recoveringEngine =
+        try (Engine recoveringEngine =
                  new InternalEngine(copy(initialEngine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG))) {
             assertThat(recoveringEngine.seqNoService().getLocalCheckpoint(), greaterThanOrEqualTo((long) (docs - 1)));
         }
@@ -3015,8 +3106,8 @@ public class InternalEngineTests extends ESTestCase {
             initialEngine = engine;
             for (int i = 0; i < docs; i++) {
                 final String id = Integer.toString(i);
-                final Term uid = newUid(id);
-                final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                final Term uid = newUid(doc);
                 // create a gap at sequence number 3 * i + 1
                 initialEngine.index(new Engine.Index(uid, doc, 3 * i, 1, v, t, REPLICA, System.nanoTime(), ts, false));
                 initialEngine.delete(new Engine.Delete("type", id, uid, 3 * i + 2, 1, v, t, REPLICA, System.nanoTime()));
@@ -3029,15 +3120,15 @@ public class InternalEngineTests extends ESTestCase {
 
             for (int i = 0; i < docs; i++) {
                 final String id = Integer.toString(i);
-                final Term uid = newUid(id);
-                final ParsedDocument doc = testParsedDocument(id, id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                final ParsedDocument doc = testParsedDocument(id, "test", null, testDocumentWithTextField(), SOURCE, null);
+                final Term uid = newUid(doc);
                 initialEngine.index(new Engine.Index(uid, doc, 3 * i + 1, 1, v, t, REPLICA, System.nanoTime(), ts, false));
             }
         } finally {
             IOUtils.close(initialEngine);
         }
 
-        try (final Engine recoveringEngine =
+        try (Engine recoveringEngine =
                  new InternalEngine(copy(initialEngine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG))) {
             assertThat(recoveringEngine.seqNoService().getLocalCheckpoint(), greaterThanOrEqualTo((long) (3 * (docs - 1) + 2 - 1)));
         }
@@ -3047,14 +3138,14 @@ public class InternalEngineTests extends ESTestCase {
         final List<Engine.Operation> operations = new ArrayList<>();
 
         final int numberOfOperations = randomIntBetween(16, 32);
-        final Term uid = newUid("1");
         final Document document = testDocumentWithTextField();
         final AtomicLong sequenceNumber = new AtomicLong();
         final Engine.Operation.Origin origin = randomFrom(LOCAL_TRANSLOG_RECOVERY, PEER_RECOVERY, PRIMARY, REPLICA);
         final LongSupplier sequenceNumberSupplier =
             origin == PRIMARY ? () -> SequenceNumbersService.UNASSIGNED_SEQ_NO : sequenceNumber::getAndIncrement;
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        final ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
+        final ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
+        final Term uid = newUid(doc);
         for (int i = 0; i < numberOfOperations; i++) {
             if (randomBoolean()) {
                 final Engine.Index index = new Engine.Index(
@@ -3116,11 +3207,55 @@ public class InternalEngineTests extends ESTestCase {
         }
 
         assertThat(engine.seqNoService().getLocalCheckpoint(), equalTo(expectedLocalCheckpoint));
-        try (final Engine.GetResult result = engine.get(new Engine.Get(true, uid))) {
+        try (Engine.GetResult result = engine.get(new Engine.Get(true, uid))) {
             assertThat(result.exists(), equalTo(exists));
         }
     }
 
+    /*
+     * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops
+     * are correctly added to the translog.
+     */
+    public void testNoOps() throws IOException {
+        engine.close();
+        InternalEngine noOpEngine = null;
+        final int maxSeqNo = randomIntBetween(0, 128);
+        final int localCheckpoint = randomIntBetween(0, maxSeqNo);
+        final int globalCheckpoint = randomIntBetween(0, localCheckpoint);
+        try {
+            final SequenceNumbersService seqNoService =
+                new SequenceNumbersService(shardId, defaultSettings, maxSeqNo, localCheckpoint, globalCheckpoint) {
+                    @Override
+                    public long generateSeqNo() {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            noOpEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, () -> seqNoService);
+            final long primaryTerm = randomNonNegativeLong();
+            final String reason = randomAsciiOfLength(16);
+            noOpEngine.noOp(
+                new Engine.NoOp(
+                    null,
+                    maxSeqNo + 1,
+                    primaryTerm,
+                    0,
+                    VersionType.INTERNAL,
+                    randomFrom(PRIMARY, REPLICA, PEER_RECOVERY, LOCAL_TRANSLOG_RECOVERY),
+                    System.nanoTime(),
+                    reason));
+            assertThat(noOpEngine.seqNoService().getLocalCheckpoint(), equalTo((long) (maxSeqNo + 1)));
+            assertThat(noOpEngine.getTranslog().totalOperations(), equalTo(1));
+            final Translog.Operation op = noOpEngine.getTranslog().newSnapshot().next();
+            assertThat(op, instanceOf(Translog.NoOp.class));
+            final Translog.NoOp noOp = (Translog.NoOp) op;
+            assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 1)));
+            assertThat(noOp.primaryTerm(), equalTo(primaryTerm));
+            assertThat(noOp.reason(), equalTo(reason));
+        } finally {
+            IOUtils.close(noOpEngine);
+        }
+    }
+
     /**
      * Return a tuple representing the sequence ID for the given {@code Get}
      * operation. The first value in the tuple is the sequence number, the
diff --git a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
index ed80b98c7f9..9161bc413c8 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/LiveVersionMapTests.java
@@ -22,11 +22,13 @@ package org.elasticsearch.index.engine;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.RamUsageTester;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.test.ESTestCase;
 
 public class LiveVersionMapTests extends ESTestCase {
 
     public void testRamBytesUsed() throws Exception {
+        assumeTrue("Test disabled for JDK 9", JavaVersion.current().compareTo(JavaVersion.parse("9")) < 0);
         LiveVersionMap map = new LiveVersionMap();
         for (int i = 0; i < 100000; ++i) {
             BytesRefBuilder uid = new BytesRefBuilder();
@@ -36,8 +38,8 @@ public class LiveVersionMapTests extends ESTestCase {
         }
         long actualRamBytesUsed = RamUsageTester.sizeOf(map);
         long estimatedRamBytesUsed = map.ramBytesUsed();
-        // less than 25% off
-        assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 4);
+        // less than 50% off
+        assertEquals(actualRamBytesUsed, estimatedRamBytesUsed, actualRamBytesUsed / 2);
 
         // now refresh
         map.beforeRefresh();
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index a3c58f25ea9..53708b28dfb 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -33,6 +33,7 @@ import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
@@ -44,21 +45,19 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.seqno.SequenceNumbersService;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.RefreshListeners;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -83,7 +82,6 @@ import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
@@ -172,8 +170,8 @@ public class ShadowEngineTests extends ESTestCase {
     }
 
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    private ParsedDocument testParsedDocument(String id, String type, String routing, ParseContext.Document document, BytesReference source, Mapping mappingsUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
         document.add(uidField);
@@ -182,7 +180,8 @@ public class ShadowEngineTests extends ESTestCase {
         document.add(seqID.seqNoDocValue);
         document.add(seqID.primaryTerm);
         document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage
-        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingsUpdate);
+        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON,
+            mappingsUpdate);
     }
 
     protected Store createStore(Path p) throws IOException {
@@ -254,18 +253,26 @@ public class ShadowEngineTests extends ESTestCase {
         return config;
     }
 
-    protected Term newUid(String id) {
-        return new Term("_uid", id);
+//    protected Term newUid(String id) {
+//        return new Term("_uid", id);
+//    }
+
+    protected Term newUid(ParsedDocument doc) {
+        return new Term("_uid", doc.uid());
+    }
+
+    private Engine.Index indexForDoc(ParsedDocument doc) {
+        return new Engine.Index(newUid(doc), doc);
     }
 
     protected static final BytesReference B_1 = new BytesArray(new byte[]{1});
     protected static final BytesReference B_2 = new BytesArray(new byte[]{2});
     protected static final BytesReference B_3 = new BytesArray(new byte[]{3});
 
-    public void testCommitStats() {
+    public void testCommitStats() throws IOException {
         // create a doc and refresh
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         CommitStats stats1 = replicaEngine.commitStats();
         assertThat(stats1.getGeneration(), greaterThan(0L));
@@ -296,11 +303,11 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(primaryEngine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
 
         // create a doc and refresh
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
-        ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("2"), doc2));
+        ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null);
+        primaryEngine.index(indexForDoc(doc2));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -358,8 +365,8 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(segments.get(0).isCompound(), equalTo(true));
 
 
-        ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("3"), doc3));
+        ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc3));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -408,7 +415,7 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
         assertThat(segments.get(1).isCompound(), equalTo(true));
 
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -430,8 +437,8 @@ public class ShadowEngineTests extends ESTestCase {
         primaryEngine.flush();
         replicaEngine.refresh("test");
 
-        ParsedDocument doc4 = testParsedDocument("4", "4", "test", null, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("4"), doc4));
+        ParsedDocument doc4 = testParsedDocument("4", "test", null, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc4));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(false);
@@ -463,19 +470,19 @@ public class ShadowEngineTests extends ESTestCase {
         List<Segment> segments = primaryEngine.segments(true);
         assertThat(segments.isEmpty(), equalTo(true));
 
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(true);
         assertThat(segments.size(), equalTo(1));
         assertThat(segments.get(0).ramTree, notNullValue());
 
-        ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, testDocumentWithTextField(), B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("2"), doc2));
+        ParsedDocument doc2 = testParsedDocument("2", "test", null, testDocumentWithTextField(), B_2, null);
+        primaryEngine.index(indexForDoc(doc2));
         primaryEngine.refresh("test");
-        ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, testDocumentWithTextField(), B_3, null);
-        primaryEngine.index(new Engine.Index(newUid("3"), doc3));
+        ParsedDocument doc3 = testParsedDocument("3", "test", null, testDocumentWithTextField(), B_3, null);
+        primaryEngine.index(indexForDoc(doc3));
         primaryEngine.refresh("test");
 
         segments = primaryEngine.segments(true);
@@ -500,9 +507,9 @@ public class ShadowEngineTests extends ESTestCase {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
+        ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
         try {
-            replicaEngine.index(new Engine.Index(newUid("1"), doc));
+            replicaEngine.index(indexForDoc(doc));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.refresh("test");
@@ -512,16 +519,16 @@ public class ShadowEngineTests extends ESTestCase {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
         searchResult.close();
-        Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        Engine.GetResult getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // index a document
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
         try {
-            replicaEngine.index(new Engine.Index(newUid("1"), doc));
+            replicaEngine.index(indexForDoc(doc));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.refresh("test");
@@ -531,15 +538,15 @@ public class ShadowEngineTests extends ESTestCase {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 0));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // Now, add a document to the primary so we can test shadow engine deletes
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
         replicaEngine.refresh("test");
 
@@ -550,14 +557,14 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // And the replica can retrieve it
-        getResult = replicaEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
 
         // try to delete it on the replica
         try {
-            replicaEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+            replicaEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
             fail("should have thrown an exception");
         } catch (UnsupportedOperationException e) {}
         replicaEngine.flush();
@@ -569,7 +576,7 @@ public class ShadowEngineTests extends ESTestCase {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -579,7 +586,7 @@ public class ShadowEngineTests extends ESTestCase {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         searchResult.close();
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -593,8 +600,8 @@ public class ShadowEngineTests extends ESTestCase {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -609,18 +616,18 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
 
         // can't get it from the replica, because it's not in the translog for a shadow replica
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
         // but, not there non realtime
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         getResult.release();
 
@@ -631,7 +638,7 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // also in non realtime
-        getResult = primaryEngine.get(new Engine.Get(false, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(false, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -646,8 +653,8 @@ public class ShadowEngineTests extends ESTestCase {
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_2, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_2, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -657,7 +664,7 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, we can still get it (in realtime)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -690,7 +697,7 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // now delete
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete("test", "1", newUid(doc)));
 
         // its not deleted yet
         searchResult = primaryEngine.acquireSearcher("test");
@@ -700,7 +707,7 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // but, get should not see it (in realtime)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(false));
         getResult.release();
 
@@ -716,8 +723,8 @@ public class ShadowEngineTests extends ESTestCase {
         // add it back
         document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -740,7 +747,7 @@ public class ShadowEngineTests extends ESTestCase {
         primaryEngine.flush();
 
         // and, verify get (in real time)
-        getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = primaryEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -752,7 +759,7 @@ public class ShadowEngineTests extends ESTestCase {
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
         MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test1")), 0));
         searchResult.close();
-        getResult = replicaEngine.get(new Engine.Get(true, newUid("1")));
+        getResult = replicaEngine.get(new Engine.Get(true, newUid(doc)));
         assertThat(getResult.exists(), equalTo(true));
         assertThat(getResult.docIdAndVersion(), notNullValue());
         getResult.release();
@@ -761,8 +768,8 @@ public class ShadowEngineTests extends ESTestCase {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        doc = testParsedDocument("1", "test", null, document, B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not updated yet...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -797,8 +804,8 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
 
         // create a document
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
 
         // its not there...
         searchResult = primaryEngine.acquireSearcher("test");
@@ -827,7 +834,7 @@ public class ShadowEngineTests extends ESTestCase {
         // don't release the replica search result yet...
 
         // delete, refresh and do a new search, it should not be there
-        primaryEngine.delete(new Engine.Delete("test", "1", newUid("1")));
+        primaryEngine.delete(new Engine.Delete(doc.type(), doc.id(), newUid(doc)));
         primaryEngine.flush();
         primaryEngine.refresh("test");
         replicaEngine.refresh("test");
@@ -841,9 +848,9 @@ public class ShadowEngineTests extends ESTestCase {
         searchResult.close();
     }
 
-    public void testFailEngineOnCorruption() {
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+    public void testFailEngineOnCorruption() throws IOException {
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
         MockDirectoryWrapper leaf = DirectoryUtils.getLeaf(replicaEngine.config().getStore().directory(), MockDirectoryWrapper.class);
         leaf.setRandomIOExceptionRate(1.0);
@@ -860,7 +867,7 @@ public class ShadowEngineTests extends ESTestCase {
             MatcherAssert.assertThat(searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(new TermQuery(new Term("value", "test")), 1));
             searchResult.close();
             fail("exception expected");
-        } catch (EngineClosedException ex) {
+        } catch (AlreadyClosedException ex) {
             // all is well
         }
     }
@@ -879,8 +886,8 @@ public class ShadowEngineTests extends ESTestCase {
      */
     public void testFailStart() throws IOException {
         // Need a commit point for this
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, testDocumentWithTextField(), B_1, null);
-        primaryEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
+        primaryEngine.index(indexForDoc(doc));
         primaryEngine.flush();
 
         // this test fails if any reader, searcher or directory is not closed - MDW FTW
@@ -965,8 +972,8 @@ public class ShadowEngineTests extends ESTestCase {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
         document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, document, B_1, null);
-        pEngine.index(new Engine.Index(newUid("1"), doc));
+        ParsedDocument doc = testParsedDocument("1", "test", null, document, B_1, null);
+        pEngine.index(indexForDoc(doc));
         pEngine.flush(true, true);
 
         t.join();
@@ -984,33 +991,6 @@ public class ShadowEngineTests extends ESTestCase {
         }
     }
 
-    public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
-        for (int i = 0; i < numDocs; i++) {
-            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
-            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
-            Engine.IndexResult indexResult = primaryEngine.index(firstIndexRequest);
-            assertThat(indexResult.getVersion(), equalTo(1L));
-        }
-        DocsStats docStats = primaryEngine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-
-        docStats = replicaEngine.getDocStats();
-        assertEquals(0, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        primaryEngine.flush();
-
-        docStats = replicaEngine.getDocStats();
-        assertEquals(0, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        replicaEngine.refresh("test");
-        docStats = replicaEngine.getDocStats();
-        assertEquals(numDocs, docStats.getCount());
-        assertEquals(0, docStats.getDeleted());
-        primaryEngine.forceMerge(randomBoolean(), 1, false, false, false);
-    }
-
     public void testRefreshListenersFails() throws IOException {
         EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(),
                 new RefreshListeners(null, null, null, logger));
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java
index 54881bad884..90995bec34d 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java
@@ -30,7 +30,7 @@ public class FieldDataStatsTests extends ESTestCase {
 
     public void testSerialize() throws IOException {
         FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
-        FieldDataStats stats = new FieldDataStats(randomPositiveLong(), randomPositiveLong(), map == null ? null :
+        FieldDataStats stats = new FieldDataStats(randomNonNegativeLong(), randomNonNegativeLong(), map == null ? null :
             map);
         BytesStreamOutput out = new BytesStreamOutput();
         stats.writeTo(out);
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
new file mode 100644
index 00000000000..f8579efae73
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesDatesTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues.Dates;
+import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
+
+public class ScriptDocValuesDatesTests extends ESTestCase {
+    public void test() {
+        long[][] values = new long[between(3, 10)][];
+        ReadableDateTime[][] expectedDates = new ReadableDateTime[values.length][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            expectedDates[d] = new ReadableDateTime[values[d].length];
+            for (int i = 0; i < values[d].length; i++) {
+                expectedDates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC);
+                values[d][i] = expectedDates[d][i].getMillis();
+            }
+        }
+        Dates dates = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            dates.setNextDocId(d);
+            assertEquals(expectedDates[d].length > 0 ? expectedDates[d][0] : new DateTime(0, DateTimeZone.UTC), dates.getValue());
+
+            assertEquals(values[d].length, dates.size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(expectedDates[d][i], dates.get(i));
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> dates.add(new DateTime()));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    private Dates wrap(long[][] values) {
+        return new Dates(new SortedNumericDocValues() {
+            long[] current;
+
+            @Override
+            public void setDocument(int doc) {
+                current = values[doc];
+            }
+            @Override
+            public int count() {
+                return current.length;
+            }
+            @Override
+            public long valueAt(int index) {
+                return current[index];
+            }
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java
rename to core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java
index 49f92e63c95..466ca0b6991 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java
@@ -25,7 +25,7 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
 
-public class ScriptDocValuesTests extends ESTestCase {
+public class ScriptDocValuesGeoPointsTests extends ESTestCase {
 
     private static MultiGeoPointValues wrap(final GeoPoint... points) {
         return new MultiGeoPointValues() {
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java
new file mode 100644
index 00000000000..934fa628d47
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues.Longs;
+import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
+
+public class ScriptDocValuesLongsTests extends ESTestCase {
+    public void testLongs() {
+        long[][] values = new long[between(3, 10)][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            for (int i = 0; i < values[d].length; i++) {
+                values[d][i] = randomLong();
+            }
+        }
+        Longs longs = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            longs.setNextDocId(d);
+            assertEquals(values[d].length > 0 ? values[d][0] : 0, longs.getValue());
+
+            assertEquals(values[d].length, longs.size());
+            assertEquals(values[d].length, longs.getValues().size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(values[d][i], longs.get(i).longValue());
+                assertEquals(values[d][i], longs.getValues().get(i).longValue());
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getValues().add(100L));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    public void testDates() {
+        long[][] values = new long[between(3, 10)][];
+        ReadableDateTime[][] dates = new ReadableDateTime[values.length][];
+        for (int d = 0; d < values.length; d++) {
+            values[d] = new long[randomBoolean() ? randomBoolean() ? 0 : 1 : between(2, 100)];
+            dates[d] = new ReadableDateTime[values[d].length];
+            for (int i = 0; i < values[d].length; i++) {
+                dates[d][i] = new DateTime(randomNonNegativeLong(), DateTimeZone.UTC);
+                values[d][i] = dates[d][i].getMillis();
+            }
+        }
+        Longs longs = wrap(values);
+
+        for (int round = 0; round < 10; round++) {
+            int d = between(0, values.length - 1);
+            longs.setNextDocId(d);
+            assertEquals(dates[d].length > 0 ? dates[d][0] : new DateTime(0, DateTimeZone.UTC), longs.getDate());
+
+            assertEquals(values[d].length, longs.getDates().size());
+            for (int i = 0; i < values[d].length; i++) {
+                assertEquals(dates[d][i], longs.getDates().get(i));
+            }
+
+            Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getDates().add(new DateTime()));
+            assertEquals("doc values are unmodifiable", e.getMessage());
+        }
+    }
+
+    private Longs wrap(long[][] values) {
+        return new Longs(new SortedNumericDocValues() {
+            long[] current;
+
+            @Override
+            public void setDocument(int doc) {
+                current = values[doc];
+            }
+            @Override
+            public int count() {
+                return current.length;
+            }
+            @Override
+            public long valueAt(int index) {
+                return current[index];
+            }
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java b/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java
new file mode 100644
index 00000000000..3fd6ca87e9b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/get/GetFieldTests.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.get;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.index.mapper.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetFieldTests extends ESTestCase {
+
+    public void testToXContent() {
+        GetField getField = new GetField("field", Arrays.asList("value1", "value2"));
+        String output = Strings.toString(getField);
+        assertEquals("{\"field\":[\"value1\",\"value2\"]}", output);
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(randomGetField(XContentType.JSON).v1(), GetFieldTests::copyGetField, GetFieldTests::mutateGetField);
+    }
+
+    public void testToAndFromXContent() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetField, GetField> tuple = randomGetField(xContentType);
+        GetField getField = tuple.v1();
+        GetField expectedGetField = tuple.v2();
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(getField, xContentType, humanReadable);
+        //test that we can parse what we print out
+        GetField parsedGetField;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            //we need to move to the next token, the start object one that we manually added is not expected
+            assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+            assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
+            parsedGetField = GetField.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_ARRAY, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetField, parsedGetField);
+        BytesReference finalBytes = toXContent(parsedGetField, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+    }
+
+    private static GetField copyGetField(GetField getField) {
+        return new GetField(getField.getName(), getField.getValues());
+    }
+
+    private static GetField mutateGetField(GetField getField) {
+        List<Supplier<GetField>> mutations = new ArrayList<>();
+        mutations.add(() -> new GetField(randomUnicodeOfCodepointLength(15), getField.getValues()));
+        mutations.add(() -> new GetField(getField.getName(), randomGetField(XContentType.JSON).v1().getValues()));
+        return randomFrom(mutations).get();
+    }
+
+    public static Tuple<GetField, GetField> randomGetField(XContentType xContentType) {
+        if (randomBoolean()) {
+            String fieldName = randomFrom(ParentFieldMapper.NAME, RoutingFieldMapper.NAME, UidFieldMapper.NAME);
+            GetField getField = new GetField(fieldName, Collections.singletonList(randomAsciiOfLengthBetween(3, 10)));
+            return Tuple.tuple(getField, getField);
+        }
+        String fieldName = randomAsciiOfLengthBetween(3, 10);
+        Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xContentType);
+        GetField input = new GetField(fieldName, tuple.v1());
+        GetField expected = new GetField(fieldName, tuple.v2());
+        return Tuple.tuple(input, expected);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java
new file mode 100644
index 00000000000..4e5b94d9c98
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.get;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.index.get.GetFieldTests.randomGetField;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class GetResultTests extends ESTestCase {
+
+    public void testToAndFromXContent() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+        GetResult expectedGetResult = tuple.v2();
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(getResult, xContentType, humanReadable);
+        //test that we can parse what we print out
+        GetResult parsedGetResult;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parsedGetResult = GetResult.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertEquals(expectedGetResult, parsedGetResult);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContent(parsedGetResult, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResult.sourceAsString(), parsedGetResult.sourceAsString());
+    }
+
+    public void testToXContent() throws IOException {
+        {
+            GetResult getResult = new GetResult("index", "type", "id", 1, true, new BytesArray("{ \"field1\" : " +
+                    "\"value1\", \"field2\":\"value2\"}"), singletonMap("field1", new GetField("field1",
+                    singletonList("value1"))));
+            String output = Strings.toString(getResult);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"_version\":1,\"found\":true,\"_source\":{ \"field1\" " +
+                    ": \"value1\", \"field2\":\"value2\"},\"fields\":{\"field1\":[\"value1\"]}}", output);
+        }
+        {
+            GetResult getResult = new GetResult("index", "type", "id", 1, false, null, null);
+            String output = Strings.toString(getResult);
+            assertEquals("{\"_index\":\"index\",\"_type\":\"type\",\"_id\":\"id\",\"found\":false}", output);
+        }
+    }
+
+    public void testToAndFromXContentEmbedded() throws Exception {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+
+        // We don't expect to retrieve the index/type/id of the GetResult because they are not rendered
+        // by the toXContentEmbedded method.
+        GetResult expectedGetResult = new GetResult(null, null, null, -1,
+                tuple.v2().isExists(), tuple.v2().sourceRef(), tuple.v2().getFields());
+
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContentEmbedded(getResult, xContentType, humanReadable);
+
+        // Test that we can parse the result of toXContentEmbedded()
+        GetResult parsedEmbeddedGetResult;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsedEmbeddedGetResult = GetResult.fromXContentEmbedded(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(expectedGetResult, parsedEmbeddedGetResult);
+        //print the parsed object out and test that the output is the same as the original output
+        BytesReference finalBytes = toXContentEmbedded(parsedEmbeddedGetResult, xContentType, humanReadable);
+        assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
+        //check that the source stays unchanged, no shuffling of keys nor anything like that
+        assertEquals(expectedGetResult.sourceAsString(), parsedEmbeddedGetResult.sourceAsString());
+    }
+
+    public void testToXContentEmbedded() throws IOException {
+        Map<String, GetField> fields = new HashMap<>();
+        fields.put("foo", new GetField("foo", singletonList("bar")));
+        fields.put("baz", new GetField("baz", Arrays.asList("baz_0", "baz_1")));
+
+        GetResult getResult = new GetResult("index", "type", "id", 2, true,
+                new BytesArray("{\"foo\":\"bar\",\"baz\":[\"baz_0\",\"baz_1\"]}"), fields);
+
+        BytesReference originalBytes = toXContentEmbedded(getResult, XContentType.JSON, false);
+        assertEquals("{\"found\":true,\"_source\":{\"foo\":\"bar\",\"baz\":[\"baz_0\",\"baz_1\"]}," +
+                "\"fields\":{\"foo\":[\"bar\"],\"baz\":[\"baz_0\",\"baz_1\"]}}", originalBytes.utf8ToString());
+    }
+
+    public void testToXContentEmbeddedNotFound() throws IOException {
+        GetResult getResult = new GetResult("index", "type", "id", 1, false, null, null);
+
+        BytesReference originalBytes = toXContentEmbedded(getResult, XContentType.JSON, false);
+        assertEquals("{\"found\":false}", originalBytes.utf8ToString());
+    }
+
+    public void testGetSourceAsBytes() {
+        XContentType xContentType = randomFrom(XContentType.values());
+        Tuple<GetResult, GetResult> tuple = randomGetResult(xContentType);
+        GetResult getResult = tuple.v1();
+        if (getResult.isExists() && getResult.isSourceEmpty() == false) {
+            assertNotNull(getResult.sourceRef());
+        } else {
+            assertNull(getResult.sourceRef());
+        }
+    }
+
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(randomGetResult(XContentType.JSON).v1(), GetResultTests::copyGetResult, GetResultTests::mutateGetResult);
+    }
+
+    public static GetResult copyGetResult(GetResult getResult) {
+        return new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields());
+    }
+
+    public static GetResult mutateGetResult(GetResult getResult) {
+        List<Supplier<GetResult>> mutations = new ArrayList<>();
+        mutations.add(() -> new GetResult(randomUnicodeOfLength(15), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), randomUnicodeOfLength(15), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), randomUnicodeOfLength(15), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), randomNonNegativeLong(),
+                getResult.isExists(), getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists() == false, getResult.internalSourceRef(), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), RandomObjects.randomSource(random()), getResult.getFields()));
+        mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(),
+                getResult.isExists(), getResult.internalSourceRef(), randomGetFields(XContentType.JSON).v1()));
+        return randomFrom(mutations).get();
+    }
+
+    public static Tuple<GetResult, GetResult> randomGetResult(XContentType xContentType) {
+        final String index = randomAsciiOfLengthBetween(3, 10);
+        final String type = randomAsciiOfLengthBetween(3, 10);
+        final String id = randomAsciiOfLengthBetween(3, 10);
+        final long version;
+        final boolean exists;
+        BytesReference source = null;
+        Map<String, GetField> fields = null;
+        Map<String, GetField> expectedFields = null;
+        if (frequently()) {
+            version = randomNonNegativeLong();
+            exists = true;
+            if (frequently()) {
+                source = RandomObjects.randomSource(random());
+            }
+            if (randomBoolean()) {
+                Tuple<Map<String, GetField>, Map<String, GetField>> tuple = randomGetFields(xContentType);
+                fields = tuple.v1();
+                expectedFields = tuple.v2();
+            }
+        } else {
+            version = -1;
+            exists = false;
+        }
+        GetResult getResult = new GetResult(index, type, id, version, exists, source, fields);
+        GetResult expectedGetResult = new GetResult(index, type, id, version, exists, source, expectedFields);
+        return Tuple.tuple(getResult, expectedGetResult);
+    }
+
+    private static Tuple<Map<String, GetField>,Map<String, GetField>> randomGetFields(XContentType xContentType) {
+        int numFields = randomIntBetween(2, 10);
+        Map<String, GetField> fields = new HashMap<>(numFields);
+        Map<String, GetField> expectedFields = new HashMap<>(numFields);
+        for (int i = 0; i < numFields; i++) {
+            Tuple<GetField, GetField> tuple = randomGetField(xContentType);
+            GetField getField = tuple.v1();
+            GetField expectedGetField = tuple.v2();
+            fields.put(getField.getName(), getField);
+            expectedFields.put(expectedGetField.getName(), expectedGetField);
+        }
+        return Tuple.tuple(fields, expectedFields);
+    }
+
+    private static BytesReference toXContentEmbedded(GetResult getResult, XContentType xContentType, boolean humanReadable)
+            throws IOException {
+        return XContentHelper.toXContent(getResult::toXContentEmbedded, xContentType, humanReadable);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java
index 7ee837e71c7..2fc2d8e382c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java
@@ -97,14 +97,6 @@ public abstract class AbstractNumericFieldMapperTestCase extends ESSingleNodeTes
 
     protected abstract void doTestCoerce(String type) throws IOException;
 
-    public void testIncludeInAll() throws Exception {
-        for (String type : TYPES) {
-            doTestIncludeInAll(type);
-        }
-    }
-
-    protected abstract void doTestIncludeInAll(String type) throws Exception;
-
     public void testNullValue() throws IOException {
         for (String type : TYPES) {
             doTestNullValue(type);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperPositionIncrementGapTests.java
deleted file mode 100644
index 0a51bb83359..00000000000
--- a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperPositionIncrementGapTests.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-
-/**
- * Tests that position_increment_gap is read from the mapper and applies as
- * expected in queries.
- */
-public class AllFieldMapperPositionIncrementGapTests extends ESSingleNodeTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
-    }
-
-    /**
-     * The default position_increment_gap should be large enough that most
-     * "sensible" queries phrase slops won't match across values.
-     */
-    public void testDefault() throws IOException {
-        assertGapIsOneHundred(client(), "test", "test");
-    }
-
-    /**
-     * Asserts that the post-2.0 default is being applied.
-     */
-    public static void assertGapIsOneHundred(Client client, String indexName, String type) throws IOException {
-        testGap(client, indexName, type, 100);
-
-        // No match across gap using default slop with default positionIncrementGap
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two")).get(), 0);
-
-        // Nor with small-ish values
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(5)).get(), 0);
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(50)).get(), 0);
-
-        // But huge-ish values still match
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(500)).get(), 1);
-    }
-
-    /**
-     * Asserts that the pre-2.0 default has been applied or explicitly
-     * configured.
-     */
-    public static void assertGapIsZero(Client client, String indexName, String type) throws IOException {
-        testGap(client, indexName, type, 0);
-        /*
-         * Phrases match across different values using default slop with pre-2.0 default
-         * position_increment_gap.
-         */
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("string", "one two")).get(), 1);
-    }
-
-    private static void testGap(Client client, String indexName,
-                                String type, int positionIncrementGap) throws IOException {
-        client.prepareIndex(indexName, type, "position_gap_test")
-            .setSource("string1", "one", "string2", "two three").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
-
-        // Baseline - phrase query finds matches in the same field value
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "two three")).get(), 1);
-
-        if (positionIncrementGap > 0) {
-            // No match across gaps when slop < position gap
-            assertHitCount(client.prepareSearch(indexName)
-                .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(positionIncrementGap - 1)).get(), 0);
-        }
-
-        // Match across gaps when slop >= position gap
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(positionIncrementGap)).get(), 1);
-        assertHitCount(client.prepareSearch(indexName)
-            .setQuery(new MatchPhraseQueryBuilder("_all", "one two").slop(positionIncrementGap+1)).get(), 1);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
deleted file mode 100644
index d7def1db8cf..00000000000
--- a/core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java
+++ /dev/null
@@ -1,490 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.lucene.all.AllTermQuery;
-import org.elasticsearch.common.lucene.all.AllTokenStream;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.engine.Engine.Searcher;
-import org.elasticsearch.index.mapper.AllFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.ParseContext.Document;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.hamcrest.Matchers;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class AllFieldMapperTests extends ESSingleNodeTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
-    }
-
-    public void testSimpleAllMappers() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
-            .parse("person", new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        String[] expected = new String[] {"banon", "last location", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(true));
-            if (i == 0) {
-                // The field "name.last" is boosted so we should see AllTokenStream used:
-                assertThat(fields[i].tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                    Matchers.instanceOf(AllTokenStream.class));
-            } else {
-                assertThat(fields[i].tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                    Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
-            }
-        }
-        AllFieldMapper mapper = docMapper.allFieldMapper();
-        assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
-            Matchers.instanceOf(AllTermQuery.class));
-    }
-
-    public void testAllMappersNoBoost() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
-        IndexService index = createIndex("test");
-        DocumentMapper docMapper = index.mapperService().documentMapperParser().parse("person",
-            new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        String[] expected = new String[] {"banon", "last location", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
-        }
-        AllFieldMapper mapper = docMapper.allFieldMapper();
-        assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
-            Matchers.instanceOf(AllTermQuery.class));
-    }
-
-    public void testAllMappersTermQuery() throws Exception {
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("person",
-            new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        String[] expected = new String[] {"banon", "last location", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
-        }
-        AllFieldMapper mapper = docMapper.allFieldMapper();
-        assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
-            Matchers.instanceOf(AllTermQuery.class));
-    }
-
-    // #6187: make sure we see AllTermQuery even when offsets are indexed in the _all field:
-    public void testAllMappersWithOffsetsTermQuery() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping_offsets_on_all.json");
-        DocumentMapper docMapper =
-            createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        String[] expected = new String[] {"banon", "last location", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
-        }
-        AllFieldMapper mapper = docMapper.allFieldMapper();
-        assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
-            Matchers.instanceOf(AllTermQuery.class));
-    }
-
-    // #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields have boost
-    public void testBoostWithOmitPositions() throws Exception {
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json");
-        DocumentMapper docMapper =
-            createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        for (IndexableField field : fields) {
-            // _all field omits positions, so we should not get AllTokenStream even though fields are boosted
-            assertThat(field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
-        }
-    }
-
-    // #6187: if no fields were boosted, we shouldn't use AllTokenStream
-    public void testNoBoost() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
-            .parse("person", new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        for (IndexableField field : fields) {
-            // no fields have boost, so we should not see AllTokenStream:
-            assertThat(field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
-        }
-    }
-
-    public void testSimpleAllMappersWithReparse() throws Exception {
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
-        DocumentMapper docMapper = parser.parse("person", new CompressedXContent(mapping));
-        String builtMapping = docMapper.mappingSource().string();
-        // reparse it
-        DocumentMapper builtDocMapper = parser.parse("person", new CompressedXContent(builtMapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(3));
-        String[] expected = new String[] {"banon", "last location", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(true));
-            if (i == 0) {
-                // The field "name.last" is boosted so we should see AllTokenStream used:
-                assertThat(fields[i].tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                    Matchers.instanceOf(AllTokenStream.class));
-            } else {
-                assertThat(fields[i].tokenStream(docMapper.mappers().indexAnalyzer(), null),
-                    Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
-            }
-        }
-    }
-
-    public void testSimpleAllMappersWithStore() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
-            .parse("person", new CompressedXContent(mapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(2));
-        String[] expected = new String[] {"banon", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
-        }
-    }
-
-
-    public void testSimpleAllMappersWithReparseWithStore() throws Exception {
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
-        DocumentMapper docMapper = parser.parse("person", new CompressedXContent(mapping));
-        String builtMapping = docMapper.mappingSource().string();
-        // reparse it
-        DocumentMapper builtDocMapper = parser.parse("person", new CompressedXContent(builtMapping));
-        byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
-        Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
-
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(2));
-        String[] expected = new String[] {"banon", "1"};
-        for (int i = 0; i < fields.length; i++) {
-            assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
-        }
-    }
-
-    public void testRandom() throws Exception {
-        boolean norms = true;
-        boolean stored = false;
-        boolean enabled = true;
-        boolean tv_stored = false;
-        boolean tv_payloads = false;
-        boolean tv_offsets = false;
-        boolean tv_positions = false;
-        String similarity = null;
-        XContentBuilder mappingBuilder = jsonBuilder();
-        mappingBuilder.startObject().startObject("test");
-        List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
-        boolean allDefault = true;
-        if (frequently()) {
-            allDefault = false;
-            mappingBuilder.startObject("_all");
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("norms", norms = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean()));
-            }
-            if (randomBoolean()) {
-                booleanOptionList.add(new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean()));
-            }
-            Collections.shuffle(booleanOptionList, random());
-            for (Tuple<String, Boolean> option : booleanOptionList) {
-                mappingBuilder.field(option.v1(), option.v2().booleanValue());
-            }
-            tv_stored |= tv_positions || tv_payloads || tv_offsets;
-            if (randomBoolean()) {
-                mappingBuilder.field("similarity", similarity = "BM25");
-            }
-            mappingBuilder.endObject();
-        }
-
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = mappingBuilder.endObject().endObject().bytes().utf8ToString();
-        logger.info("Mapping: {}", mapping);
-        DocumentMapper docMapper = parser.parse("test", new CompressedXContent(mapping));
-        String builtMapping = docMapper.mappingSource().string();
-        // reparse it
-        DocumentMapper builtDocMapper = parser.parse("test", new CompressedXContent(builtMapping));
-
-        byte[] json = BytesReference.toBytes(jsonBuilder().startObject()
-                .field("foo", "bar")
-                .field("foobar", "foobar")
-                .endObject().bytes());
-        Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        if (enabled) {
-            assertThat(fields.length, equalTo(2));
-            String[] expected = new String[] {"bar", "foobar"};
-            for (int i = 0; i < fields.length; i++) {
-                assertThat(fields[i].fieldType().omitNorms(), equalTo(!norms));
-                assertThat(fields[i].fieldType().stored(), equalTo(stored));
-                assertThat(fields[i].fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
-                assertThat(fields[i].fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
-                assertThat(fields[i].fieldType().storeTermVectorPositions(), equalTo(tv_positions));
-                assertThat(fields[i].fieldType().storeTermVectors(), equalTo(tv_stored));
-                assertThat(fields[i].stringValue(), equalTo(expected[i]));
-            }
-        } else {
-            assertThat(fields.length, equalTo(0));
-        }
-        if (similarity == null) {
-            assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue());
-        }   else {
-            assertThat(similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name()));
-        }
-        if (allDefault) {
-            BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
-            XContentBuilder b =  new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
-            XContentBuilder xContentBuilder = builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
-            xContentBuilder.flush();
-            assertThat(bytesStreamOutput.size(), equalTo(0));
-        }
-
-    }
-
-    public void testMultiField_includeInAllSetToFalse() throws IOException {
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/" +
-                "multifield-mapping_include_in_all_set_to_false.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
-            .parse("test", new CompressedXContent(mapping));
-
-        XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject()
-                .field("foo")
-                    .startObject()
-                        .field("bar", "Elasticsearch rules!")
-                    .endObject()
-                .endObject();
-
-        Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(0));
-    }
-
-    public void testMultiField_defaults() throws IOException {
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/multifield-mapping_default.json");
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser()
-            .parse("test", new CompressedXContent(mapping));
-
-        XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject()
-                .field("foo")
-                .startObject()
-                .field("bar", "Elasticsearch rules!")
-                .endObject()
-                .endObject();
-
-        Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
-        IndexableField[] fields = doc.getFields("_all");
-        assertThat(fields.length, equalTo(1));
-        assertThat(fields[0].stringValue(), equalTo("Elasticsearch rules!"));
-    }
-
-    public void testMisplacedTypeInRoot() throws IOException {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_type_in_root.json");
-        try {
-            createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
-            fail("Expected MapperParsingException");
-        } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters"));
-            assertThat(e.getMessage(), containsString("[type : text]"));
-        }
-    }
-
-    // related to https://github.com/elastic/elasticsearch/issues/5864
-    public void testMistypedTypeInRoot() throws IOException {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mistyped_type_in_root.json");
-        try {
-            createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
-            fail("Expected MapperParsingException");
-        } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters"));
-            assertThat(e.getMessage(), containsString("type=text"));
-        }
-    }
-
-    // issue https://github.com/elastic/elasticsearch/issues/5864
-    public void testMisplacedMappingAsRoot() throws IOException {
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/misplaced_mapping_key_in_root.json");
-        try {
-            createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
-            fail("Expected MapperParsingException");
-        } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), containsString("Root mapping definition has unsupported parameters"));
-            assertThat(e.getMessage(), containsString("type=text"));
-        }
-    }
-
-    // issue https://github.com/elastic/elasticsearch/issues/5864
-    // test that RootObjectMapping still works
-    public void testRootObjectMapperPropertiesDoNotCauseException() throws IOException {
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_dynamic_template_mapping.json");
-        parser.parse("test", new CompressedXContent(mapping));
-        mapping =
-            copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_dynamic_date_formats_mapping.json");
-        parser.parse("test", new CompressedXContent(mapping));
-        mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_date_detection_mapping.json");
-        parser.parse("test", new CompressedXContent(mapping));
-        mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/type_numeric_detection_mapping.json");
-        parser.parse("test", new CompressedXContent(mapping));
-    }
-
-    public void testDocValuesNotAllowed() throws IOException {
-        String mapping = jsonBuilder().startObject().startObject("type")
-            .startObject("_all")
-                .field("doc_values", true)
-            .endObject().endObject().endObject().string();
-        try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getDetailedMessage(),
-                containsString("[_all] is always tokenized and cannot have doc values"));
-        }
-
-    }
-
-    public void testAutoBoost() throws Exception {
-        for (boolean boost : new boolean[] {false, true}) {
-            String index = "test_" + boost;
-            IndexService indexService =
-                createIndex(index, client().admin().indices().prepareCreate(index)
-                    .addMapping("type", "foo", "type=text" + (boost ? ",boost=2" : "")));
-            client().prepareIndex(index, "type").setSource("foo", "bar").get();
-            client().admin().indices().prepareRefresh(index).get();
-            Query query = indexService.mapperService()
-                .documentMapper("type").allFieldMapper().fieldType().termQuery("bar", null);
-            try (Searcher searcher = indexService.getShardOrNull(0).acquireSearcher("tests")) {
-                query = searcher.searcher().rewrite(query);
-                final Class<?> expected = boost ? AllTermQuery.class : TermQuery.class;
-                assertThat(query, Matchers.instanceOf(expected));
-            }
-        }
-    }
-
-    public void testIncludeInObjectNotAllowed() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject()
-            .startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService()
-            .documentMapperParser().parse("type", new CompressedXContent(mapping));
-
-        try {
-            docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject().field("_all", "foo").endObject().bytes());
-            fail("Expected failure to parse metadata field");
-        } catch (MapperParsingException e) {
-            assertTrue(e.getMessage(),
-                e.getMessage().contains("Field [_all] is a metadata field and cannot be added inside a document"));
-        }
-    }
-
-    public void testAllDefaults() {
-        // We use to have a bug with the default mapping having null analyzers because
-        // it was not fully constructed and was in particular lacking analyzers
-        IndexService index = createIndex("index", Settings.EMPTY, "type");
-        AllFieldMapper all = index.mapperService().documentMapper("type").allFieldMapper();
-        assertNotNull(all.fieldType().indexAnalyzer());
-        assertNotNull(all.fieldType().searchAnalyzer());
-        assertNotNull(all.fieldType().searchQuoteAnalyzer());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
index 308f7755275..2486f91ccd3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java
@@ -30,8 +30,12 @@ import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -43,19 +47,25 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
 
 public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
-
-    IndexService indexService;
-    DocumentMapperParser parser;
+    private IndexService indexService;
+    private DocumentMapperParser parser;
+    private DocumentMapperParser preEs6Parser;
 
     @Before
     public void setup() {
         indexService = createIndex("test");
         parser = indexService.mapperService().documentMapperParser();
+
+        IndexService preEs6IndexService = createIndex("legacy",
+            Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_0_0).build());
+        preEs6Parser = preEs6IndexService.mapperService().documentMapperParser();
     }
 
     @Override
@@ -121,6 +131,69 @@ public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("{\"field\":{\"type\":\"boolean\",\"doc_values\":false,\"null_value\":true}}", builder.string());
     }
 
+    public void testParsesPreEs6BooleansLenient() throws IOException {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field1")
+                            .field("type", "boolean")
+                        .endObject()
+                        .startObject("field2")
+                            .field("type", "boolean")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().string();
+        DocumentMapper defaultMapper = preEs6Parser.parse("type", new CompressedXContent(mapping));
+
+        String falsy = randomFrom("false", "off", "no", "0");
+        String truthy = randomFrom("true", "on", "yes", "1");
+
+        ParsedDocument parsedDoc = defaultMapper.parse("legacy", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+                .field("field1", falsy)
+                .field("field2", truthy)
+            .endObject()
+            .bytes());
+        Document doc = parsedDoc.rootDoc();
+        assertEquals("F", doc.getField("field1").stringValue());
+        assertEquals("T", doc.getField("field2").stringValue());
+
+        List<String> expectedDeprecationWarnings = new ArrayList<>();
+        if (Booleans.isBoolean(falsy) == false) {
+            expectedDeprecationWarnings.add("Expected a boolean for property [field1] but got ["+ falsy + "]");
+        }
+        if (Booleans.isBoolean(truthy) == false) {
+            expectedDeprecationWarnings.add("Expected a boolean for property [field2] but got [" + truthy + "]");
+        }
+
+        if (expectedDeprecationWarnings.isEmpty() == false) {
+            assertWarnings(expectedDeprecationWarnings.toArray(new String[1]));
+        }
+    }
+
+    public void testParsesEs6BooleansStrict() throws IOException {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "boolean")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().string();
+        DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
+        BytesReference source = XContentFactory.jsonBuilder()
+                .startObject()
+                    // omit "false"/"true" here as they should still be parsed correctly
+                    .field("field", randomFrom("off", "no", "0", "on", "yes", "1"))
+                .endObject().bytes();
+        MapperParsingException ex = expectThrows(MapperParsingException.class, () -> defaultMapper.parse("test", "type", "1", source));
+        assertEquals("failed to parse [field]", ex.getMessage());
+    }
+
     public void testMultiFields() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties")
@@ -133,7 +206,8 @@ public class BooleanFieldMapperTests extends ESSingleNodeTestCase {
                         .endObject()
                     .endObject().endObject()
                 .endObject().endObject().string();
-        DocumentMapper mapper = indexService.mapperService().merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        DocumentMapper mapper = indexService.mapperService()
+            .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
         assertEquals(mapping, mapper.mappingSource().toString());
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
index 4242581ffdb..300203b1a35 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
@@ -183,44 +183,6 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals(0, fields.length);
     }
 
-    public void testIncludeInAll() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "date").endObject().endObject()
-                .endObject().endObject().string();
-
-        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "2016-03-11")
-                .endObject()
-                .bytes());
-
-        IndexableField[] fields = doc.rootDoc().getFields("_all");
-        assertEquals(1, fields.length);
-        assertEquals("2016-03-11", fields[0].stringValue());
-
-        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "date")
-                .field("include_in_all", false).endObject().endObject()
-                .endObject().endObject().string();
-
-        mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "2016-03-11")
-                .endObject()
-                .bytes());
-
-        fields = doc.rootDoc().getFields("_all");
-        assertEquals(0, fields.length);
-    }
-
     public void testChangeFormat() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("field").field("type", "date")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
index b22b48767a8..c714a0f280d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
@@ -67,11 +67,12 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
                 ((DateFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA));
             }
         });
-        nowInMillis = randomPositiveLong();
+        nowInMillis = randomNonNegativeLong();
     }
 
     public void testIsFieldWithinQueryEmptyReader() throws IOException {
-        QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, null, null, () -> nowInMillis);
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
         IndexReader reader = new MultiReader();
         DateFieldType ft = new DateFieldType();
         ft.setName("my_date");
@@ -81,7 +82,8 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
 
     private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader,
             DateTimeZone zone, DateMathParser alternateFormat) throws IOException {
-        QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, null, null, () -> nowInMillis);
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02",
                 randomBoolean(), randomBoolean(), null, null, context));
         assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-06-20",
@@ -128,7 +130,8 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
         DateFieldType ft2 = new DateFieldType();
         ft2.setName("my_date2");
 
-        QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, null, null, () -> nowInMillis);
+        QueryRewriteContext context = new QueryRewriteContext(null, null, null, xContentRegistry(), null, null,
+                () -> nowInMillis);
         assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context));
         IOUtils.close(reader, w, dir);
     }
@@ -163,7 +166,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
         QueryShardContext context = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(),
                         indexSettings),
-                null, null, null, null, null, null, null, null, () -> nowInMillis);
+                null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date = "2015-10-12T14:10:55";
@@ -182,7 +185,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase {
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build();
         QueryShardContext context = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings),
-                null, null, null, null, null, null, null, null, () -> nowInMillis);
+                null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
         String date1 = "2015-10-12T14:10:55";
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
index 6cd16ffc072..27b90801ecc 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
@@ -43,7 +43,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
 
         private final String output;
 
-        public FakeAnalyzer(String output) {
+        FakeAnalyzer(String output) {
             this.output = output;
         }
 
@@ -70,7 +70,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
 
     static class FakeFieldType extends TermBasedFieldType {
 
-        public FakeFieldType() {
+        FakeFieldType() {
             super();
         }
 
@@ -94,7 +94,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
 
         private static final Settings SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
 
-        public FakeFieldMapper(String simpleName, MappedFieldType fieldType) {
+        FakeFieldMapper(String simpleName, MappedFieldType fieldType) {
             super(simpleName, fieldType.clone(), fieldType.clone(), SETTINGS, null, null);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
index 90784a6df37..a29566bfa4e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java
@@ -32,13 +32,11 @@ import static org.hamcrest.Matchers.equalTo;
 public class DocumentMapperParserTests extends ESSingleNodeTestCase {
     public void testTypeLevel() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_all").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.allFieldMapper().enabled(), equalTo(false));
     }
 
     public void testFieldNameWithDots() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index f5a92d3f979..314157650d6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -47,8 +47,10 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 
 // TODO: make this a real unit test
 public class DocumentParserTests extends ESSingleNodeTestCase {
@@ -1217,47 +1219,80 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         assertThat(doc.rootDoc().get("type.inner.inner_field"), equalTo("inner_value"));
     }
 
-    public void testIncludeInAllPropagation() throws IOException {
-        String defaultMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type")
-                    .field("dynamic", "strict")
-                    .startObject("properties")
-                        .startObject("a")
-                            .field("type", "keyword")
-                        .endObject()
-                        .startObject("o")
-                            .field("include_in_all", false)
-                            .startObject("properties")
-                                .startObject("a")
-                                    .field("type", "keyword")
-                                .endObject()
-                                .startObject("o")
-                                    .field("include_in_all", true)
-                                    .startObject("properties")
-                                        .startObject("a")
-                                            .field("type", "keyword")
-                                        .endObject()
-                                    .endObject()
-                                .endObject()
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject().endObject().string();
-        DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(defaultMapping));
-        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+    public void testDynamicDateDetectionDisabledOnNumbers() throws IOException {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_date_formats")
+                    .value("yyyy")
+                .endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject()
+                .field("foo", "2016")
+            .endObject().bytes();
+
+        // Even though we matched the dynamic format, we do not match on numbers,
+        // which are too likely to be false positives
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        Mapping update = doc.dynamicMappingsUpdate();
+        assertNotNull(update);
+        Mapper dateMapper = update.root().getMapper("foo");
+        assertNotNull(dateMapper);
+        assertThat(dateMapper, not(instanceOf(DateFieldMapper.class)));
+    }
+
+    public void testDynamicDateDetectionEnabledWithNoSpecialCharacters() throws IOException {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_date_formats")
+                    .value("yyyy MM")
+                .endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject()
+                .field("foo", "2016 12")
+            .endObject().bytes();
+
+        // We should have generated a date field
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        Mapping update = doc.dynamicMappingsUpdate();
+        assertNotNull(update);
+        Mapper dateMapper = update.root().getMapper("foo");
+        assertNotNull(dateMapper);
+        assertThat(dateMapper, instanceOf(DateFieldMapper.class));
+    }
+
+    public void testDynamicFieldsStartingAndEndingWithDot() throws Exception {
+        BytesReference bytes = XContentFactory.jsonBuilder().startObject().startArray("top.")
+                .startObject().startArray("foo.")
                 .startObject()
-                    .field("a", "b")
-                    .startObject("o")
-                        .field("a", "c")
-                        .startObject("o")
-                            .field("a", "d")
-                        .endObject()
-                    .endObject()
-                .endObject().bytes());
-        Set<String> values = new HashSet<>();
-        for (IndexableField f : doc.rootDoc().getFields("_all")) {
-            values.add(f.stringValue());
+                .field("thing", "bah")
+                .endObject().endArray()
+                .endObject().endArray()
+                .endObject().bytes();
+
+        client().prepareIndex("idx", "type").setSource(bytes).get();
+
+        bytes = XContentFactory.jsonBuilder().startObject().startArray("top.")
+                .startObject().startArray("foo.")
+                .startObject()
+                .startObject("bar.")
+                .startObject("aoeu")
+                .field("a", 1).field("b", 2)
+                .endObject()
+                .endObject()
+                .endObject()
+                .endArray().endObject().endArray()
+                .endObject().bytes();
+
+        try {
+            client().prepareIndex("idx", "type").setSource(bytes).get();
+            fail("should have failed to dynamically introduce a double-dot field");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(),
+                    containsString("object field starting or ending with a [.] makes object resolution ambiguous: [top..foo..bar]"));
         }
-        assertEquals(new HashSet<>(Arrays.asList("b", "d")), values);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 23e33546db2..46749e792ed 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -20,11 +20,15 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.TransportBulkAction;
+import org.elasticsearch.action.bulk.TransportShardBulkAction;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
+import org.elasticsearch.action.update.UpdateHelper;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -55,40 +59,39 @@ import static org.hamcrest.CoreMatchers.instanceOf;
 
 public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
-    private static ThreadPool THREAD_POOL;
+    private static ThreadPool threadPool;
     private ClusterService clusterService;
-    private Transport transport;
     private TransportService transportService;
-    private IndicesService indicesService;
-    private ShardStateAction shardStateAction;
-    private ActionFilters actionFilters;
-    private IndexNameExpressionResolver indexNameExpressionResolver;
-    private AutoCreateIndex autoCreateIndex;
-    private Settings settings;
+    private TransportBulkAction transportBulkAction;
 
     @BeforeClass
     public static void createThreadPool() {
-        THREAD_POOL = new TestThreadPool("DynamicMappingDisabledTests");
+        threadPool = new TestThreadPool("DynamicMappingDisabledTests");
     }
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        settings = Settings.builder()
+        Settings settings = Settings.builder()
                 .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
                 .build();
-        clusterService = createClusterService(THREAD_POOL);
-        transport = new MockTcpTransport(settings, THREAD_POOL, BigArrays.NON_RECYCLING_INSTANCE,
-                    new NoneCircuitBreakerService(), new NamedWriteableRegistry(Collections.emptyList()),
-            new NetworkService(settings, Collections.emptyList()));
-        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
-        indicesService = getInstanceFromNode(IndicesService.class);
-        shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, THREAD_POOL);
-        actionFilters = new ActionFilters(Collections.emptySet());
-        indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
-        autoCreateIndex = new AutoCreateIndex(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                indexNameExpressionResolver);
+        clusterService = createClusterService(threadPool);
+        Transport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+                new NoneCircuitBreakerService(), new NamedWriteableRegistry(Collections.emptyList()),
+                new NetworkService(settings, Collections.emptyList()));
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        ShardStateAction shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, threadPool);
+        ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
+        AutoCreateIndex autoCreateIndex = new AutoCreateIndex(settings, new ClusterSettings(settings,
+                ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), indexNameExpressionResolver);
+        UpdateHelper updateHelper = new UpdateHelper(settings, null);
+        TransportShardBulkAction shardBulkAction = new TransportShardBulkAction(settings, transportService, clusterService,
+                indicesService, threadPool, shardStateAction, null, updateHelper, actionFilters, indexNameExpressionResolver);
+        transportBulkAction = new TransportBulkAction(settings, threadPool, transportService, clusterService,
+                null, shardBulkAction, null, actionFilters, indexNameExpressionResolver, autoCreateIndex, System::currentTimeMillis);
     }
 
     @After
@@ -101,31 +104,29 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
     @AfterClass
     public static void destroyThreadPool() {
-        ThreadPool.terminate(THREAD_POOL, 30, TimeUnit.SECONDS);
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
         // since static must set to null to be eligible for collection
-        THREAD_POOL = null;
+        threadPool = null;
     }
 
     public void testDynamicDisabled() {
-        TransportIndexAction action = new TransportIndexAction(settings, transportService, clusterService,
-                indicesService, null, THREAD_POOL, shardStateAction, null, null, actionFilters, indexNameExpressionResolver,
-                autoCreateIndex);
-
         IndexRequest request = new IndexRequest("index", "type", "1");
-        request.source("foo", 3);
+        request.source(Requests.INDEX_CONTENT_TYPE, "foo", 3);
+        BulkRequest bulkRequest = new BulkRequest();
+        bulkRequest.add(request);
         final AtomicBoolean onFailureCalled = new AtomicBoolean();
 
-        action.execute(request, new ActionListener<IndexResponse>() {
+        transportBulkAction.execute(bulkRequest, new ActionListener<BulkResponse>() {
             @Override
-            public void onResponse(IndexResponse indexResponse) {
-                fail("Indexing request should have failed");
+            public void onResponse(BulkResponse bulkResponse) {
+                fail("onResponse shouldn't be called");
             }
 
             @Override
             public void onFailure(Exception e) {
                 onFailureCalled.set(true);
                 assertThat(e, instanceOf(IndexNotFoundException.class));
-                assertEquals(e.getMessage(), "no such index");
+                assertEquals("no such index and [index.mapper.dynamic] is [false]", e.getMessage());
             }
         });
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 28a24f67ac9..41256113cd6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -197,7 +197,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
 
     private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception {
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes());
+        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes(), builder.contentType());
         try (XContentParser xContentParser = createParser(JsonXContent.jsonXContent, source.source())) {
             ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, xContentParser);
             assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
@@ -550,7 +550,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                     .field("field", "foo")
                 .endObject();
-        SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes());
+        SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes(), json.contentType());
         DocumentMapper mapper = indexService.mapperService().documentMapper("type1");
         assertNull(mapper.mappers().getMapper("field.raw"));
         ParsedDocument parsed = mapper.parse(source);
@@ -644,6 +644,59 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         assertThat(mapper, instanceOf(TextFieldMapper.class));
     }
 
+    public void testDateDetectionInheritsFormat() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_date_formats")
+                    .value("yyyy-MM-dd")
+                .endArray()
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("dates")
+                            .field("match_mapping_type", "date")
+                            .field("match", "*2")
+                            .startObject("mapping")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                    .startObject()
+                        .startObject("dates")
+                            .field("match_mapping_type", "date")
+                            .field("match", "*3")
+                            .startObject("mapping")
+                                .field("format", "yyyy-MM-dd||epoch_millis")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endArray()
+                .endObject().endObject().string();
+
+        IndexService index = createIndex("test");
+        client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
+        DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
+
+        ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("date1", "2016-11-20")
+                    .field("date2", "2016-11-20")
+                    .field("date3", "2016-11-20")
+                .endObject()
+                .bytes());
+        assertNotNull(doc.dynamicMappingsUpdate());
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource(doc.dynamicMappingsUpdate().toString()).get());
+
+        defaultMapper = index.mapperService().documentMapper("type");
+
+        DateFieldMapper dateMapper1 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date1");
+        DateFieldMapper dateMapper2 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date2");
+        DateFieldMapper dateMapper3 = (DateFieldMapper) defaultMapper.mappers().smartNameFieldMapper("date3");
+        // inherited from dynamic date format
+        assertEquals("yyyy-MM-dd", dateMapper1.fieldType().dateTimeFormatter().format());
+        // inherited from dynamic date format since the mapping in the template did not specify a format
+        assertEquals("yyyy-MM-dd", dateMapper2.fieldType().dateTimeFormatter().format());
+        // not inherited from the dynamic date format since the template defined an explicit format
+        assertEquals("yyyy-MM-dd||epoch_millis", dateMapper3.fieldType().dateTimeFormatter().format());
+    }
+
     public void testDynamicTemplateOrder() throws IOException {
         // https://github.com/elastic/elasticsearch/issues/18625
         // elasticsearch used to apply templates that do not have a match_mapping_type first
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
index f41b36068ad..ffc921d013a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java
@@ -49,14 +49,15 @@ public class DynamicTemplateTests extends ESTestCase {
         templateDef.put("mapping", Collections.singletonMap("store", true));
         // if a wrong match type is specified, we ignore the template
         assertNull(DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5));
-
+        assertWarnings("match_mapping_type [short] is invalid and will be ignored: No field type matched on [short], " +
+                "possible values are [object, string, long, double, boolean, date, binary]");
         Map<String, Object> templateDef2 = new HashMap<>();
         templateDef2.put("match_mapping_type", "text");
         templateDef2.put("mapping", Collections.singletonMap("store", true));
         // if a wrong match type is specified, we ignore the template
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> DynamicTemplate.parse("my_template", templateDef2, Version.V_6_0_0_alpha1_UNRELEASED));
-        assertEquals("No xcontent type matched on [text], possible values are [object, string, long, double, boolean, date, binary]",
+        assertEquals("No field type matched on [text], possible values are [object, string, long, double, boolean, date, binary]",
                 e.getMessage());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
index af5e2553be7..46ffc4e5864 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java
@@ -64,7 +64,8 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject(ExternalMetadataMapper.CONTENT_TYPE)
@@ -112,7 +113,8 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -175,7 +177,8 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
index 94f129219f2..cb1de56808a 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
@@ -114,7 +114,7 @@ public class ExternalMapper extends FieldMapper {
 
     static class ExternalFieldType extends TermBasedFieldType {
 
-        public ExternalFieldType() {}
+        ExternalFieldType() {}
 
         protected ExternalFieldType(ExternalFieldType ref) {
             super(ref);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
index 7de66511f59..dc2de3e3dd9 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java
@@ -97,7 +97,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject()
                 .bytes());
 
-        assertFieldNames(set("a", "a.keyword", "b", "b.c", "_uid", "_type", "_version", "_seq_no", "_primary_term", "_source", "_all"), doc);
+        assertFieldNames(set("a", "a.keyword", "b", "b.c", "_uid", "_type", "_version", "_seq_no", "_primary_term", "_source"), doc);
     }
 
     public void testExplicitEnabled() throws Exception {
@@ -114,7 +114,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
             .endObject()
             .bytes());
 
-        assertFieldNames(set("field", "field.keyword", "_uid", "_type", "_version", "_seq_no", "_primary_term", "_source", "_all"), doc);
+        assertFieldNames(set("field", "field.keyword", "_uid", "_type", "_version", "_seq_no", "_primary_term", "_source"), doc);
     }
 
     public void testDisabled() throws Exception {
@@ -175,7 +175,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
 
         private static class DummyFieldType extends TermBasedFieldType {
 
-            public DummyFieldType() {
+            DummyFieldType() {
                 super();
             }
 
@@ -237,9 +237,11 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
         Supplier<QueryShardContext> queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
-        MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+        MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.getIndexAnalyzers(),
+                indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService,
-                indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+                indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry,
+                queryShardContext);
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         ParsedDocument parsedDocument = mapper.parse("index", "type", "id", new BytesArray("{}"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
index 1091e313416..4ae9b004413 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
@@ -207,7 +207,7 @@ public class FieldTypeLookupTests extends ESTestCase {
     }
 
     static class OtherFakeFieldType extends TermBasedFieldType {
-        public OtherFakeFieldType() {
+        OtherFakeFieldType() {
         }
 
         protected OtherFakeFieldType(OtherFakeFieldType ref) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index 96ca2e72b95..5d86602c4ca 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -19,6 +19,8 @@
 package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -32,6 +34,8 @@ import java.util.List;
 /** Base test case for subclasses of MappedFieldType */
 public abstract class FieldTypeTestCase extends ESTestCase {
 
+    private static final Settings INDEX_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+
     /** Abstraction for mutating a property of a MappedFieldType */
     public abstract static class Modifier {
         /** The name of the property that is being modified. Used in test failure messages. */
@@ -117,17 +121,17 @@ public abstract class FieldTypeTestCase extends ESTestCase {
         new Modifier("similarity", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSimilarity(new BM25SimilarityProvider("foo", Settings.EMPTY));
+                ft.setSimilarity(new BM25SimilarityProvider("foo", Settings.EMPTY, INDEX_SETTINGS));
             }
         },
         new Modifier("similarity", false) {
             @Override
             public void modify(MappedFieldType ft) {
-                ft.setSimilarity(new BM25SimilarityProvider("foo", Settings.EMPTY));
+                ft.setSimilarity(new BM25SimilarityProvider("foo", Settings.EMPTY, INDEX_SETTINGS));
             }
             @Override
             public void normalizeOther(MappedFieldType other) {
-                other.setSimilarity(new BM25SimilarityProvider("bar", Settings.EMPTY));
+                other.setSimilarity(new BM25SimilarityProvider("bar", Settings.EMPTY, INDEX_SETTINGS));
             }
         },
         new Modifier("eager_global_ordinals", true) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
index 572188d7a5d..5972a8ecee8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
@@ -22,27 +22,17 @@ import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.Collection;
 
-import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
index 55b6e7bf71c..b7ad6a7e4c3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -33,7 +34,7 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class IdFieldMapperTests extends ESSingleNodeTestCase {
-    
+
     public void testId() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject().string();
@@ -54,7 +55,7 @@ public class IdFieldMapperTests extends ESSingleNodeTestCase {
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject().field("_id", "1").endObject().bytes()));
+                .startObject().field("_id", "1").endObject().bytes(), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
index 682e487035b..9878267d353 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java
@@ -187,44 +187,6 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals(0, fields.length);
     }
 
-    public void testIncludeInAll() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
-                .endObject().endObject().string();
-
-        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "::1")
-                .endObject()
-                .bytes());
-
-        IndexableField[] fields = doc.rootDoc().getFields("_all");
-        assertEquals(1, fields.length);
-        assertEquals("::1", fields[0].stringValue());
-
-        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "ip")
-                .field("include_in_all", false).endObject().endObject()
-                .endObject().endObject().string();
-
-        mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "::1")
-                .endObject()
-                .bytes());
-
-        fields = doc.rootDoc().getFields("_all");
-        assertEquals(0, fields.length);
-    }
-
     public void testNullValue() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject()
                 .startObject("type")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
index 6d3c5bcbce1..bffe58db3a6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java
@@ -25,8 +25,10 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -51,7 +53,11 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
 
     @Before
     public void setup() {
-        indexService = createIndex("test");
+        indexService = createIndex("test", Settings.builder()
+                .put("index.analysis.normalizer.my_lowercase.type", "custom")
+                .putArray("index.analysis.normalizer.my_lowercase.filter", "lowercase")
+                .put("index.analysis.normalizer.my_asciifolding.type", "custom")
+                .putArray("index.analysis.normalizer.my_asciifolding.filter", "asciifolding").build());
         parser = indexService.mapperService().documentMapperParser();
     }
 
@@ -283,6 +289,62 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
         assertFalse(fields[0].fieldType().omitNorms());
     }
 
+    public void testNormalizer() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_lowercase").endObject().endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .field("field", "AbC")
+                .endObject()
+                .bytes());
+
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+
+        assertEquals(new BytesRef("abc"), fields[0].binaryValue());
+        IndexableFieldType fieldType = fields[0].fieldType();
+        assertThat(fieldType.omitNorms(), equalTo(true));
+        assertFalse(fieldType.tokenized());
+        assertFalse(fieldType.stored());
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS));
+        assertThat(fieldType.storeTermVectors(), equalTo(false));
+        assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPositions(), equalTo(false));
+        assertThat(fieldType.storeTermVectorPayloads(), equalTo(false));
+        assertEquals(DocValuesType.NONE, fieldType.docValuesType());
+
+        assertEquals(new BytesRef("abc"), fields[1].binaryValue());
+        fieldType = fields[1].fieldType();
+        assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
+        assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType());
+    }
+
+    public void testUpdateNormalizer() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_lowercase").endObject().endObject()
+                .endObject().endObject().string();
+        indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, randomBoolean());
+
+        String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field")
+                .field("type", "keyword").field("normalizer", "my_asciifolding").endObject().endObject()
+                .endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> indexService.mapperService().merge("type",
+                        new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, randomBoolean()));
+        assertEquals(
+                "Mapper for [field] conflicts with existing mapping in other types:\n[mapper [field] has different [normalizer]]",
+                e.getMessage());
+    }
+
     public void testEmptyName() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject()
                 .startObject("type")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
index fbbabf8ee3a..00eecc669f8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java
@@ -20,22 +20,41 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Arrays;
 
 public class KeywordFieldTypeTests extends FieldTypeTestCase {
+
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("normalizer", false) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                ((KeywordFieldType) ft).setNormalizer(Lucene.KEYWORD_ANALYZER);
+            }
+        });
+    }
+
     @Override
     protected MappedFieldType createDefaultFieldType() {
         return new KeywordFieldMapper.KeywordFieldType();
@@ -62,6 +81,31 @@ public class KeywordFieldTypeTests extends FieldTypeTestCase {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testTermQueryWithNormalizer() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        Analyzer normalizer = new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                Tokenizer in = new WhitespaceTokenizer();
+                TokenFilter out = new LowerCaseFilter(in);
+                return new TokenStreamComponents(in, out);
+            }
+            @Override
+            protected TokenStream normalize(String fieldName, TokenStream in) {
+                return new LowerCaseFilter(in);
+            }
+        };
+        ft.setSearchAnalyzer(new NamedAnalyzer("my_normalizer", AnalyzerScope.INDEX, normalizer));
+        assertEquals(new TermQuery(new Term("field", "foo bar")), ft.termQuery("fOo BaR", null));
+
+        ft.setIndexOptions(IndexOptions.NONE);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> ft.termQuery("bar", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
+
     public void testTermsQuery() {
         MappedFieldType ft = createDefaultFieldType();
         ft.setName("field");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index b32339b2357..5543baff1a2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -49,7 +51,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         String index = "test-index";
         String type = ".test-type";
         String field = "field";
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
             client().admin().indices().prepareCreate(index)
                     .addMapping(type, field, "type=text")
                     .execute().actionGet();
@@ -62,7 +64,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         String field = "field";
         String type = new String(new char[256]).replace("\0", "a");
 
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+        MapperException e = expectThrows(MapperException.class, () -> {
             client().admin().indices().prepareCreate(index)
                     .addMapping(type, field, "type=text")
                     .execute().actionGet();
@@ -91,7 +93,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
     public void testIndexIntoDefaultMapping() throws Throwable {
         // 1. test implicit index creation
         ExecutionException e = expectThrows(ExecutionException.class, () -> {
-            client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}").execute().get();
+            client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "1").setSource("{}", XContentType.JSON).execute().get();
         });
         Throwable throwable = ExceptionsHelper.unwrapCause(e.getCause());
         if (throwable instanceof IllegalArgumentException) {
@@ -166,6 +168,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         assertThat(mapperService.unmappedFieldType("long"), instanceOf(NumberFieldType.class));
         // back compat
         assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
+        assertWarnings("[unmapped_type:string] should be replaced with [unmapped_type:keyword]");
     }
 
     public void testMergeWithMap() throws Throwable {
@@ -173,16 +176,16 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         MapperService mapperService = indexService1.mapperService();
         Map<String, Map<String, Object>> mappings = new HashMap<>();
 
-        mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping("{}"));
+        mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping(xContentRegistry(), "{}"));
         MapperException e = expectThrows(MapperParsingException.class,
-            () -> mapperService.merge(mappings, false));
+            () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: "));
 
         mappings.clear();
-        mappings.put("type1", MapperService.parseMapping("{}"));
+        mappings.put("type1", MapperService.parseMapping(xContentRegistry(), "{}"));
 
         e = expectThrows( MapperParsingException.class,
-            () -> mapperService.merge(mappings, false));
+            () -> mapperService.merge(mappings, MergeReason.MAPPING_UPDATE, false));
         assertThat(e.getMessage(), startsWith("Failed to parse mapping [type1]: "));
     }
 
@@ -206,9 +209,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
             .startObject("properties")
             .startObject("field")
             .field("type", "text")
-            .startObject("norms")
-            .field("enabled", false)
-            .endObject()
+            .field("norms", false)
             .endObject()
             .endObject().endObject().bytes());
 
@@ -230,16 +231,42 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
                     .field("enabled", false)
                 .endObject().endObject().bytes());
 
-        indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, enabledAll,
-                MergeReason.MAPPING_UPDATE, random().nextBoolean());
-        assertFalse(indexService.mapperService().allEnabled()); // _default_ does not count
+        Exception e = expectThrows(MapperParsingException.class,
+                () -> indexService.mapperService().merge(MapperService.DEFAULT_MAPPING, enabledAll,
+                        MergeReason.MAPPING_UPDATE, random().nextBoolean()));
+        assertThat(e.getMessage(), containsString("[_all] is disabled in 6.0"));
+    }
 
-        indexService.mapperService().merge("some_type", enabledAll,
-                MergeReason.MAPPING_UPDATE, random().nextBoolean());
-        assertTrue(indexService.mapperService().allEnabled());
+     public void testPartitionedConstraints() {
+        // partitioned index must have routing
+         IllegalArgumentException noRoutingException = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin().indices().prepareCreate("test-index")
+                    .addMapping("type", "{\"type\":{}}")
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", 4)
+                        .put("index.routing_partition_size", 2))
+                    .execute().actionGet();
+        });
+        assertTrue(noRoutingException.getMessage(), noRoutingException.getMessage().contains("must have routing"));
 
-        indexService.mapperService().merge("other_type", disabledAll,
-                MergeReason.MAPPING_UPDATE, random().nextBoolean());
-        assertTrue(indexService.mapperService().allEnabled()); // this returns true if any of the types has _all enabled
+        // partitioned index cannot have parent/child relationships
+        IllegalArgumentException parentException = expectThrows(IllegalArgumentException.class, () -> {
+            client().admin().indices().prepareCreate("test-index")
+                    .addMapping("parent", "{\"parent\":{\"_routing\":{\"required\":true}}}")
+                    .addMapping("child", "{\"child\": {\"_routing\":{\"required\":true}, \"_parent\": {\"type\": \"parent\"}}}")
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", 4)
+                        .put("index.routing_partition_size", 2))
+                    .execute().actionGet();
+        });
+        assertTrue(parentException.getMessage(), parentException.getMessage().contains("cannot have a _parent field"));
+
+        // valid partitioned index
+        assertTrue(client().admin().indices().prepareCreate("test-index")
+            .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+            .setSettings(Settings.builder()
+                .put("index.number_of_shards", 4)
+                .put("index.routing_partition_size", 2))
+            .execute().actionGet().isAcknowledged());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
index 5a5e2ddb509..44484530856 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldCopyToMapperTests.java
@@ -37,7 +37,7 @@ public class MultiFieldCopyToMapperTests extends ESTestCase {
         XContentBuilder mapping = createMappinmgWithCopyToInMultiField();
 
         // first check that for newer versions we throw exception if copy_to is found withing multi field
-        MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY);
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY);
         try {
             mapperService.parse("type", new CompressedXContent(mapping.string()), true);
             fail("Parsing should throw an exception because the mapping contains a copy_to in a multi field");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java
new file mode 100644
index 00000000000..8c6ee8da042
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldIncludeInAllMapperTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+
+public class MultiFieldIncludeInAllMapperTests extends ESTestCase {
+    public void testExceptionForIncludeInAllInMultiFields() throws IOException {
+        XContentBuilder mapping = createMappingWithIncludeInAllInMultiField();
+
+        // first check that for newer versions we throw exception if include_in_all is found withing multi field
+        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY);
+        Exception e = expectThrows(MapperParsingException.class, () ->
+            mapperService.parse("type", new CompressedXContent(mapping.string()), true));
+        assertEquals("include_in_all in multi fields is not allowed. Found the include_in_all in field [c] which is within a multi field.",
+                e.getMessage());
+    }
+
+    private static XContentBuilder createMappingWithIncludeInAllInMultiField() throws IOException {
+        XContentBuilder mapping = jsonBuilder();
+        mapping.startObject()
+                .startObject("type")
+                .startObject("properties")
+                .startObject("a")
+                .field("type", "text")
+                .endObject()
+                .startObject("b")
+                .field("type", "text")
+                .startObject("fields")
+                .startObject("c")
+                .field("type", "text")
+                .field("include_in_all", false)
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        return mapping;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
index 5c70465d77f..8f17b3e0e0d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MultiFieldTests.java
@@ -186,7 +186,8 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         Arrays.sort(multiFieldNames);
 
-        Map<String, Object> sourceAsMap = XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true).v2();
+        Map<String, Object> sourceAsMap =
+            XContentHelper.convertToMap(docMapper.mappingSource().compressedReference(), true, builder.contentType()).v2();
         @SuppressWarnings("unchecked")
         Map<String, Object> multiFields = (Map<String, Object>) XContentMapValues.extractValue("type.properties.my_field.fields", sourceAsMap);
         assertThat(multiFields.size(), equalTo(multiFieldNames.length));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
index f639b9c2041..de3a15f865c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java
@@ -217,45 +217,6 @@ public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase {
         assertEquals(0, fields.length);
     }
 
-    @Override
-    public void doTestIncludeInAll(String type) throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
-                .endObject().endObject().string();
-
-        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", 123)
-                .endObject()
-                .bytes());
-
-        IndexableField[] fields = doc.rootDoc().getFields("_all");
-        assertEquals(1, fields.length);
-        assertEquals("123", fields[0].stringValue());
-
-        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", type)
-                .field("include_in_all", false).endObject().endObject()
-                .endObject().endObject().string();
-
-        mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", 123)
-                .endObject()
-                .bytes());
-
-        fields = doc.rootDoc().getFields("_all");
-        assertEquals(0, fields.length);
-    }
-
     public void testRejectNorms() throws IOException {
         // not supported as of 5.0
         for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
index d7e178404f1..43636b6abd8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java
@@ -20,26 +20,26 @@
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FloatPoint;
 import org.apache.lucene.document.HalfFloatPoint;
+import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 public class NumberFieldTypeTests extends FieldTypeTestCase {
 
@@ -62,6 +62,52 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
                 randomBoolean(), randomBoolean(), null, null, null));
     }
 
+    public void testIntegerTermsQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null));
+        assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null));
+        assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testLongTermsQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null));
+        assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null));
+        assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testByteTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testShortTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testIntegerTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
+    public void testLongTermQueryWithDecimalPart() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery);
+    }
+
     public void testTermQuery() {
         MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
         ft.setName("field");
@@ -74,6 +120,82 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
 
+    public void testRangeQueryWithNegativeBounds() {
+        MappedFieldType ftInt = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ftInt.setName("field");
+        ftInt.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newRangeQuery("field", -3, -3), ftInt.rangeQuery(-3.5, -2.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", -3, -3), ftInt.rangeQuery(-3.5, -2.5, false, false, null));
+        assertEquals(IntPoint.newRangeQuery("field", 0, 0), ftInt.rangeQuery(-0.5, 0.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 0, 0), ftInt.rangeQuery(-0.5, 0.5, false, false, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 2), ftInt.rangeQuery(0.5, 2.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 2), ftInt.rangeQuery(0.5, 2.5, false, false, null));
+        assertEquals(IntPoint.newRangeQuery("field", 0, 2), ftInt.rangeQuery(-0.5, 2.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 0, 2), ftInt.rangeQuery(-0.5, 2.5, false, false, null));
+
+        assertEquals(IntPoint.newRangeQuery("field", -2, 0), ftInt.rangeQuery(-2.5, 0.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", -2, 0), ftInt.rangeQuery(-2.5, 0.5, false, false, null));
+        assertEquals(IntPoint.newRangeQuery("field", -2, -1), ftInt.rangeQuery(-2.5, -0.5, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", -2, -1), ftInt.rangeQuery(-2.5, -0.5, false, false, null));
+
+        MappedFieldType ftLong = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ftLong.setName("field");
+        ftLong.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(LongPoint.newRangeQuery("field", -3, -3), ftLong.rangeQuery(-3.5, -2.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", -3, -3), ftLong.rangeQuery(-3.5, -2.5, false, false, null));
+        assertEquals(LongPoint.newRangeQuery("field", 0, 0), ftLong.rangeQuery(-0.5, 0.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 0, 0), ftLong.rangeQuery(-0.5, 0.5, false, false, null));
+        assertEquals(LongPoint.newRangeQuery("field", 1, 2), ftLong.rangeQuery(0.5, 2.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 1, 2), ftLong.rangeQuery(0.5, 2.5, false, false, null));
+        assertEquals(LongPoint.newRangeQuery("field", 0, 2), ftLong.rangeQuery(-0.5, 2.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 0, 2), ftLong.rangeQuery(-0.5, 2.5, false, false, null));
+
+        assertEquals(LongPoint.newRangeQuery("field", -2, 0), ftLong.rangeQuery(-2.5, 0.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", -2, 0), ftLong.rangeQuery(-2.5, 0.5, false, false, null));
+        assertEquals(LongPoint.newRangeQuery("field", -2, -1), ftLong.rangeQuery(-2.5, -0.5, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", -2, -1), ftLong.rangeQuery(-2.5, -0.5, false, false, null));
+    }
+
+    public void testByteRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testShortRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testIntegerRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
+    public void testLongRangeQueryWithDecimalParts() {
+        MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG);
+        ft.setName("field");
+        ft.setIndexOptions(IndexOptions.DOCS);
+        assertEquals(LongPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null));
+        assertEquals(LongPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null));
+    }
+
     public void testRangeQuery() {
         MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
         ft.setName("field");
@@ -87,36 +209,33 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
     }
 
     public void testConversions() {
-        assertEquals((byte) 3, NumberType.BYTE.parse(3d));
-        assertEquals((short) 3, NumberType.SHORT.parse(3d));
-        assertEquals(3, NumberType.INTEGER.parse(3d));
-        assertEquals(3L, NumberType.LONG.parse(3d));
-        assertEquals(3f, NumberType.HALF_FLOAT.parse(3d));
-        assertEquals(3f, NumberType.FLOAT.parse(3d));
-        assertEquals(3d, NumberType.DOUBLE.parse(3d));
+        assertEquals((byte) 3, NumberType.BYTE.parse(3d, true));
+        assertEquals((short) 3, NumberType.SHORT.parse(3d, true));
+        assertEquals(3, NumberType.INTEGER.parse(3d, true));
+        assertEquals(3L, NumberType.LONG.parse(3d, true));
+        assertEquals(3f, NumberType.HALF_FLOAT.parse(3d, true));
+        assertEquals(3f, NumberType.FLOAT.parse(3d, true));
+        assertEquals(3d, NumberType.DOUBLE.parse(3d, true));
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(3.5));
-        assertEquals("Value [3.5] has a decimal part", e.getMessage());
-        assertEquals(3.5f, NumberType.FLOAT.parse(3.5));
-        assertEquals(3.5d, NumberType.DOUBLE.parse(3.5));
+        assertEquals((byte) 3, NumberType.BYTE.parse(3.5, true));
+        assertEquals((short) 3, NumberType.SHORT.parse(3.5, true));
+        assertEquals(3, NumberType.INTEGER.parse(3.5, true));
+        assertEquals(3L, NumberType.LONG.parse(3.5, true));
 
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128));
+        assertEquals(3.5f, NumberType.FLOAT.parse(3.5, true));
+        assertEquals(3.5d, NumberType.DOUBLE.parse(3.5, true));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128, true));
         assertEquals("Value [128] is out of range for a byte", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536, true));
         assertEquals("Value [65536] is out of range for a short", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L, true));
         assertEquals("Value [2147483648] is out of range for an integer", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d));
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d, true));
         assertEquals("Value [1.0E19] is out of range for a long", e.getMessage());
-        assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1));
-        assertEquals(1.1f, NumberType.FLOAT.parse(1.1));
-        assertEquals(1.1d, NumberType.DOUBLE.parse(1.1));
+        assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1, true));
+        assertEquals(1.1f, NumberType.FLOAT.parse(1.1, true));
+        assertEquals(1.1d, NumberType.DOUBLE.parse(1.1, true));
     }
 
     public void testHalfFloatRange() throws IOException {
@@ -149,4 +268,20 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
         }
         IOUtils.close(reader, dir);
     }
+
+    public void testNegativeZero() {
+        assertEquals(
+                NumberType.DOUBLE.rangeQuery("field", null, -0d, true, true),
+                NumberType.DOUBLE.rangeQuery("field", null, +0d, true, false));
+        assertEquals(
+                NumberType.FLOAT.rangeQuery("field", null, -0f, true, true),
+                NumberType.FLOAT.rangeQuery("field", null, +0f, true, false));
+        assertEquals(
+                NumberType.HALF_FLOAT.rangeQuery("field", null, -0f, true, true),
+                NumberType.HALF_FLOAT.rangeQuery("field", null, +0f, true, false));
+
+        assertFalse(NumberType.DOUBLE.termQuery("field", -0d).equals(NumberType.DOUBLE.termQuery("field", +0d)));
+        assertFalse(NumberType.FLOAT.termQuery("field", -0f).equals(NumberType.FLOAT.termQuery("field", +0f)));
+        assertFalse(NumberType.HALF_FLOAT.termQuery("field", -0f).equals(NumberType.HALF_FLOAT.termQuery("field", +0f)));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
index 9ee9ed16bd6..af5061e62ae 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ParentFieldMapperTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
@@ -53,7 +54,7 @@ public class ParentFieldMapperTests extends ESSingleNodeTestCase {
 
         try {
             docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject().field("_parent", "1122").endObject().bytes()));
+                .startObject().field("_parent", "1122").endObject().bytes(), XContentType.JSON));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
@@ -72,13 +73,14 @@ public class ParentFieldMapperTests extends ESSingleNodeTestCase {
 
         // Indexing parent doc:
         DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type");
-        ParsedDocument doc = parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}")));
+        ParsedDocument doc =
+            parentDocMapper.parse(SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"), XContentType.JSON));
         assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
         assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
 
         // Indexing child doc:
         DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type");
-        doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122"));
+        doc = childDocMapper.parse(SourceToParse.source("test", "child_type", "1", new BytesArray("{}"), XContentType.JSON).parent("1122"));
 
         assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
         assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
@@ -92,7 +94,7 @@ public class ParentFieldMapperTests extends ESSingleNodeTestCase {
                 .startObject()
                 .field("x_field", "x_value")
                 .endObject()
-                .bytes()));
+                .bytes(), XContentType.JSON));
         assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
     }
 
@@ -101,9 +103,9 @@ public class ParentFieldMapperTests extends ESSingleNodeTestCase {
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
         NamedAnalyzer namedAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
         IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer,
-            Collections.emptyMap());
+            Collections.emptyMap(), Collections.emptyMap());
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
-        MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, similarityService,
+        MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry(), similarityService,
             new IndicesModule(emptyList()).getMapperRegistry(), () -> null);
         XContentBuilder mappingSource = jsonBuilder().startObject().startObject("some_type")
             .startObject("properties")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
index 532be6a9b6c..c065888f911 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java
@@ -20,12 +20,14 @@ package org.elasticsearch.index.mapper;
 
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.Locale;
 
 import static org.elasticsearch.index.query.RangeQueryBuilder.GT_FIELD;
 import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD;
@@ -228,56 +230,6 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
         assertThat(e.getCause().getMessage(), anyOf(containsString("passed as String"), containsString("failed to parse date")));
     }
 
-    @Override
-    protected void doTestIncludeInAll(String type) throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("properties").startObject("field").field("type", type);
-        if (type.equals("date_range")) {
-            mapping = mapping.field("format", DATE_FORMAT);
-        }
-        mapping = mapping.endObject().endObject().endObject().endObject();
-
-        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.string()));
-
-        assertEquals(mapping.string(), mapper.mappingSource().toString());
-
-        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("field")
-            .field(getFromField(), getFrom(type))
-            .field(getToField(), getTo(type))
-            .endObject()
-            .endObject().bytes());
-
-        IndexableField[] fields = doc.rootDoc().getFields("_all");
-        assertEquals(1, fields.length);
-
-        assertThat(fields[0].stringValue(), containsString(type.equals("date_range") ? "1477872000000" : "5"));
-
-        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-            .startObject("properties").startObject("field").field("type", type);
-        if (type.equals("date_range")) {
-            mapping = mapping.field("format", DATE_FORMAT);
-        }
-        mapping = mapping.field("include_in_all", false).endObject().endObject()
-            .endObject().endObject();
-
-        mapper = parser.parse("type", new CompressedXContent(mapping.string()));
-
-        assertEquals(mapping.string(), mapper.mappingSource().toString());
-
-        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("field")
-            .field(getFromField(), getFrom(type))
-            .field(getToField(), getTo(type))
-            .endObject()
-            .endObject().bytes());
-
-        fields = doc.rootDoc().getFields("_all");
-        assertEquals(0, fields.length);
-    }
-
     @Override
     protected void doTestNullValue(String type) throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -367,4 +319,23 @@ public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, runnable);
         assertThat(e.getMessage(), containsString("should not define a dateTimeFormatter"));
     }
+
+    public void testSerializeDefaults() throws Exception {
+        for (String type : TYPES) {
+            String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", type).endObject().endObject()
+                .endObject().endObject().string();
+
+            DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+            RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
+            String got = builder.endObject().string();
+
+            // if type is date_range we check that the mapper contains the default format and locale
+            // otherwise it should not contain a locale or format
+            assertTrue(got, got.contains("\"format\":\"strict_date_optional_time||epoch_millis\"") == type.equals("date_range"));
+            assertTrue(got, got.contains("\"locale\":" + "\"" + Locale.ROOT + "\"") == type.equals("date_range"));
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
index 9010164fe03..f70db120fcf 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java
@@ -48,7 +48,7 @@ public class RangeFieldTypeTests extends FieldTypeTestCase {
     @Before
     public void setupProperties() {
         type = RandomPicks.randomFrom(random(), RangeType.values());
-        nowInMillis = randomPositiveLong();
+        nowInMillis = randomNonNegativeLong();
         if (type == RangeType.DATE) {
             addModifier(new Modifier("format", true) {
                 @Override
@@ -74,8 +74,8 @@ public class RangeFieldTypeTests extends FieldTypeTestCase {
         Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
-        QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, null, null, null,
-            () -> nowInMillis);
+        QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
+                null, null, () -> nowInMillis);
         RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type);
         ft.setName(FIELDNAME);
         ft.setIndexOptions(IndexOptions.DOCS);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
index 92e93ede9aa..9c26a9806e3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -40,7 +41,7 @@ public class RoutingFieldMapperTests extends ESSingleNodeTestCase {
             .startObject()
             .field("field", "value")
             .endObject()
-            .bytes()).routing("routing_value"));
+            .bytes(), XContentType.JSON).routing("routing_value"));
 
         assertThat(doc.rootDoc().get("_routing"), equalTo("routing_value"));
         assertThat(doc.rootDoc().get("field"), equalTo("value"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
index c172217aa6c..4163378db01 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
@@ -250,46 +250,6 @@ public class ScaledFloatFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals(0, fields.length);
     }
 
-    public void testIncludeInAll() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "scaled_float")
-                .field("scaling_factor", 10.0).endObject().endObject()
-                .endObject().endObject().string();
-
-        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", 123)
-                .endObject()
-                .bytes());
-
-        IndexableField[] fields = doc.rootDoc().getFields("_all");
-        assertEquals(1, fields.length);
-        assertEquals("123", fields[0].stringValue());
-
-        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field")
-                .field("type", "scaled_float").field("scaling_factor", 10.0)
-                .field("include_in_all", false).endObject().endObject()
-                .endObject().endObject().string();
-
-        mapper = parser.parse("type", new CompressedXContent(mapping));
-
-        assertEquals(mapping, mapper.mappingSource().toString());
-
-        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", 123)
-                .endObject()
-                .bytes());
-
-        fields = doc.rootDoc().getFields("_all");
-        assertEquals(0, fields.length);
-    }
-
     public void testNullValue() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject()
                 .startObject("type")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
index fd390c5da8a..1ba58fa2ded 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java
@@ -182,6 +182,7 @@ public class ScaledFloatFieldTypeTests extends FieldTypeTestCase {
             // single-valued
             ft.setName("scaled_float1");
             IndexNumericFieldData fielddata = (IndexNumericFieldData) ft.fielddataBuilder().build(indexSettings, ft, null, null, null);
+            assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE);
             AtomicNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
             SortedNumericDoubleValues values = leafFieldData.getDoubleValues();
             values.setDocument(0);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index adf05b9b338..5803d7d957e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -144,7 +144,8 @@ public class SourceFieldMapperTests extends ESSingleNodeTestCase {
                 .startObject("_source").field("enabled", true).endObject()
                 .endObject().endObject().string();
 
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", new CompressedXContent(mapping), defaultMapping);
+        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser()
+            .parse("my_type", new CompressedXContent(mapping), defaultMapping);
         assertThat(mapper.type(), equalTo("my_type"));
         assertThat(mapper.sourceMapper().enabled(), equalTo(true));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
index e234beb7904..4df4361db6a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -214,7 +214,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("b", fields[1].stringValue());
 
         IndexShard shard = indexService.getShard(0);
-        shard.index(new Engine.Index(new Term("_uid", "1"), doc));
+        shard.index(new Engine.Index(new Term("_uid", doc.uid() ), doc));
         shard.refresh("test");
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader();
@@ -253,7 +253,7 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals("b", fields[1].stringValue());
 
         IndexShard shard = indexService.getShard(0);
-        shard.index(new Engine.Index(new Term("_uid", "1"), doc));
+        shard.index(new Engine.Index(new Term("_uid", doc.uid()), doc));
         shard.refresh("test");
         try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
             LeafReader leaf = searcher.getDirectoryReader().leaves().get(0).reader();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
index 9ae01b51a64..67d7873b7b4 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java
@@ -19,17 +19,18 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
-import java.util.HashMap;
+import java.util.Arrays;
+import java.util.Collection;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -38,109 +39,13 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
     private static final String INDEX = "index";
     private static final String TYPE = "type";
 
-    public void testAllEnabled() throws Exception {
-        XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", "false").endObject().endObject().endObject().endObject();
-        XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", "true").endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        String errorMessage = "[_all] enabled is false now encountering true";
-        testConflict(mapping.string(), mappingUpdate.string(), errorMessage);
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created
     }
 
-    public void testAllConflicts() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_create_index.json");
-        String mappingUpdate = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json");
-        String[] errorMessage = {
-                "[_all] has different [norms] values",
-                "[_all] has different [store] values",
-                "[_all] has different [store_term_vector] values",
-                "[_all] has different [store_term_vector_offsets] values",
-                "[_all] has different [store_term_vector_positions] values",
-                "[_all] has different [store_term_vector_payloads] values",
-                "[_all] has different [analyzer]",
-                "[_all] has different [similarity]"};
-        // fielddata and search_analyzer should not report conflict
-        testConflict(mapping, mappingUpdate, errorMessage);
-    }
-
-    public void testAllDisabled() throws Exception {
-        XContentBuilder mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").field("enabled", true).endObject().endObject().endObject().endObject();
-        XContentBuilder mappingUpdate = jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        String errorMessage = "[_all] enabled is true now encountering false";
-        testConflict(mapping.string(), mappingUpdate.string(), errorMessage);
-    }
-
-    public void testAllWithDefault() throws Exception {
-        String defaultMapping = jsonBuilder().startObject().startObject("_default_")
-                .startObject("_all")
-                .field("enabled", false)
-                .endObject()
-                .endObject().endObject().string();
-        client().admin().indices().prepareCreate("index").addMapping("_default_", defaultMapping).get();
-        String docMapping = jsonBuilder().startObject()
-                .startObject("doc")
-                .endObject()
-                .endObject().string();
-        PutMappingResponse response = client().admin().indices().preparePutMapping("index").setType("doc").setSource(docMapping).get();
-        assertTrue(response.isAcknowledged());
-        String docMappingUpdate = jsonBuilder().startObject().startObject("doc")
-                .startObject("properties")
-                .startObject("text")
-                .field("type", "text")
-                .endObject()
-                .endObject()
-                .endObject()
-                .endObject().string();
-        response = client().admin().indices().preparePutMapping("index").setType("doc").setSource(docMappingUpdate).get();
-        assertTrue(response.isAcknowledged());
-        String docMappingAllExplicitEnabled = jsonBuilder().startObject()
-                .startObject("doc_all_enabled")
-                .startObject("_all")
-                .field("enabled", true)
-                .endObject()
-                .endObject()
-                .endObject().string();
-        response = client().admin().indices().preparePutMapping("index").setType("doc_all_enabled").setSource(docMappingAllExplicitEnabled).get();
-        assertTrue(response.isAcknowledged());
-
-        GetMappingsResponse mapping = client().admin().indices().prepareGetMappings("index").get();
-        HashMap props = (HashMap)mapping.getMappings().get("index").get("doc").getSourceAsMap().get("_all");
-        assertThat((Boolean)props.get("enabled"), equalTo(false));
-        props = (HashMap)mapping.getMappings().get("index").get("doc").getSourceAsMap().get("properties");
-        assertNotNull(props);
-        assertNotNull(props.get("text"));
-        props = (HashMap)mapping.getMappings().get("index").get("doc_all_enabled").getSourceAsMap().get("_all");
-        assertThat((Boolean)props.get("enabled"), equalTo(true));
-        props = (HashMap)mapping.getMappings().get("index").get("_default_").getSourceAsMap().get("_all");
-        assertThat((Boolean)props.get("enabled"), equalTo(false));
-
-    }
-
-    public void testDocValuesInvalidMapping() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject("mappings").startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().endObject().string();
-        try {
-            prepareCreate(INDEX).setSource(mapping).get();
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
-        }
-    }
-
-    public void testDocValuesInvalidMappingOnUpdate() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject().string();
-        prepareCreate(INDEX).addMapping(TYPE, mapping).get();
-        String mappingUpdate = jsonBuilder().startObject().startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().string();
-        GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
-        try {
-            client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mappingUpdate).get();
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getDetailedMessage(), containsString("[_all] is always tokenized and cannot have doc values"));
-        }
-        // make sure all nodes have same cluster state
-        compareMappingOnNodes(mappingsBeforeUpdateResponse);
-    }
-
-    protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException {
-        assertAcked(prepareCreate(INDEX).setSource(mapping).get());
+    protected void testConflict(String mapping, String mappingUpdate, Version idxVersion, String... errorMessages) throws InterruptedException {
+        assertAcked(prepareCreate(INDEX).setSource(mapping).setSettings("index.version.created", idxVersion.id));
         ensureGreen(INDEX);
         GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
         try {
@@ -155,6 +60,44 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
 
     }
 
+    public void testUpdatingAllSettingsOnOlderIndex() throws Exception {
+        XContentBuilder mapping = jsonBuilder()
+                .startObject()
+                .startObject("mappings")
+                .startObject(TYPE)
+                .startObject("_all").field("enabled", "true").endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        XContentBuilder mappingUpdate = jsonBuilder()
+                .startObject()
+                .startObject("_all").field("enabled", "false").endObject()
+                .startObject("properties").startObject("text").field("type", "text").endObject()
+                .endObject()
+                .endObject();
+        String errorMessage = "[_all] enabled is true now encountering false";
+        testConflict(mapping.string(), mappingUpdate.string(), Version.V_5_0_0, errorMessage);
+    }
+
+    public void testUpdatingAllSettingsOnOlderIndexDisabledToEnabled() throws Exception {
+        XContentBuilder mapping = jsonBuilder()
+                .startObject()
+                .startObject("mappings")
+                .startObject(TYPE)
+                .startObject("_all").field("enabled", "false").endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        XContentBuilder mappingUpdate = jsonBuilder()
+                .startObject()
+                .startObject("_all").field("enabled", "true").endObject()
+                .startObject("properties").startObject("text").field("type", "text").endObject()
+                .endObject()
+                .endObject();
+        String errorMessage = "[_all] enabled is false now encountering true";
+        testConflict(mapping.string(), mappingUpdate.string(), Version.V_5_0_0, errorMessage);
+    }
+
     private void compareMappingOnNodes(GetMappingsResponse previousMapping) {
         // make sure all nodes have same cluster state
         for (Client client : cluster().getClients()) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
index 7aec1ecd0bb..9321801d1f4 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java
@@ -19,17 +19,11 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -37,9 +31,7 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.LinkedHashMap;
 
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -50,38 +42,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    public void testAllEnabledAfterDisabled() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
-    }
-
-    public void testAllDisabledAfterEnabled() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
-    }
-
-    public void testAllDisabledAfterDefaultEnabled() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("some_text").field("type", "text").endObject().endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
-    }
-
-    public void testAllEnabledAfterEnabled() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject();
-        testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
-    }
-
-    public void testAllDisabledAfterDisabled() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
-        XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject();
-        testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
-    }
-
     private void testNoConflictWhileMergingAndMappingChanged(XContentBuilder mapping, XContentBuilder mappingUpdate, XContentBuilder expectedMapping) throws IOException {
         IndexService indexService = createIndex("test", Settings.builder().build(), "type", mapping);
         // simulate like in MetaDataMappingService#putMapping
@@ -93,13 +53,29 @@ public class UpdateMappingTests extends ESSingleNodeTestCase {
 
     public void testConflictFieldsMapping(String fieldName) throws Exception {
         //test store, ... all the parameters that are not to be changed just like in other fields
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject(fieldName).field("enabled", true).field("store", "no").endObject()
-                .endObject().endObject();
-        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject(fieldName).field("enabled", true).field("store", "yes").endObject()
-                .startObject("properties").startObject("text").field("type", "text").endObject().endObject()
-                .endObject().endObject();
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject(fieldName)
+                        .field("enabled", true)
+                            .field("store", false)
+                        .endObject()
+                    .endObject()
+            .endObject();
+        XContentBuilder mappingUpdate = XContentFactory.jsonBuilder()
+            .startObject()
+                .startObject("type")
+                    .startObject(fieldName)
+                        .field("enabled", true)
+                        .field("store", true)
+                    .endObject()
+                    .startObject("properties")
+                        .startObject("text")
+                            .field("type", "text")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject();
         testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 13854ffc1e2..339c9a9136e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -24,13 +24,11 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
@@ -59,7 +57,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
             query.disableCoord(randomBoolean());
         }
         if (randomBoolean()) {
-            query.minimumNumberShouldMatch(randomMinimumShouldMatch());
+            query.minimumShouldMatch(randomMinimumShouldMatch());
         }
         int mustClauses = randomIntBetween(0, 3);
         for (int i = 0; i < mustClauses; i++) {
@@ -245,14 +243,14 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
             boolQuery()
                 .should(termQuery("foo", "bar"))
                 .should(termQuery("foo2", "bar2"))
-                .minimumNumberShouldMatch("3")).toQuery(createShardContext());
+                .minimumShouldMatch("3")).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
 
         bq = (BooleanQuery) parseQuery(
             boolQuery()
                 .should(termQuery("foo", "bar"))
                 .should(termQuery("foo2", "bar2"))
-                .minimumNumberShouldMatch(3)).toQuery(createShardContext());
+                .minimumShouldMatch(3)).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
     }
 
@@ -261,7 +259,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
                 boolQuery()
                         .should(termQuery("foo", "bar"))
                         .should(termQuery("foo2", "bar2"))
-                        .minimumNumberShouldMatch("3")
+                        .minimumShouldMatch("3")
                         .disableCoord(true)).toQuery(createShardContext());
         assertEquals(3, bq.getMinimumNumberShouldMatch());
     }
@@ -333,7 +331,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
     public void testUnknownQueryName() throws IOException {
         String query = "{\"bool\" : {\"must\" : { \"unknown_query\" : { } } } }";
 
-        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query));
         assertEquals("no [query] registered for [unknown_query]", ex.getMessage());
     }
 
@@ -357,7 +355,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
                 "    }\n" +
                 "  }\n" +
                 "}";
-        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query));
         assertEquals("[match] malformed query, expected [END_OBJECT] but found [FIELD_NAME]", ex.getMessage());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index 49266ebe9fd..b2d1b957bef 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -19,13 +19,8 @@
 
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -40,7 +35,6 @@ import java.io.IOException;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.Matchers.equalTo;
 
 public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBoundingBoxQueryBuilder> {
     /** Randomly generate either NaN or one of the two infinity values. */
@@ -118,7 +112,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
 
     public void testExceptionOnMissingTypes() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length == 0);
-        QueryShardException e = expectThrows(QueryShardException.class, () -> super.testToQuery());
+        QueryShardException e = expectThrows(QueryShardException.class, super::testToQuery);
         assertEquals("failed to find geo_point field [mapped_geo_point]", e.getMessage());
     }
 
@@ -412,7 +406,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
         assertEquals(json, GeoExecType.MEMORY, parsed.type());
     }
 
-    public void testFromJsonCoerceFails() throws IOException {
+    public void testFromJsonCoerceIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_bounding_box\" : {\n" +
@@ -426,11 +420,12 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
-    public void testFromJsonIgnoreMalformedFails() throws IOException {
+    public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_bounding_box\" : {\n" +
@@ -444,8 +439,8 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index 6c92fde6843..529a6de50a6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -303,7 +303,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
@@ -316,53 +316,54 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         assertEquals(json, 12000.0, parsed.distance(), 0.0001);
     }
 
-    public void testOptimizeBboxFails() throws IOException {
+    public void testOptimizeBboxIsDeprecated() throws IOException {
         String json =
             "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"optimize_bbox\" : \"memory\",\n" +
                 "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [optimize_bbox] used, replaced by [no replacement: " +
+                "`optimize_bbox` is no longer supported due to recent improvements]");
     }
 
-    public void testFromCoerceFails() throws IOException {
+    public void testFromCoerceIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"coerce\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
-    public void testFromJsonIgnoreMalformedFails() throws IOException {
+    public void testFromJsonIgnoreMalformedIsDeprecated() throws IOException {
         String json =
                 "{\n" +
                 "  \"geo_distance\" : {\n" +
                 "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
                 "    \"distance\" : 12000.0,\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"ignore_malformed\" : true,\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index b77ff3bbdef..6df61d8cb45 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -139,8 +139,8 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         builder.field("normalize", true); // deprecated
         builder.endObject();
         builder.endObject();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(builder.string()));
-        assertEquals("Deprecated field [normalize] used, replaced by [use validation_method instead]", e.getMessage());
+        parseQuery(builder.string());
+        assertWarnings("Deprecated field [normalize] used, replaced by [validation_method]");
     }
 
     public void testParsingAndToQueryParsingExceptions() throws IOException {
@@ -265,9 +265,8 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-
+        parseQuery(json);
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testFromJsonCoerceDeprecated() throws IOException {
@@ -282,8 +281,8 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
                 "}";
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        parseQuery(json);
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 0caf112bd26..032ebc392cb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -31,7 +31,9 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.MapperService;
@@ -104,13 +106,13 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
         HasChildQueryBuilder hqb = new HasChildQueryBuilder(CHILD_TYPE, innerQueryBuilder,
                 RandomPicks.randomFrom(random(), ScoreMode.values()));
         hqb.minMaxChildren(min, max);
+        hqb.ignoreUnmapped(randomBoolean());
         if (randomBoolean()) {
             hqb.innerHit(new InnerHitBuilder()
                     .setName(randomAsciiOfLengthBetween(1, 10))
                     .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)));
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)), hqb.ignoreUnmapped());
         }
-        hqb.ignoreUnmapped(randomBoolean());
         return hqb;
     }
 
@@ -189,6 +191,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
                 "    \"_name\" : \"WNzYMJKRwePuRBh\",\n" +
                 "    \"inner_hits\" : {\n" +
                 "      \"name\" : \"inner_hits_name\",\n" +
+                "      \"ignore_unmapped\" : false,\n" +
                 "      \"from\" : 0,\n" +
                 "      \"size\" : 100,\n" +
                 "      \"version\" : false,\n" +
@@ -211,7 +214,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
         assertEquals(query, queryBuilder.childType(), "child");
         assertEquals(query, queryBuilder.scoreMode(), ScoreMode.Avg);
         assertNotNull(query, queryBuilder.innerHit());
-        InnerHitBuilder expected = new InnerHitBuilder(new InnerHitBuilder(), queryBuilder.query(), "child")
+        InnerHitBuilder expected = new InnerHitBuilder(new InnerHitBuilder(), queryBuilder.query(), "child", false)
                 .setName("inner_hits_name")
                 .setSize(100)
                 .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC));
@@ -272,7 +275,9 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
         QueryShardContext shardContext = createShardContext();
         HasChildQueryBuilder hasChildQueryBuilder = QueryBuilders.hasChildQuery(CHILD_TYPE, new TermQueryBuilder("custom_string", "value"), ScoreMode.None);
         HasChildQueryBuilder.LateParsingQuery query = (HasChildQueryBuilder.LateParsingQuery) hasChildQueryBuilder.toQuery(shardContext);
-        Similarity expected = SimilarityService.BUILT_IN.get(similarity).apply(similarity, Settings.EMPTY).get();
+        Similarity expected = SimilarityService.BUILT_IN.get(similarity)
+            .apply(similarity, Settings.EMPTY, Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build())
+            .get();
         assertThat(((PerFieldSimilarityWrapper) query.getSimilarity()).get("custom_string"), instanceOf(expected.getClass()));
     }
 
@@ -330,4 +335,15 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
         QueryShardException e = expectThrows(QueryShardException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
         assertThat(e.getMessage(), containsString("[" + HasChildQueryBuilder.NAME + "] no mapping found for type [unmapped]"));
     }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasChildQueryBuilder queryBuilder
+            = new HasChildQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
index ca402782e8c..7e0fc30aa75 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -86,13 +85,13 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
             innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString());
         }
         HasParentQueryBuilder hqb = new HasParentQueryBuilder(PARENT_TYPE, innerQueryBuilder, randomBoolean());
+        hqb.ignoreUnmapped(randomBoolean());
         if (randomBoolean()) {
             hqb.innerHit(new InnerHitBuilder()
                     .setName(randomAsciiOfLengthBetween(1, 10))
                     .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)));
+                    .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)), hqb.ignoreUnmapped());
         }
-        hqb.ignoreUnmapped(randomBoolean());
         return hqb;
     }
 
@@ -149,12 +148,9 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         builder.field("type", "foo"); // deprecated
         builder.endObject();
         builder.endObject();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(builder.string()));
-        assertEquals("Deprecated field [type] used, expected [parent_type] instead", e.getMessage());
-
-        HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string(), ParseFieldMatcher.EMPTY);
+        HasParentQueryBuilder queryBuilder = (HasParentQueryBuilder) parseQuery(builder.string());
         assertEquals("foo", queryBuilder.type());
-        checkWarningHeaders("Deprecated field [type] used, expected [parent_type] instead");
+        assertWarnings("Deprecated field [type] used, expected [parent_type] instead");
     }
 
     public void testToQueryInnerQueryType() throws IOException {
@@ -217,4 +213,15 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         assertThat(e.getMessage(),
                     containsString("[" + HasParentQueryBuilder.NAME + "] query configured 'parent_type' [unmapped] is not a valid type"));
     }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final HasParentQueryBuilder queryBuilder =
+            new HasParentQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), false);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
index 5913a038661..ccac82362a5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
@@ -24,7 +24,6 @@ import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
@@ -161,14 +160,11 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
                 "    }\n" +
                 "}";
 
-        IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(contentString, ParseFieldMatcher.EMPTY);
+        IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(contentString);
         assertEquals(testQuery, parsed);
 
-        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(contentString));
-        checkWarningHeaders("Deprecated field [_type] used, expected [type] instead");
-        assertEquals("Deprecated field [_type] used, expected [type] instead", e.getMessage());
-        assertEquals(3, e.getLineNumber());
-        assertEquals(19, e.getColumnNumber());
+        parseQuery(contentString);
+        assertWarnings("Deprecated field [_type] used, expected [type] instead");
 
         //array of types can also be called types rather than type
         final String contentString2 = "{\n" +
@@ -177,13 +173,10 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
                 "        \"values\" : [ ]\n" +
                 "    }\n" +
                 "}";
-        parsed = (IdsQueryBuilder) parseQuery(contentString2, ParseFieldMatcher.EMPTY);
+        parsed = (IdsQueryBuilder) parseQuery(contentString2);
         assertEquals(testQuery, parsed);
 
-        e = expectThrows(ParsingException.class, () -> parseQuery(contentString2));
-        checkWarningHeaders("Deprecated field [types] used, expected [type] instead");
-        assertEquals("Deprecated field [types] used, expected [type] instead", e.getMessage());
-        assertEquals(3, e.getLineNumber());
-        assertEquals(19, e.getColumnNumber());
+        parseQuery(contentString2);
+        assertWarnings("Deprecated field [types] used, expected [type] instead");
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 9c021e249a8..9774af189da 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -19,22 +19,25 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.ShardSearchLocalRequest;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
@@ -57,24 +60,31 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class InnerHitBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 
     public void testSerialization() throws Exception {
@@ -87,6 +97,30 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
     }
 
+    /**
+     * Test that if we serialize and deserialize an object, further
+     * serialization leads to identical bytes representation.
+     *
+     * This is necessary to ensure because we use the serialized BytesReference
+     * of this builder as part of the cacheKey in
+     * {@link ShardSearchLocalRequest} (via
+     * {@link SearchSourceBuilder#collapse(org.elasticsearch.search.collapse.CollapseBuilder)})
+     */
+    public void testSerializationOrder() throws Exception {
+        for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
+            InnerHitBuilder original = randomInnerHits();
+            InnerHitBuilder deserialized = serializedCopy(original);
+            assertEquals(deserialized, original);
+            assertEquals(deserialized.hashCode(), original.hashCode());
+            assertNotSame(deserialized, original);
+            BytesStreamOutput out1 = new BytesStreamOutput();
+            BytesStreamOutput out2 = new BytesStreamOutput();
+            original.writeTo(out1);
+            deserialized.writeTo(out2);
+            assertEquals(out1.bytes(), out2.bytes());
+        }
+    }
+
     public void testFromAndToXContent() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             InnerHitBuilder innerHit = randomInnerHits(true, false);
@@ -98,7 +132,7 @@ public class InnerHitBuilderTests extends ESTestCase {
             }
 
             XContentParser parser = createParser(shuffled);
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+            QueryParseContext context = new QueryParseContext(parser);
             InnerHitBuilder secondInnerHits = InnerHitBuilder.fromXContent(context);
             assertThat(innerHit, not(sameInstance(secondInnerHits)));
             assertThat(innerHit, equalTo(secondInnerHits));
@@ -106,7 +140,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
     }
 
-    public void testEqualsAndHashcode() throws IOException {
+    public void testEqualsAndHashcode() {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             checkEqualsAndHashCode(randomInnerHits(), InnerHitBuilderTests::serializedCopy, InnerHitBuilderTests::mutate);
         }
@@ -115,7 +149,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsNestedQuery() throws Exception {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
-        nestedQueryBuilder.innerHit(leafInnerHits);
+        nestedQueryBuilder.innerHit(leafInnerHits, false);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         nestedQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
         assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
@@ -124,7 +158,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsHasChildQuery() throws Exception {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
+                .innerHit(leafInnerHits, false);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         hasChildQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
         assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
@@ -133,7 +167,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsHasParentQuery() throws Exception {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false)
-                .innerHit(leafInnerHits);
+                .innerHit(leafInnerHits, false);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         hasParentQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
         assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
@@ -142,7 +176,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
+                .innerHit(leafInnerHits, false);
         BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         boolQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
@@ -152,7 +186,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
+                .innerHit(leafInnerHits, false);
         ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
@@ -162,10 +196,10 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() {
         InnerHitBuilder leafInnerHits1 = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits1);
+                .innerHit(leafInnerHits1, false);
         InnerHitBuilder leafInnerHits2 = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits2);
+                .innerHit(leafInnerHits2, false);
         BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders);
@@ -176,13 +210,68 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() {
         InnerHitBuilder leafInnerHits = randomInnerHits();
         NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None)
-                .innerHit(leafInnerHits);
+                .innerHit(leafInnerHits, false);
         FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
         ((AbstractQueryBuilder<?>) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders);
         assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
     }
 
+    public void testBuild_ingoreUnmappedNestQuery() throws Exception {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.getObjectMapper("path")).thenReturn(null);
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
+        query1.innerHit(leafInnerHits, false);
+        expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext()));
+
+        NestedQueryBuilder query2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
+        query2.innerHit(leafInnerHits, true);
+        InnerHitsContext innerHitsContext = new InnerHitsContext();
+        query2.innerHit().build(searchContext, innerHitsContext);
+        assertThat(innerHitsContext.getInnerHits().size(), equalTo(0));
+    }
+
+    public void testBuild_ignoreUnmappedHasChildQuery() throws Exception {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.documentMapper("type")).thenReturn(null);
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        HasChildQueryBuilder query1 = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None)
+                .innerHit(leafInnerHits, false);
+        expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext()));
+
+        HasChildQueryBuilder query2 = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None)
+                .innerHit(leafInnerHits, true);
+        InnerHitsContext innerHitsContext = new InnerHitsContext();
+        query2.innerHit().build(searchContext, innerHitsContext);
+        assertThat(innerHitsContext.getInnerHits().size(), equalTo(0));
+    }
+
+    public void testBuild_ingoreUnmappedHasParentQuery() throws Exception {
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        when(queryShardContext.documentMapper("type")).thenReturn(null);
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+
+        InnerHitBuilder leafInnerHits = randomInnerHits();
+        HasParentQueryBuilder query1 = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false)
+                .innerHit(leafInnerHits, false);
+        expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext()));
+
+        HasParentQueryBuilder query2 = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false)
+                .innerHit(leafInnerHits, true);
+        InnerHitsContext innerHitsContext = new InnerHitsContext();
+        query2.innerHit().build(searchContext, innerHitsContext);
+        assertThat(innerHitsContext.getInnerHits().size(), equalTo(0));
+    }
+
+
     public static InnerHitBuilder randomInnerHits() {
         return randomInnerHits(true, true);
     }
@@ -231,9 +320,9 @@ public class InnerHitBuilderTests extends ESTestCase {
         if (includeQueryTypeOrPath) {
             QueryBuilder query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
             if (randomBoolean()) {
-                return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query);
+                return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query, randomBoolean());
             } else {
-                return new InnerHitBuilder(innerHits, query, randomAsciiOfLength(8));
+                return new InnerHitBuilder(innerHits, query, randomAsciiOfLength(8), randomBoolean());
             }
         } else {
             return innerHits;
@@ -243,8 +332,8 @@ public class InnerHitBuilderTests extends ESTestCase {
     public void testCopyConstructor() throws Exception {
         InnerHitBuilder original = randomInnerHits();
         InnerHitBuilder copy = original.getNestedPath() != null ?
-                new InnerHitBuilder(original, original.getNestedPath(), original.getQuery()) :
-                new InnerHitBuilder(original, original.getQuery(), original.getParentChildType());
+                new InnerHitBuilder(original, original.getNestedPath(), original.getQuery(), original.isIgnoreUnmapped()) :
+                new InnerHitBuilder(original, original.getQuery(), original.getParentChildType(), original.isIgnoreUnmapped());
         assertThat(copy, equalTo(original));
         copy = mutate(copy);
         assertThat(copy, not(equalTo(original)));
@@ -326,7 +415,8 @@ public class InnerHitBuilderTests extends ESTestCase {
                 randomMap.put(String.valueOf(i), randomAsciiOfLength(16));
             }
         }
-        Script script = new Script(randomScriptType, randomAsciiOfLengthBetween(1, 4), randomAsciiOfLength(128), randomMap);
+        Script script = new Script(randomScriptType, randomScriptType == ScriptType.STORED ? null : randomAsciiOfLengthBetween(1, 4),
+            randomAsciiOfLength(128), randomMap);
         return new SearchSourceBuilder.ScriptField(randomAsciiOfLengthBetween(1, 32), script, randomBoolean());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index a32eafd850c..58212d161ed 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -32,7 +32,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
@@ -311,20 +310,15 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
                 "    }\n" +
                 "  }\n" +
                 "}";
-        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, qb);
 
         assertEquals(json, expectedQB, qb);
 
         assertSerialization(qb);
 
-        checkWarningHeaders("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
                 "Deprecated field [slop] used, replaced by [match_phrase query]");
-
-        // Now check with strict parsing an exception is thrown
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json, ParseFieldMatcher.STRICT));
-        assertThat(e.getMessage(),
-                containsString("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"));
     }
 
     public void testLegacyMatchPhraseQuery() throws IOException {
@@ -347,20 +341,13 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
                 "    }\n" +
                 "  }\n" +
                 "}";
-        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        MatchQueryBuilder qb = (MatchQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, qb);
 
         assertEquals(json, expectedQB, qb);
-
         assertSerialization(qb);
-
-        checkWarningHeaders("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
+        assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]",
                 "Deprecated field [slop] used, replaced by [match_phrase query]");
-
-        // Now check with strict parsing an exception is thrown
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json, ParseFieldMatcher.STRICT));
-        assertThat(e.getMessage(),
-                containsString("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]"));
     }
 
     public void testFuzzinessOnNonStringField() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java b/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
index fc76a0d464c..f4a0600bd94 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MockRepeatAnalyzer.java
@@ -35,7 +35,7 @@ public class MockRepeatAnalyzer extends Analyzer {
         PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
         String repeat;
 
-        public MockRepeatFilter(TokenStream input) {
+        MockRepeatFilter(TokenStream input) {
             super(input);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index 7560afb928c..ee1265eca65 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -26,18 +26,20 @@ import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.termvectors.MultiTermVectorsItemResponse;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.search.MoreLikeThisQuery;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
@@ -47,6 +49,7 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -298,10 +301,30 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
         Item expectedItem = generateRandomItem();
         String json = expectedItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string();
         XContentParser parser = createParser(JsonXContent.jsonXContent, json);
-        Item newItem = Item.parse(parser, ParseFieldMatcher.STRICT, new Item());
+        Item newItem = Item.parse(parser, new Item());
         assertEquals(expectedItem, newItem);
     }
 
+    public void testItemSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AQVpbmRleAEEdHlwZQEODXsiZm9vIjoiYmFyIn0A/wD//////////QAAAAAAAAAA");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            Item item = new Item(in);
+            assertEquals(XContentType.JSON, item.xContentType());
+            assertEquals("{\"foo\":\"bar\"}", item.doc().utf8ToString());
+            assertEquals("index", item.index());
+            assertEquals("type", item.type());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                item.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
     @Override
     protected boolean isCachable(MoreLikeThisQueryBuilder queryBuilder) {
         return queryBuilder.likeItems().length == 0; // items are always fetched
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index 988f6f5c4ba..6fdac686287 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
@@ -73,13 +72,13 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
         }
         NestedQueryBuilder nqb = new NestedQueryBuilder("nested1", innerQueryBuilder,
                 RandomPicks.randomFrom(random(), ScoreMode.values()));
+        nqb.ignoreUnmapped(randomBoolean());
         if (randomBoolean()) {
             nqb.innerHit(new InnerHitBuilder()
                     .setName(randomAsciiOfLengthBetween(1, 10))
                     .setSize(randomIntBetween(0, 100))
-                    .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC)));
+                    .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC)), nqb.ignoreUnmapped());
         }
-        nqb.ignoreUnmapped(randomBoolean());
         return nqb;
     }
 
@@ -194,4 +193,15 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> failingQueryBuilder.toQuery(createShardContext()));
         assertThat(e.getMessage(), containsString("[" + NestedQueryBuilder.NAME + "] failed to find nested object under path [unmapped]"));
     }
+
+    public void testIgnoreUnmappedWithRewrite() throws IOException {
+        // WrapperQueryBuilder makes sure we always rewrite
+        final NestedQueryBuilder queryBuilder =
+            new NestedQueryBuilder("unmapped", new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()), ScoreMode.None);
+        queryBuilder.ignoreUnmapped(true);
+        QueryShardContext queryShardContext = createShardContext();
+        Query query = queryBuilder.rewrite(queryShardContext).toQuery(queryShardContext);
+        assertThat(query, notNullValue());
+        assertThat(query, instanceOf(MatchNoDocsQuery.class));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
index da76c2796f6..3f8837fe745 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.index.query;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
@@ -38,12 +38,16 @@ import java.io.IOException;
 import static java.util.Collections.emptyList;
 
 public class QueryParseContextTests extends ESTestCase {
-
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, false, emptyList()).getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents());
+    }
+
+    @AfterClass
+    public static void cleanup() {
+        xContentRegistry = null;
     }
 
     private ThreadContext threadContext;
@@ -64,7 +68,7 @@ public class QueryParseContextTests extends ESTestCase {
         QueryBuilder query = new MatchQueryBuilder("foo", "bar");
         String requestBody = "{ \"query\" : " + query.toString() + "}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             QueryBuilder actual = context.parseTopLevelQueryBuilder();
             assertEquals(query, actual);
         }
@@ -73,7 +77,7 @@ public class QueryParseContextTests extends ESTestCase {
     public void testParseTopLevelBuilderEmptyObject() throws IOException {
         String requestBody = "{}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             QueryBuilder query = context.parseTopLevelQueryBuilder();
             assertNull(query);
         }
@@ -82,7 +86,7 @@ public class QueryParseContextTests extends ESTestCase {
     public void testParseTopLevelBuilderUnknownParameter() throws IOException {
         String requestBody = "{ \"foo\" : \"bar\"}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, requestBody)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseTopLevelQueryBuilder());
             assertEquals("request does not support [foo]", exception.getMessage());
         }
@@ -92,7 +96,7 @@ public class QueryParseContextTests extends ESTestCase {
         QueryBuilder query = new MatchQueryBuilder("foo", "bar");
         String source = query.toString();
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             QueryBuilder actual = context.parseInnerQueryBuilder();
             assertEquals(query, actual);
         }
@@ -103,30 +107,35 @@ public class QueryParseContextTests extends ESTestCase {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
             parser.nextToken();
             parser.nextToken(); // don't start with START_OBJECT to provoke exception
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("[_na] query malformed, must start with start_object", exception.getMessage());
         }
 
         source = "{}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+            QueryParseContext context = new QueryParseContext(parser);
             IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("query malformed, empty clause found at [1:2]", exception.getMessage());
         }
 
         source = "{ \"foo\" : \"bar\" }";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("[foo] query malformed, no start_object after query name", exception.getMessage());
         }
 
         source = "{ \"foo\" : {} }";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) {
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             ParsingException exception = expectThrows(ParsingException.class, () ->  context.parseInnerQueryBuilder());
             assertEquals("no [query] registered for [foo]", exception.getMessage());
         }
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index 73449dbf5d9..b83c1b2897a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -46,9 +46,10 @@ public class QueryShardContextTests extends ESTestCase {
         IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY);
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
-        final long nowInMillis = randomPositiveLong();
+        final long nowInMillis = randomNonNegativeLong();
         QueryShardContext context = new QueryShardContext(
-            0, indexSettings, null, null, mapperService, null, null, null, null, null, () -> nowInMillis);
+            0, indexSettings, null, null, mapperService, null, null, xContentRegistry(), null, null,
+            () -> nowInMillis);
 
         context.setAllowUnmappedFields(false);
         MappedFieldType fieldType = new TextFieldMapper.TextFieldType();
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index f9eca12d4bd..33cae2dc35c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
@@ -27,6 +28,7 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.GraphQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
@@ -43,6 +45,8 @@ import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
@@ -114,9 +118,6 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         if (randomBoolean()) {
             queryStringQueryBuilder.enablePositionIncrements(randomBoolean());
         }
-        if (randomBoolean()) {
-            queryStringQueryBuilder.lenient(randomBoolean());
-        }
         if (randomBoolean()) {
             queryStringQueryBuilder.escape(randomBoolean());
         }
@@ -376,6 +377,121 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         }
     }
 
+    public void testToQueryWithGraph() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        for (Operator op : Operator.values()) {
+            BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
+            MapperQueryParser queryParser = new MapperQueryParser(createShardContext());
+            QueryParserSettings settings = new QueryParserSettings("");
+            settings.defaultField(STRING_FIELD_NAME);
+            settings.fieldsAndWeights(Collections.emptyMap());
+            settings.fuzziness(Fuzziness.AUTO);
+            settings.analyzeWildcard(true);
+            settings.rewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
+            settings.defaultOperator(op.toQueryParserOperator());
+            settings.forceAnalyzer(new MockSynonymAnalyzer());
+            settings.forceQuoteAnalyzer(new MockSynonymAnalyzer());
+            queryParser.reset(settings);
+
+            // simple multi-term
+            Query query = queryParser.parse("guinea pig");
+            Query expectedQuery = new GraphQuery(
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), defaultOp))
+                    .build(),
+                new TermQuery(new Term(STRING_FIELD_NAME, "cavy"))
+            );
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // simple with additional tokens
+            query = queryParser.parse("that guinea pig smells");
+            expectedQuery = new GraphQuery(
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "that")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), defaultOp))
+                    .build(),
+                new BooleanQuery.Builder()
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "that")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "cavy")), defaultOp))
+                    .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), defaultOp))
+                    .build()
+            );
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // complex
+            query = queryParser.parse("+that -(guinea pig) +smells");
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "that")), BooleanClause.Occur.MUST))
+                .add(new BooleanClause(new GraphQuery(
+                    new BooleanQuery.Builder()
+                        .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), defaultOp))
+                        .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), defaultOp))
+                        .build(),
+                    new TermQuery(new Term(STRING_FIELD_NAME, "cavy"))
+                ), BooleanClause.Occur.MUST_NOT))
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), BooleanClause.Occur.MUST))
+                .build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // no paren should cause guinea and pig to be treated as separate tokens
+            query = queryParser.parse("+that -guinea pig +smells");
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "that")), BooleanClause.Occur.MUST))
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "guinea")), BooleanClause.Occur.MUST_NOT))
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "pig")), defaultOp))
+                .add(new BooleanClause(new TermQuery(new Term(STRING_FIELD_NAME, "smells")), BooleanClause.Occur.MUST))
+                .build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // phrase
+            query = queryParser.parse("\"that guinea pig smells\"");
+            expectedQuery = new BooleanQuery.Builder()
+                .setDisableCoord(true)
+                .add(new BooleanClause(new GraphQuery(
+                    new PhraseQuery.Builder()
+                        .add(new Term(STRING_FIELD_NAME, "that"))
+                        .add(new Term(STRING_FIELD_NAME, "guinea"))
+                        .add(new Term(STRING_FIELD_NAME, "pig"))
+                        .add(new Term(STRING_FIELD_NAME, "smells"))
+                        .build(),
+                    new PhraseQuery.Builder()
+                        .add(new Term(STRING_FIELD_NAME, "that"))
+                        .add(new Term(STRING_FIELD_NAME, "cavy"))
+                        .add(new Term(STRING_FIELD_NAME, "smells"))
+                        .build()
+                ), BooleanClause.Occur.SHOULD)).build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+
+            // phrase with slop
+            query = queryParser.parse("\"that guinea pig smells\"~2");
+            expectedQuery = new BooleanQuery.Builder()
+                .setDisableCoord(true)
+                .add(new BooleanClause(new GraphQuery(
+                    new PhraseQuery.Builder()
+                        .add(new Term(STRING_FIELD_NAME, "that"))
+                        .add(new Term(STRING_FIELD_NAME, "guinea"))
+                        .add(new Term(STRING_FIELD_NAME, "pig"))
+                        .add(new Term(STRING_FIELD_NAME, "smells"))
+                        .setSlop(2)
+                        .build(),
+                    new PhraseQuery.Builder()
+                        .add(new Term(STRING_FIELD_NAME, "that"))
+                        .add(new Term(STRING_FIELD_NAME, "cavy"))
+                        .add(new Term(STRING_FIELD_NAME, "smells"))
+                        .setSlop(2)
+                        .build()
+                ), BooleanClause.Occur.SHOULD)).build();
+
+            assertThat(query, Matchers.equalTo(expectedQuery));
+        }
+    }
+
     public void testToQueryRegExpQuery() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
         Query query = queryStringQuery("/foo*bar/").defaultField(STRING_FIELD_NAME)
@@ -396,6 +512,29 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         assertThat(e.getMessage(), containsString("would result in more than 10000 states"));
     }
 
+    /**
+     * Validates that {@code max_determinized_states} can be parsed and lowers the allowed number of determinized states.
+     */
+    public void testToQueryRegExpQueryMaxDeterminizedStatesParsing() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject(); {
+            builder.startObject("query_string"); {
+                builder.field("query", "/[ac]*a[ac]{1,10}/");
+                builder.field("default_field", STRING_FIELD_NAME);
+                builder.field("max_determinized_states", 10);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+
+        QueryBuilder queryBuilder = new QueryParseContext(createParser(builder)).parseInnerQueryBuilder();
+        TooComplexToDeterminizeException e = expectThrows(TooComplexToDeterminizeException.class,
+                () -> queryBuilder.toQuery(createShardContext()));
+        assertThat(e.getMessage(), containsString("Determinizing [ac]*"));
+        assertThat(e.getMessage(), containsString("would result in more than 10 states"));
+    }
+
     public void testToQueryFuzzyQueryAutoFuzziness() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
 
@@ -669,7 +808,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
                 "    \"tie_breaker\" : 0.0,\n" +
                 "    \"default_operator\" : \"or\",\n" +
                 "    \"auto_generate_phrase_queries\" : false,\n" +
-                "    \"max_determined_states\" : 10000,\n" +
+                "    \"max_determinized_states\" : 10000,\n" +
                 "    \"enable_position_increment\" : true,\n" +
                 "    \"fuzziness\" : \"AUTO\",\n" +
                 "    \"fuzzy_prefix_length\" : 0,\n" +
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index 09627d00d76..2640998f3c4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.index.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.lucene.BytesRefs;
@@ -39,7 +39,6 @@ import org.elasticsearch.test.AbstractQueryTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.chrono.ISOChronology;
-import org.locationtech.spatial4j.shape.SpatialRelation;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -388,14 +387,8 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                 "  }\n" +
                 "}";
 
-        // non strict parsing should accept "_name" on top level
-        assertNotNull(parseQuery(json, ParseFieldMatcher.EMPTY));
-
-        // with strict parsing, ParseField will throw exception
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> parseQuery(deprecatedJson, ParseFieldMatcher.STRICT));
-        assertEquals("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]",
-                e.getMessage());
+        assertNotNull(parseQuery(deprecatedJson));
+        assertWarnings("Deprecated field [_name] used, replaced by [query name is not supported in short version of range query]");
     }
 
     public void testRewriteDateToMatchAll() throws IOException {
@@ -419,6 +412,31 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         assertThat(rewrittenRange.to(), equalTo(null));
     }
 
+    public void testRewriteDateToMatchAllWithTimezoneAndFormat() throws IOException {
+        String fieldName = randomAsciiOfLengthBetween(1, 20);
+        RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
+            @Override
+            protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteContext) throws IOException {
+                return Relation.WITHIN;
+            }
+        };
+        DateTime queryFromValue = new DateTime(2015, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2016, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        query.from(queryFromValue);
+        query.to(queryToValue);
+        query.timeZone(randomFrom(DateTimeZone.getAvailableIDs()));
+        query.format("yyyy-MM-dd");
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+        RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
+        assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
+        assertThat(rewrittenRange.from(), equalTo(null));
+        assertThat(rewrittenRange.to(), equalTo(null));
+        assertThat(rewrittenRange.timeZone(), equalTo(null));
+        assertThat(rewrittenRange.format(), equalTo(null));
+    }
+
     public void testRewriteDateToMatchNone() throws IOException {
         String fieldName = randomAsciiOfLengthBetween(1, 20);
         RangeQueryBuilder query = new RangeQueryBuilder(fieldName) {
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
index fd6f0670a57..d7ef534bcb2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java
@@ -37,7 +37,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase {
         IndexService indexService = createIndex("test");
         IndexReader reader = new MultiReader();
         QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-            null, null, null, null, reader, null);
+                null, null, xContentRegistry(), null, reader, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         assertEquals(Relation.DISJOINT, range.getRelation(context));
     }
@@ -54,7 +54,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase {
         indexService.mapperService().merge("type",
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
         QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-            null, null, null, null, null, null);
+                null, null, xContentRegistry(), null, null, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // can't make assumptions on a missing reader, so it must return INTERSECT
         assertEquals(Relation.INTERSECTS, range.getRelation(context));
@@ -73,7 +73,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase {
                 new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
         IndexReader reader = new MultiReader();
         QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, indexService.mapperService(),
-            null, null, null, null, reader, null);
+                null, null, xContentRegistry(), null, reader, null);
         RangeQueryBuilder range = new RangeQueryBuilder("foo");
         // no values -> DISJOINT
         assertEquals(Relation.DISJOINT, range.getRelation(context));
diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
index 5e971abb2bd..746caa0e805 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryParserTests.java
@@ -20,10 +20,13 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockSynonymAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.GraphQuery;
+import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
@@ -34,39 +37,17 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MockFieldMapper;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SimpleQueryParserTests extends ESTestCase {
-
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-
-    /**
-     * setup for the whole base test class
-     */
-    @BeforeClass
-    public static void init() {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        indicesQueriesRegistry = null;
-    }
-
     private static class MockSimpleQueryParser extends SimpleQueryParser {
-        public MockSimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, Settings settings) {
+        MockSimpleQueryParser(Analyzer analyzer, Map<String, Float> weights, int flags, Settings settings) {
             super(analyzer, weights, flags, settings, null);
         }
 
@@ -135,6 +116,53 @@ public class SimpleQueryParserTests extends ESTestCase {
         }
     }
 
+    public void testAnalyzerWithGraph() {
+        SimpleQueryParser.Settings settings = new SimpleQueryParser.Settings();
+        settings.analyzeWildcard(true);
+        Map<String, Float> weights = new HashMap<>();
+        weights.put("field1", 1.0f);
+        SimpleQueryParser parser = new MockSimpleQueryParser(new MockSynonymAnalyzer(), weights, -1, settings);
+
+        for (Operator op : Operator.values()) {
+            BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
+            parser.setDefaultOperator(defaultOp);
+
+            // non-phrase won't detect multi-word synonym because of whitespace splitting
+            Query query = parser.parse("guinea pig");
+
+            Query expectedQuery = new BooleanQuery.Builder()
+                .add(new BooleanClause(new TermQuery(new Term("field1", "guinea")), defaultOp))
+                .add(new BooleanClause(new TermQuery(new Term("field1", "pig")), defaultOp))
+                .build();
+            assertThat(query, equalTo(expectedQuery));
+
+            // phrase will pick it up
+            query = parser.parse("\"guinea pig\"");
+
+            expectedQuery = new GraphQuery(
+                new PhraseQuery("field1", "guinea", "pig"),
+                new TermQuery(new Term("field1", "cavy")));
+
+            assertThat(query, equalTo(expectedQuery));
+
+            // phrase with slop
+            query = parser.parse("big \"guinea pig\"~2");
+
+            expectedQuery = new BooleanQuery.Builder()
+                .add(new BooleanClause(new TermQuery(new Term("field1", "big")), defaultOp))
+                .add(new BooleanClause(new GraphQuery(
+                    new PhraseQuery.Builder()
+                        .add(new Term("field1", "guinea"))
+                        .add(new Term("field1", "pig"))
+                        .setSlop(2)
+                        .build(),
+                    new TermQuery(new Term("field1", "cavy"))), defaultOp))
+                .build();
+
+            assertThat(query, equalTo(expectedQuery));
+        }
+    }
+
     public void testQuoteFieldSuffix() {
         SimpleQueryParser.Settings sqpSettings = new SimpleQueryParser.Settings();
         sqpSettings.quoteFieldSuffix(".quote");
@@ -147,7 +175,7 @@ public class SimpleQueryParserTests extends ESTestCase {
                 .build();
         IndexMetaData indexState = IndexMetaData.builder("index").settings(indexSettings).build();
         IndexSettings settings = new IndexSettings(indexState, Settings.EMPTY);
-        QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, indicesQueriesRegistry,
+        QueryShardContext mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(),
                 null, null, System::currentTimeMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
@@ -161,7 +189,7 @@ public class SimpleQueryParserTests extends ESTestCase {
         assertEquals(new TermQuery(new Term("foo.quote", "bar")), parser.parse("\"bar\""));
 
         // Now check what happens if foo.quote does not exist
-        mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, indicesQueriesRegistry,
+        mockShardContext = new QueryShardContext(0, settings, null, null, null, null, null, xContentRegistry(),
                 null, null, System::currentTimeMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 98c6ac00344..417032493f6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -44,6 +44,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -59,9 +60,6 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
         if (randomBoolean()) {
             result.analyzeWildcard(randomBoolean());
         }
-        if (randomBoolean()) {
-            result.lenient(randomBoolean());
-        }
         if (randomBoolean()) {
             result.minimumShouldMatch(randomMinimumShouldMatch());
         }
@@ -214,9 +212,7 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
         // the remaining tests requires either a mapping that we register with types in base test setup
         if (getCurrentTypes().length > 0) {
             Query luceneQuery = queryBuilder.toQuery(shardContext);
-            assertThat(luceneQuery, instanceOf(TermQuery.class));
-            TermQuery termQuery = (TermQuery) luceneQuery;
-            assertThat(termQuery.getTerm(), equalTo(new Term(MetaData.ALL, query)));
+            assertThat(luceneQuery, instanceOf(BooleanQuery.class));
         }
     }
 
@@ -259,7 +255,7 @@ public class SimpleQueryStringBuilderTests extends AbstractQueryTestCase<SimpleQ
             if (ms.allEnabled()) {
                 assertTermQuery(query, MetaData.ALL, queryBuilder.value());
             } else {
-                assertThat(query.getClass(), equalTo(MatchNoDocsQuery.class));
+                assertThat(query.getClass(), anyOf(equalTo(BooleanQuery.class), equalTo(MatchNoDocsQuery.class)));
             }
         } else {
             fail("Encountered lucene query type we do not have a validation implementation for in our "
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
index 8afe80e5f71..b26b93cf1ab 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanQuery;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.AbstractQueryTestCase;
@@ -149,7 +148,7 @@ public class SpanNearQueryBuilderTests extends AbstractQueryTestCase<SpanNearQue
 
         final ParsingException e = expectThrows(
                 ParsingException.class,
-                () -> parseQuery(json, ParseFieldMatcher.EMPTY));
+                () -> parseQuery(json));
         assertThat(e.getMessage(), containsString("[span_near] query does not support [collect_payloads]"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 50c63e66f58..20f4475ec76 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -20,12 +20,10 @@
 package org.elasticsearch.index.query.functionscore;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -39,7 +37,6 @@ import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -617,7 +614,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
                 "  }\n" +
                 "}";
 
-        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json);
         // this should be equivalent to the same with a match_all query
         String expected =
                 "{\n" +
@@ -639,7 +636,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
                     "  }\n" +
                     "}";
 
-        FunctionScoreQueryBuilder expectedParsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        FunctionScoreQueryBuilder expectedParsed = (FunctionScoreQueryBuilder) parseQuery(json);
         assertEquals(expectedParsed, parsed);
 
         assertEquals(json, 2, parsed.filterFunctionBuilders().length);
@@ -775,13 +772,13 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
     static class RandomScoreFunctionBuilderWithFixedSeed extends RandomScoreFunctionBuilder {
         public static final String NAME = "random_with_fixed_seed";
 
-        public RandomScoreFunctionBuilderWithFixedSeed() {
+        RandomScoreFunctionBuilderWithFixedSeed() {
         }
 
         /**
          * Read from a stream.
          */
-        public RandomScoreFunctionBuilderWithFixedSeed(StreamInput in) throws IOException {
+        RandomScoreFunctionBuilderWithFixedSeed(StreamInput in) throws IOException {
             super(in);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
index 652c4b4aef3..471a02feb47 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
@@ -751,6 +751,33 @@ public class FunctionScoreTests extends ESTestCase {
         assertThat(searchResult.scoreDocs[0].score, equalTo(explanation.getValue()));
     }
 
+    public void testWeightFactorNeedsScore() {
+        for (boolean needsScore : new boolean[] {true, false}) {
+            WeightFactorFunction function = new WeightFactorFunction(10.0f, new ScoreFunction(CombineFunction.REPLACE) {
+                @Override
+                public LeafScoreFunction getLeafScoreFunction(LeafReaderContext ctx) throws IOException {
+                    return null;
+                }
+
+                @Override
+                public boolean needsScores() {
+                    return needsScore;
+                }
+
+                @Override
+                protected boolean doEquals(ScoreFunction other) {
+                    return false;
+                }
+
+                @Override
+                protected int doHashCode() {
+                    return 0;
+                }
+            });
+            assertEquals(needsScore, function.needsScores());
+        }
+    }
+
     private static class DummyScoreFunction extends ScoreFunction {
         protected DummyScoreFunction(CombineFunction scoreCombiner) {
             super(scoreCombiner);
diff --git a/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java b/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java
new file mode 100644
index 00000000000..91ac42628e6
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/refresh/RefreshStatsTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.refresh;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.test.AbstractStreamableTestCase;
+
+import java.io.IOException;
+
+public class RefreshStatsTests extends AbstractStreamableTestCase<RefreshStats> {
+    @Override
+    protected RefreshStats createTestInstance() {
+        return new RefreshStats(randomNonNegativeLong(), randomNonNegativeLong(), between(0, Integer.MAX_VALUE));
+    }
+
+    @Override
+    protected RefreshStats createBlankInstance() {
+        return new RefreshStats();
+    }
+
+    public void testPre5Dot2() throws IOException {
+        // We can drop the compatibility once the assertion just below this list fails
+        assertTrue(Version.CURRENT.minimumCompatibilityVersion().before(Version.V_5_2_0_UNRELEASED));
+
+        RefreshStats instance = createTestInstance();
+        RefreshStats copied = copyInstance(instance, Version.V_5_1_1_UNRELEASED);
+        assertEquals(instance.getTotal(), copied.getTotal());
+        assertEquals(instance.getTotalTimeInMillis(), copied.getTotalTimeInMillis());
+        assertEquals(0, copied.getListeners());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 2425552c246..b33b082f5c4 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -35,10 +35,14 @@ import org.elasticsearch.action.support.replication.TransportWriteActionTestHelp
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.Uid;
@@ -46,6 +50,7 @@ import org.elasticsearch.index.seqno.GlobalCheckpointSyncAction;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardTestCase;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
 
@@ -65,8 +70,8 @@ import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
-import static org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary;
-import static org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnReplica;
+import static org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary;
+import static org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnReplica;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -96,9 +101,9 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
     }
 
     protected class ReplicationGroup implements AutoCloseable, Iterable<IndexShard> {
-        private final IndexShard primary;
+        private IndexShard primary;
+        private IndexMetaData indexMetaData;
         private final List<IndexShard> replicas;
-        private final IndexMetaData indexMetaData;
         private final AtomicInteger replicaId = new AtomicInteger();
         private final AtomicInteger docId = new AtomicInteger();
         boolean closed = false;
@@ -116,7 +121,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
         public int indexDocs(final int numOfDoc) throws Exception {
             for (int doc = 0; doc < numOfDoc; doc++) {
                 final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet()))
-                    .source("{}");
+                    .source("{}", XContentType.JSON);
                 final IndexResponse response = index(indexRequest);
                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
             }
@@ -126,7 +131,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
 
         public int appendDocs(final int numOfDoc) throws Exception {
             for (int doc = 0; doc < numOfDoc; doc++) {
-                final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}");
+                final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON);
                 final IndexResponse response = index(indexRequest);
                 assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
             }
@@ -170,13 +175,56 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
         }
 
         public synchronized IndexShard addReplica() throws IOException {
-            final IndexShard replica = newShard(shardId, false, "s" + replicaId.incrementAndGet(), indexMetaData,
-                () -> { throw new AssertionError("replicas can't sync global checkpoint"); }, null);
+            final IndexShard replica =
+                newShard(shardId, false, "s" + replicaId.incrementAndGet(), indexMetaData, this::syncGlobalCheckpoint, null);
             replicas.add(replica);
             updateAllocationIDsOnPrimary();
             return replica;
         }
 
+        public synchronized IndexShard addReplicaWithExistingPath(final ShardPath shardPath, final String nodeId) throws IOException {
+            final ShardRouting shardRouting = TestShardRouting.newShardRouting(
+                shardId,
+                nodeId,
+                false, ShardRoutingState.INITIALIZING,
+                RecoverySource.PeerRecoverySource.INSTANCE);
+
+            final IndexShard newReplica = newShard(shardRouting, shardPath, indexMetaData, null, this::syncGlobalCheckpoint);
+            replicas.add(newReplica);
+            updateAllocationIDsOnPrimary();
+            return newReplica;
+        }
+
+        public synchronized List<IndexShard> getReplicas() {
+            return Collections.unmodifiableList(replicas);
+        }
+
+        /**
+         * promotes the specific replica as the new primary
+         */
+        public synchronized void promoteReplicaToPrimary(IndexShard replica) throws IOException {
+            final long newTerm = indexMetaData.primaryTerm(shardId.id()) + 1;
+            IndexMetaData.Builder newMetaData =
+                IndexMetaData.builder(indexMetaData).primaryTerm(shardId.id(), newTerm);
+            indexMetaData = newMetaData.build();
+            for (IndexShard shard: replicas) {
+                shard.updatePrimaryTerm(newTerm);
+            }
+            boolean found = replicas.remove(replica);
+            assert found;
+            primary = replica;
+            replica.updateRoutingEntry(replica.routingEntry().moveActiveReplicaToPrimary());
+            updateAllocationIDsOnPrimary();
+        }
+
+        synchronized boolean removeReplica(IndexShard replica) {
+            final boolean removed = replicas.remove(replica);
+            if (removed) {
+                updateAllocationIDsOnPrimary();
+            }
+            return removed;
+        }
+
         public void recoverReplica(IndexShard replica) throws IOException {
             recoverReplica(replica, (r, sourceNode) -> new RecoveryTarget(r, sourceNode, recoveryListener, version -> {}));
         }
@@ -186,8 +234,10 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
             recoverReplica(replica, targetSupplier, true);
         }
 
-        public void recoverReplica(IndexShard replica, BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
-                                   boolean markAsRecovering) throws IOException {
+        public void recoverReplica(
+            IndexShard replica,
+            BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
+            boolean markAsRecovering) throws IOException {
             ESIndexLevelReplicationTestCase.this.recoverReplica(replica, primary, targetSupplier, markAsRecovering);
             updateAllocationIDsOnPrimary();
         }
@@ -288,7 +338,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
         private final ReplicationGroup replicationGroup;
         private final String opType;
 
-        public ReplicationAction(Request request, ActionListener<Response> listener,
+        ReplicationAction(Request request, ActionListener<Response> listener,
                                  ReplicationGroup group, String opType) {
             this.request = request;
             this.listener = listener;
@@ -329,7 +379,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
 
         protected abstract PrimaryResult performOnPrimary(IndexShard primary, Request request) throws Exception;
 
-        protected abstract void performOnReplica(ReplicaRequest request, IndexShard replica);
+        protected abstract void performOnReplica(ReplicaRequest request, IndexShard replica) throws IOException;
 
         class PrimaryRef implements ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult> {
 
@@ -395,7 +445,7 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
             final ReplicaRequest replicaRequest;
             final Response finalResponse;
 
-            public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponse) {
+            PrimaryResult(ReplicaRequest replicaRequest, Response finalResponse) {
                 this.replicaRequest = replicaRequest;
                 this.finalResponse = finalResponse;
             }
@@ -419,46 +469,61 @@ public abstract class ESIndexLevelReplicationTestCase extends IndexShardTestCase
 
     class IndexingAction extends ReplicationAction<IndexRequest, IndexRequest, IndexResponse> {
 
-        public IndexingAction(IndexRequest request, ActionListener<IndexResponse> listener, ReplicationGroup replicationGroup) {
+        IndexingAction(IndexRequest request, ActionListener<IndexResponse> listener, ReplicationGroup replicationGroup) {
             super(request, listener, replicationGroup, "indexing");
             request.process(null, true, request.index());
         }
 
         @Override
         protected PrimaryResult performOnPrimary(IndexShard primary, IndexRequest request) throws Exception {
-            final Engine.IndexResult indexResult = executeIndexRequestOnPrimary(request, primary,
-                    null);
-            if (indexResult.hasFailure() == false) {
-                // update the version on request so it will happen on the replicas
-                final long version = indexResult.getVersion();
-                request.version(version);
-                request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
-                request.seqNo(indexResult.getSeqNo());
-                assert request.versionType().validateVersionForWrites(request.version());
-            }
-            request.primaryTerm(primary.getPrimaryTerm());
-            TransportWriteActionTestHelper.performPostWriteActions(primary, request, indexResult.getTranslogLocation(), logger);
-            IndexResponse response = new IndexResponse(
-                primary.shardId(),
-                request.type(),
-                request.id(),
-                indexResult.getSeqNo(),
-                indexResult.getVersion(),
-                indexResult.isCreated());
+            IndexResponse response = indexOnPrimary(request, primary);
             return new PrimaryResult(request, response);
         }
 
         @Override
-        protected void performOnReplica(IndexRequest request, IndexShard replica) {
-            final Engine.IndexResult result = executeIndexRequestOnReplica(request, replica);
-            TransportWriteActionTestHelper.performPostWriteActions(replica, request, result.getTranslogLocation(), logger);
+        protected void performOnReplica(IndexRequest request, IndexShard replica) throws IOException {
+            indexOnReplica(request, replica);
         }
     }
 
+    /**
+     * indexes the given requests on the supplied primary, modifying it for replicas
+     */
+    protected IndexResponse indexOnPrimary(IndexRequest request, IndexShard primary) throws Exception {
+        final Engine.IndexResult indexResult = executeIndexRequestOnPrimary(request, primary,
+                null);
+        if (indexResult.hasFailure() == false) {
+            // update the version on request so it will happen on the replicas
+            final long version = indexResult.getVersion();
+            request.version(version);
+            request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
+            request.setSeqNo(indexResult.getSeqNo());
+            assert request.versionType().validateVersionForWrites(request.version());
+        }
+        request.primaryTerm(primary.getPrimaryTerm());
+        TransportWriteActionTestHelper.performPostWriteActions(primary, request, indexResult.getTranslogLocation(), logger);
+        return new IndexResponse(
+            primary.shardId(),
+            request.type(),
+            request.id(),
+            indexResult.getSeqNo(),
+            indexResult.getVersion(),
+            indexResult.isCreated());
+    }
+
+    /**
+     * indexes the given requests on the supplied replica shard
+     */
+    protected void indexOnReplica(IndexRequest request, IndexShard replica) throws IOException {
+        final Engine.IndexResult result = executeIndexRequestOnReplica(request, replica);
+        TransportWriteActionTestHelper.performPostWriteActions(replica, request, result.getTranslogLocation(), logger);
+    }
+
+
     class GlobalCheckpointSync extends
         ReplicationAction<GlobalCheckpointSyncAction.PrimaryRequest, GlobalCheckpointSyncAction.ReplicaRequest, ReplicationResponse> {
 
-        public GlobalCheckpointSync(ActionListener<ReplicationResponse> listener, ReplicationGroup replicationGroup) {
+        GlobalCheckpointSync(ActionListener<ReplicationResponse> listener, ReplicationGroup replicationGroup) {
             super(new GlobalCheckpointSyncAction.PrimaryRequest(replicationGroup.getPrimary().shardId()), listener,
                 replicationGroup, "global_ckp");
         }
diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
index 41edc303f9d..39d8778c2a4 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.InternalEngine;
 import org.elasticsearch.index.engine.InternalEngineTests;
@@ -110,7 +111,7 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
     public void testInheritMaxValidAutoIDTimestampOnRecovery() throws Exception {
         try (ReplicationGroup shards = createGroup(0)) {
             shards.startAll();
-            final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}");
+            final IndexRequest indexRequest = new IndexRequest(index.getName(), "type").source("{}", XContentType.JSON);
             indexRequest.onRetry(); // force an update of the timestamp
             final IndexResponse response = shards.index(indexRequest);
             assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
diff --git a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
index 93b20633cf1..03f051c1c02 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.index.replication;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.store.Store;
@@ -27,6 +30,7 @@ import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -34,6 +38,10 @@ import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Future;
 
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
 public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestCase {
 
     public void testIndexingDuringFileRecovery() throws Exception {
@@ -57,11 +65,138 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
         }
     }
 
+    public void testRecoveryOfDisconnectedReplica() throws Exception {
+        try (ReplicationGroup shards = createGroup(1)) {
+            shards.startAll();
+            int docs = shards.indexDocs(randomInt(50));
+            shards.flush();
+            shards.getPrimary().updateGlobalCheckpointOnPrimary();
+            final IndexShard originalReplica = shards.getReplicas().get(0);
+            long replicaCommittedLocalCheckpoint = docs - 1;
+            boolean replicaHasDocsSinceLastFlushedCheckpoint = false;
+            for (int i = 0; i < randomInt(2); i++) {
+                final int indexedDocs = shards.indexDocs(randomInt(5));
+                docs += indexedDocs;
+                if (indexedDocs > 0) {
+                    replicaHasDocsSinceLastFlushedCheckpoint = true;
+                }
+
+                final boolean flush = randomBoolean();
+                if (flush) {
+                    originalReplica.flush(new FlushRequest());
+                    replicaHasDocsSinceLastFlushedCheckpoint = false;
+                    replicaCommittedLocalCheckpoint = docs - 1;
+                }
+
+                final boolean sync = randomBoolean();
+                if (sync) {
+                    shards.getPrimary().updateGlobalCheckpointOnPrimary();
+                }
+            }
+
+            shards.removeReplica(originalReplica);
+
+            final int missingOnReplica = shards.indexDocs(randomInt(5));
+            docs += missingOnReplica;
+            replicaHasDocsSinceLastFlushedCheckpoint |= missingOnReplica > 0;
+
+            if (randomBoolean()) {
+                shards.getPrimary().updateGlobalCheckpointOnPrimary();
+            }
+
+            final boolean flushPrimary = randomBoolean();
+            if (flushPrimary) {
+                shards.flush();
+            }
+
+            originalReplica.close("disconnected", false);
+            IOUtils.close(originalReplica.store());
+            final IndexShard recoveredReplica =
+                shards.addReplicaWithExistingPath(originalReplica.shardPath(), originalReplica.routingEntry().currentNodeId());
+            shards.recoverReplica(recoveredReplica);
+            if (flushPrimary && replicaHasDocsSinceLastFlushedCheckpoint) {
+                // replica has something to catch up with, but since we flushed the primary, we should fall back to full recovery
+                assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), not(empty()));
+            } else {
+                assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), empty());
+                assertThat(
+                    recoveredReplica.recoveryState().getTranslog().recoveredOperations(),
+                    equalTo(Math.toIntExact(docs - (replicaCommittedLocalCheckpoint + 1))));
+            }
+
+            docs += shards.indexDocs(randomInt(5));
+
+            shards.assertAllEqual(docs);
+        }
+    }
+
+    @TestLogging("org.elasticsearch.index.shard:TRACE,org.elasticsearch.indices.recovery:TRACE")
+    public void testRecoveryAfterPrimaryPromotion() throws Exception {
+        try (ReplicationGroup shards = createGroup(2)) {
+            shards.startAll();
+            int totalDocs = shards.indexDocs(randomInt(10));
+            int committedDocs = 0;
+            if (randomBoolean()) {
+                shards.flush();
+                committedDocs = totalDocs;
+            }
+            // we need some indexing to happen to transfer local checkpoint information to the primary
+            // so it can update the global checkpoint and communicate to replicas
+            boolean expectSeqNoRecovery = totalDocs > 0;
+
+
+            final IndexShard oldPrimary = shards.getPrimary();
+            final IndexShard newPrimary = shards.getReplicas().get(0);
+            final IndexShard replica = shards.getReplicas().get(1);
+            if (randomBoolean()) {
+                // simulate docs that were inflight when primary failed, these will be rolled back
+                final int rollbackDocs = randomIntBetween(1, 5);
+                logger.info("--> indexing {} rollback docs", rollbackDocs);
+                for (int i = 0; i < rollbackDocs; i++) {
+                    final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", "rollback_" + i).source("{}");
+                    indexOnPrimary(indexRequest, oldPrimary);
+                    indexOnReplica(indexRequest, replica);
+                }
+                if (randomBoolean()) {
+                    oldPrimary.flush(new FlushRequest(index.getName()));
+                    expectSeqNoRecovery = false;
+                }
+            }
+
+            shards.promoteReplicaToPrimary(newPrimary);
+            // index some more
+            totalDocs += shards.indexDocs(randomIntBetween(0, 5));
+
+            oldPrimary.close("demoted", false);
+            oldPrimary.store().close();
+
+            IndexShard newReplica = shards.addReplicaWithExistingPath(oldPrimary.shardPath(), oldPrimary.routingEntry().currentNodeId());
+            shards.recoverReplica(newReplica);
+
+            if (expectSeqNoRecovery) {
+                assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
+                assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(totalDocs - committedDocs));
+            } else {
+                assertThat(newReplica.recoveryState().getIndex().fileDetails(), not(empty()));
+                assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(totalDocs - committedDocs));
+            }
+
+            shards.removeReplica(replica);
+            replica.close("resync", false);
+            replica.store().close();
+            newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
+            shards.recoverReplica(newReplica);
+
+            shards.assertAllEqual(totalDocs);
+        }
+    }
+
     private static class BlockingTarget extends RecoveryTarget {
+
         private final CountDownLatch recoveryBlocked;
         private final CountDownLatch releaseRecovery;
         private final RecoveryState.Stage stageToBlock;
-        public static final EnumSet<RecoveryState.Stage> SUPPORTED_STAGES =
+        static final EnumSet<RecoveryState.Stage> SUPPORTED_STAGES =
             EnumSet.of(RecoveryState.Stage.INDEX, RecoveryState.Stage.TRANSLOG, RecoveryState.Stage.FINALIZE);
         private final Logger logger;
 
@@ -119,4 +254,5 @@ public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestC
         }
 
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java b/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java
index f6fbc3410ac..2381b8bdc38 100644
--- a/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/index/search/MatchQueryIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.search;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -66,7 +65,9 @@ public class MatchQueryIT extends ESIntegTestCase {
 
         assertAcked(builder.addMapping(INDEX, createMapping()));
         ensureGreen();
+    }
 
+    private List<IndexRequestBuilder> getDocs() {
         List<IndexRequestBuilder> builders = new ArrayList<>();
         builders.add(client().prepareIndex("test", "test", "1").setSource("field", "say wtf happened foo"));
         builders.add(client().prepareIndex("test", "test", "2").setSource("field", "bar baz what the fudge man"));
@@ -75,7 +76,7 @@ public class MatchQueryIT extends ESIntegTestCase {
         builders.add(client().prepareIndex("test", "test", "5").setSource("field", "bar two three"));
         builders.add(client().prepareIndex("test", "test", "6").setSource("field", "bar baz two three"));
 
-        indexRandom(true, false, builders);
+        return builders;
     }
 
     /**
@@ -98,6 +99,8 @@ public class MatchQueryIT extends ESIntegTestCase {
     }
 
     public void testSimpleMultiTermPhrase() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
         // first search using regular synonym field using phrase
         SearchResponse searchResponse = client().prepareSearch(INDEX)
             .setQuery(QueryBuilders.matchPhraseQuery("field", "foo two three").analyzer("lower_syns")).get();
@@ -116,13 +119,15 @@ public class MatchQueryIT extends ESIntegTestCase {
     }
 
     public void testSimpleMultiTermAnd() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
         // first search using regular synonym field using phrase
         SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "say what the fudge")
             .operator(Operator.AND).analyzer("lower_syns")).get();
 
-        // 0 = say, 1 = OR(wtf, what), 2 = the, 3 = fudge
-        // "the" and "fudge" are required here, even though they were part of the synonym which is also expanded
-        assertNoSearchHits(searchResponse);
+        // Old synonyms work fine in that case, but it is coincidental
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
 
         // same query using graph should find correct result
         searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "say what the fudge")
@@ -133,6 +138,8 @@ public class MatchQueryIT extends ESIntegTestCase {
     }
 
     public void testMinShouldMatch() throws ExecutionException, InterruptedException {
+        indexRandom(true, false, getDocs());
+
         // no min should match
         SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchQuery("field", "three what the fudge foo")
             .operator(Operator.OR).analyzer("lower_graphsyns")).get();
@@ -151,4 +158,46 @@ public class MatchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 3L);
         assertSearchHits(searchResponse, "1", "2", "6");
     }
+
+    public void testPhrasePrefix() throws ExecutionException, InterruptedException {
+        List<IndexRequestBuilder> builders = getDocs();
+        builders.add(client().prepareIndex("test", "test", "7").setSource("field", "WTFD!"));
+        builders.add(client().prepareIndex("test", "test", "8").setSource("field", "Weird Al's WHAT THE FUDGESICLE"));
+        indexRandom(true, false, builders);
+
+        SearchResponse searchResponse = client().prepareSearch(INDEX).setQuery(QueryBuilders.matchPhrasePrefixQuery("field", "wtf")
+            .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 5L);
+        assertSearchHits(searchResponse, "1", "2", "3", "7", "8");
+    }
+
+    public void testCommonTerms() throws ExecutionException, InterruptedException {
+        String route = "commonTermsTest";
+        List<IndexRequestBuilder> builders = getDocs();
+        for (IndexRequestBuilder indexRequet : builders) {
+            // route all docs to same shard for this test
+            indexRequet.setRouting(route);
+        }
+        indexRandom(true, false, builders);
+
+        // do a search with no cutoff frequency to show which docs should match
+        SearchResponse searchResponse = client().prepareSearch(INDEX)
+            .setRouting(route)
+            .setQuery(QueryBuilders.matchQuery("field", "foo three happened")
+                .operator(Operator.OR).analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 4L);
+        assertSearchHits(searchResponse, "1", "2", "5", "6");
+
+        // do same search with cutoff and see less documents match
+        // in this case, essentially everything but "happened" gets excluded
+        searchResponse = client().prepareSearch(INDEX)
+            .setRouting(route)
+            .setQuery(QueryBuilders.matchQuery("field", "foo three happened")
+                .operator(Operator.OR).analyzer("lower_graphsyns").cutoffFrequency(1f)).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
index 510371c2d51..0e12795b6d4 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -36,10 +36,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Collections;
-import java.util.HashSet;
 
 import static org.elasticsearch.mock.orig.Mockito.when;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
@@ -60,7 +57,7 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,  boundAddress -> clusterService.localNode(), null);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new ShardStateAction(Settings.EMPTY, clusterService, transportService, null, null, threadPool);
@@ -103,7 +100,8 @@ public class GlobalCheckpointSyncActionTests extends ESTestCase {
         if (randomBoolean()) {
             action.shardOperationOnPrimary(primaryRequest, indexShard);
         } else {
-            action.shardOperationOnReplica(new GlobalCheckpointSyncAction.ReplicaRequest(primaryRequest, randomPositiveLong()), indexShard);
+            action.shardOperationOnReplica(
+                new GlobalCheckpointSyncAction.ReplicaRequest(primaryRequest, randomNonNegativeLong()), indexShard);
         }
 
         verify(translog).sync();
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointTests.java b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointTests.java
index f27dfd189be..7d6dd25403b 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointTests.java
@@ -42,19 +42,22 @@ import static org.hamcrest.Matchers.not;
 
 public class GlobalCheckpointTests extends ESTestCase {
 
-    GlobalCheckpointService checkpointService;
+    GlobalCheckpointTracker tracker;
 
     @Override
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        checkpointService = new GlobalCheckpointService(new ShardId("test", "_na_", 0),
-            IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), UNASSIGNED_SEQ_NO);
+        tracker =
+            new GlobalCheckpointTracker(
+                new ShardId("test", "_na_", 0),
+                IndexSettingsModule.newIndexSettings("test", Settings.EMPTY),
+                UNASSIGNED_SEQ_NO);
     }
 
     public void testEmptyShards() {
-        assertFalse("checkpoint shouldn't be updated when the are no active shards", checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+        assertFalse("checkpoint shouldn't be updated when the are no active shards", tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
     }
 
     private final AtomicInteger aIdGenerator = new AtomicInteger();
@@ -81,7 +84,7 @@ public class GlobalCheckpointTests extends ESTestCase {
         // it is however nice not to assume this on this level and check we do the right thing.
         final long maxLocalCheckpoint = allocations.values().stream().min(Long::compare).orElse(UNASSIGNED_SEQ_NO);
 
-        assertThat(checkpointService.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+        assertThat(tracker.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
         logger.info("--> using allocations");
         allocations.keySet().forEach(aId -> {
@@ -96,42 +99,42 @@ public class GlobalCheckpointTests extends ESTestCase {
             logger.info("  - [{}], local checkpoint [{}], [{}]", aId, allocations.get(aId), type);
         });
 
-        checkpointService.updateAllocationIdsFromMaster(active, initializing);
-        initializing.forEach(aId -> checkpointService.markAllocationIdAsInSync(aId));
-        allocations.keySet().forEach(aId -> checkpointService.updateLocalCheckpoint(aId, allocations.get(aId)));
+        tracker.updateAllocationIdsFromMaster(active, initializing);
+        initializing.forEach(aId -> tracker.markAllocationIdAsInSync(aId));
+        allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId, allocations.get(aId)));
 
 
-        assertThat(checkpointService.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+        assertThat(tracker.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
-        assertThat(checkpointService.updateCheckpointOnPrimary(), equalTo(maxLocalCheckpoint != UNASSIGNED_SEQ_NO));
-        assertThat(checkpointService.getCheckpoint(), equalTo(maxLocalCheckpoint));
+        assertThat(tracker.updateCheckpointOnPrimary(), equalTo(maxLocalCheckpoint != UNASSIGNED_SEQ_NO));
+        assertThat(tracker.getCheckpoint(), equalTo(maxLocalCheckpoint));
 
         // increment checkpoints
         active.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4)));
         initializing.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4)));
-        allocations.keySet().forEach(aId -> checkpointService.updateLocalCheckpoint(aId, allocations.get(aId)));
+        allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId, allocations.get(aId)));
 
         // now insert an unknown active/insync id , the checkpoint shouldn't change but a refresh should be requested.
         final String extraId = "extra_" + randomAsciiOfLength(5);
 
         // first check that adding it without the master blessing doesn't change anything.
-        checkpointService.updateLocalCheckpoint(extraId, maxLocalCheckpoint + 1 + randomInt(4));
-        assertThat(checkpointService.getLocalCheckpointForAllocation(extraId), equalTo(UNASSIGNED_SEQ_NO));
+        tracker.updateLocalCheckpoint(extraId, maxLocalCheckpoint + 1 + randomInt(4));
+        assertThat(tracker.getLocalCheckpointForAllocationId(extraId), equalTo(UNASSIGNED_SEQ_NO));
 
         Set<String> newActive = new HashSet<>(active);
         newActive.add(extraId);
-        checkpointService.updateAllocationIdsFromMaster(newActive, initializing);
+        tracker.updateAllocationIdsFromMaster(newActive, initializing);
 
         // we should ask for a refresh , but not update the checkpoint
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), equalTo(maxLocalCheckpoint));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), equalTo(maxLocalCheckpoint));
 
         // now notify for the new id
-        checkpointService.updateLocalCheckpoint(extraId, maxLocalCheckpoint + 1 + randomInt(4));
+        tracker.updateLocalCheckpoint(extraId, maxLocalCheckpoint + 1 + randomInt(4));
 
         // now it should be incremented
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), greaterThan(maxLocalCheckpoint));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), greaterThan(maxLocalCheckpoint));
     }
 
     public void testMissingActiveIdsPreventAdvance() {
@@ -140,60 +143,60 @@ public class GlobalCheckpointTests extends ESTestCase {
         final Map<String, Long> assigned = new HashMap<>();
         assigned.putAll(active);
         assigned.putAll(initializing);
-        checkpointService.updateAllocationIdsFromMaster(
+        tracker.updateAllocationIdsFromMaster(
             new HashSet<>(randomSubsetOf(randomInt(active.size() - 1), active.keySet())),
             initializing.keySet());
-        randomSubsetOf(initializing.keySet()).forEach(checkpointService::markAllocationIdAsInSync);
-        assigned.forEach(checkpointService::updateLocalCheckpoint);
+        randomSubsetOf(initializing.keySet()).forEach(tracker::markAllocationIdAsInSync);
+        assigned.forEach(tracker::updateLocalCheckpoint);
 
         // now mark all active shards
-        checkpointService.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
+        tracker.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
 
         // global checkpoint can't be advanced, but we need a sync
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
         // update again
-        assigned.forEach(checkpointService::updateLocalCheckpoint);
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
+        assigned.forEach(tracker::updateLocalCheckpoint);
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
 
     public void testMissingInSyncIdsPreventAdvance() {
         final Map<String, Long> active = randomAllocationsWithLocalCheckpoints(0, 5);
         final Map<String, Long> initializing = randomAllocationsWithLocalCheckpoints(1, 5);
-        checkpointService.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
-        initializing.keySet().forEach(checkpointService::markAllocationIdAsInSync);
+        tracker.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
+        initializing.keySet().forEach(tracker::markAllocationIdAsInSync);
         randomSubsetOf(randomInt(initializing.size() - 1),
-            initializing.keySet()).forEach(aId -> checkpointService.updateLocalCheckpoint(aId, initializing.get(aId)));
+            initializing.keySet()).forEach(aId -> tracker.updateLocalCheckpoint(aId, initializing.get(aId)));
 
-        active.forEach(checkpointService::updateLocalCheckpoint);
+        active.forEach(tracker::updateLocalCheckpoint);
 
         // global checkpoint can't be advanced, but we need a sync
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
         // update again
-        initializing.forEach(checkpointService::updateLocalCheckpoint);
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
+        initializing.forEach(tracker::updateLocalCheckpoint);
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
 
     public void testInSyncIdsAreIgnoredIfNotValidatedByMaster() {
         final Map<String, Long> active = randomAllocationsWithLocalCheckpoints(1, 5);
         final Map<String, Long> initializing = randomAllocationsWithLocalCheckpoints(1, 5);
         final Map<String, Long> nonApproved = randomAllocationsWithLocalCheckpoints(1, 5);
-        checkpointService.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
-        initializing.keySet().forEach(checkpointService::markAllocationIdAsInSync);
-        nonApproved.keySet().forEach(checkpointService::markAllocationIdAsInSync);
+        tracker.updateAllocationIdsFromMaster(active.keySet(), initializing.keySet());
+        initializing.keySet().forEach(tracker::markAllocationIdAsInSync);
+        nonApproved.keySet().forEach(tracker::markAllocationIdAsInSync);
 
         List<Map<String, Long>> allocations = Arrays.asList(active, initializing, nonApproved);
         Collections.shuffle(allocations, random());
-        allocations.forEach(a -> a.forEach(checkpointService::updateLocalCheckpoint));
+        allocations.forEach(a -> a.forEach(tracker::updateLocalCheckpoint));
 
         // global checkpoint can be advanced, but we need a sync
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO)));
     }
 
     public void testInSyncIdsAreRemovedIfNotValidatedByMaster() {
@@ -212,33 +215,33 @@ public class GlobalCheckpointTests extends ESTestCase {
         if (randomBoolean()) {
             allocations.putAll(initializingToBeRemoved);
         }
-        checkpointService.updateAllocationIdsFromMaster(active, initializing);
+        tracker.updateAllocationIdsFromMaster(active, initializing);
         if (randomBoolean()) {
-            initializingToStay.keySet().forEach(checkpointService::markAllocationIdAsInSync);
+            initializingToStay.keySet().forEach(tracker::markAllocationIdAsInSync);
         } else {
-            initializing.forEach(checkpointService::markAllocationIdAsInSync);
+            initializing.forEach(tracker::markAllocationIdAsInSync);
         }
         if (randomBoolean()) {
-            allocations.forEach(checkpointService::updateLocalCheckpoint);
+            allocations.forEach(tracker::updateLocalCheckpoint);
         }
 
         // global checkpoint may be advanced, but we need a sync in any case
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
+        assertTrue(tracker.updateCheckpointOnPrimary());
 
         // now remove shards
         if (randomBoolean()) {
-            checkpointService.updateAllocationIdsFromMaster(activeToStay.keySet(), initializingToStay.keySet());
-            allocations.forEach((aid, ckp) -> checkpointService.updateLocalCheckpoint(aid, ckp + 10L));
+            tracker.updateAllocationIdsFromMaster(activeToStay.keySet(), initializingToStay.keySet());
+            allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid, ckp + 10L));
         } else {
-            allocations.forEach((aid, ckp) -> checkpointService.updateLocalCheckpoint(aid, ckp + 10L));
-            checkpointService.updateAllocationIdsFromMaster(activeToStay.keySet(), initializingToStay.keySet());
+            allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid, ckp + 10L));
+            tracker.updateAllocationIdsFromMaster(activeToStay.keySet(), initializingToStay.keySet());
         }
 
         final long checkpoint = Stream.concat(activeToStay.values().stream(), initializingToStay.values().stream())
             .min(Long::compare).get() + 10; // we added 10 to make sure it's advanced in the second time
 
         // global checkpoint is advanced and we need a sync
-        assertTrue(checkpointService.updateCheckpointOnPrimary());
-        assertThat(checkpointService.getCheckpoint(), equalTo(checkpoint));
+        assertTrue(tracker.updateCheckpointOnPrimary());
+        assertThat(tracker.getCheckpoint(), equalTo(checkpoint));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointServiceTests.java b/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
similarity index 57%
rename from core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointServiceTests.java
rename to core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
index d1f4c3987b2..74183670ecb 100644
--- a/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/seqno/LocalCheckpointTrackerTests.java
@@ -16,12 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.index.seqno;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.junit.Before;
@@ -31,16 +32,18 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.isOneOf;
 
-public class LocalCheckpointServiceTests extends ESTestCase {
+public class LocalCheckpointTrackerTests extends ESTestCase {
 
-    private LocalCheckpointService checkpointService;
+    private LocalCheckpointTracker tracker;
 
     private final int SMALL_CHUNK_SIZE = 4;
 
@@ -48,45 +51,47 @@ public class LocalCheckpointServiceTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        checkpointService = getCheckpointService();
+        tracker = getTracker();
     }
 
-    private LocalCheckpointService getCheckpointService() {
-        return new LocalCheckpointService(
-                new ShardId("test", "_na_", 0),
-                IndexSettingsModule.newIndexSettings("test",
-                        Settings.builder()
-                                .put(LocalCheckpointService.SETTINGS_BIT_ARRAYS_SIZE.getKey(), SMALL_CHUNK_SIZE)
-                                .build()),
-                SequenceNumbersService.NO_OPS_PERFORMED,
-                SequenceNumbersService.NO_OPS_PERFORMED);
+    private LocalCheckpointTracker getTracker() {
+        return new LocalCheckpointTracker(
+            IndexSettingsModule.newIndexSettings(
+                "test",
+                Settings
+                    .builder()
+                    .put(LocalCheckpointTracker.SETTINGS_BIT_ARRAYS_SIZE.getKey(), SMALL_CHUNK_SIZE)
+                    .build()),
+            SequenceNumbersService.NO_OPS_PERFORMED,
+            SequenceNumbersService.NO_OPS_PERFORMED
+        );
     }
 
     public void testSimplePrimary() {
         long seqNo1, seqNo2;
-        assertThat(checkpointService.getCheckpoint(), equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
-        seqNo1 = checkpointService.generateSeqNo();
+        assertThat(tracker.getCheckpoint(), equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
+        seqNo1 = tracker.generateSeqNo();
         assertThat(seqNo1, equalTo(0L));
-        checkpointService.markSeqNoAsCompleted(seqNo1);
-        assertThat(checkpointService.getCheckpoint(), equalTo(0L));
-        seqNo1 = checkpointService.generateSeqNo();
-        seqNo2 = checkpointService.generateSeqNo();
+        tracker.markSeqNoAsCompleted(seqNo1);
+        assertThat(tracker.getCheckpoint(), equalTo(0L));
+        seqNo1 = tracker.generateSeqNo();
+        seqNo2 = tracker.generateSeqNo();
         assertThat(seqNo1, equalTo(1L));
         assertThat(seqNo2, equalTo(2L));
-        checkpointService.markSeqNoAsCompleted(seqNo2);
-        assertThat(checkpointService.getCheckpoint(), equalTo(0L));
-        checkpointService.markSeqNoAsCompleted(seqNo1);
-        assertThat(checkpointService.getCheckpoint(), equalTo(2L));
+        tracker.markSeqNoAsCompleted(seqNo2);
+        assertThat(tracker.getCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsCompleted(seqNo1);
+        assertThat(tracker.getCheckpoint(), equalTo(2L));
     }
 
     public void testSimpleReplica() {
-        assertThat(checkpointService.getCheckpoint(), equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
-        checkpointService.markSeqNoAsCompleted(0L);
-        assertThat(checkpointService.getCheckpoint(), equalTo(0L));
-        checkpointService.markSeqNoAsCompleted(2L);
-        assertThat(checkpointService.getCheckpoint(), equalTo(0L));
-        checkpointService.markSeqNoAsCompleted(1L);
-        assertThat(checkpointService.getCheckpoint(), equalTo(2L));
+        assertThat(tracker.getCheckpoint(), equalTo(SequenceNumbersService.NO_OPS_PERFORMED));
+        tracker.markSeqNoAsCompleted(0L);
+        assertThat(tracker.getCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsCompleted(2L);
+        assertThat(tracker.getCheckpoint(), equalTo(0L));
+        tracker.markSeqNoAsCompleted(1L);
+        assertThat(tracker.getCheckpoint(), equalTo(2L));
     }
 
     public void testSimpleOverFlow() {
@@ -99,11 +104,11 @@ public class LocalCheckpointServiceTests extends ESTestCase {
         }
         Collections.shuffle(seqNoList, random());
         for (Integer seqNo : seqNoList) {
-            checkpointService.markSeqNoAsCompleted(seqNo);
+            tracker.markSeqNoAsCompleted(seqNo);
         }
-        assertThat(checkpointService.checkpoint, equalTo(maxOps - 1L));
-        assertThat(checkpointService.processedSeqNo.size(), equalTo(aligned ? 0 : 1));
-        assertThat(checkpointService.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
+        assertThat(tracker.checkpoint, equalTo(maxOps - 1L));
+        assertThat(tracker.processedSeqNo.size(), equalTo(aligned ? 0 : 1));
+        assertThat(tracker.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
     }
 
     public void testConcurrentPrimary() throws InterruptedException {
@@ -125,10 +130,10 @@ public class LocalCheckpointServiceTests extends ESTestCase {
                 protected void doRun() throws Exception {
                     barrier.await();
                     for (int i = 0; i < opsPerThread; i++) {
-                        long seqNo = checkpointService.generateSeqNo();
+                        long seqNo = tracker.generateSeqNo();
                         logger.info("[t{}] started   [{}]", threadId, seqNo);
                         if (seqNo != unFinishedSeq) {
-                            checkpointService.markSeqNoAsCompleted(seqNo);
+                            tracker.markSeqNoAsCompleted(seqNo);
                             logger.info("[t{}] completed [{}]", threadId, seqNo);
                         }
                     }
@@ -139,12 +144,12 @@ public class LocalCheckpointServiceTests extends ESTestCase {
         for (Thread thread : threads) {
             thread.join();
         }
-        assertThat(checkpointService.getMaxSeqNo(), equalTo(maxOps - 1L));
-        assertThat(checkpointService.getCheckpoint(), equalTo(unFinishedSeq - 1L));
-        checkpointService.markSeqNoAsCompleted(unFinishedSeq);
-        assertThat(checkpointService.getCheckpoint(), equalTo(maxOps - 1L));
-        assertThat(checkpointService.processedSeqNo.size(), isOneOf(0, 1));
-        assertThat(checkpointService.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
+        assertThat(tracker.getMaxSeqNo(), equalTo(maxOps - 1L));
+        assertThat(tracker.getCheckpoint(), equalTo(unFinishedSeq - 1L));
+        tracker.markSeqNoAsCompleted(unFinishedSeq);
+        assertThat(tracker.getCheckpoint(), equalTo(maxOps - 1L));
+        assertThat(tracker.processedSeqNo.size(), isOneOf(0, 1));
+        assertThat(tracker.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
     }
 
     public void testConcurrentReplica() throws InterruptedException {
@@ -177,7 +182,7 @@ public class LocalCheckpointServiceTests extends ESTestCase {
                     Integer[] ops = seqNoPerThread[threadId];
                     for (int seqNo : ops) {
                         if (seqNo != unFinishedSeq) {
-                            checkpointService.markSeqNoAsCompleted(seqNo);
+                            tracker.markSeqNoAsCompleted(seqNo);
                             logger.info("[t{}] completed [{}]", threadId, seqNo);
                         }
                     }
@@ -188,11 +193,48 @@ public class LocalCheckpointServiceTests extends ESTestCase {
         for (Thread thread : threads) {
             thread.join();
         }
-        assertThat(checkpointService.getMaxSeqNo(), equalTo(maxOps - 1L));
-        assertThat(checkpointService.getCheckpoint(), equalTo(unFinishedSeq - 1L));
-        checkpointService.markSeqNoAsCompleted(unFinishedSeq);
-        assertThat(checkpointService.getCheckpoint(), equalTo(maxOps - 1L));
-        assertThat(checkpointService.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
+        assertThat(tracker.getMaxSeqNo(), equalTo(maxOps - 1L));
+        assertThat(tracker.getCheckpoint(), equalTo(unFinishedSeq - 1L));
+        tracker.markSeqNoAsCompleted(unFinishedSeq);
+        assertThat(tracker.getCheckpoint(), equalTo(maxOps - 1L));
+        assertThat(tracker.firstProcessedSeqNo, equalTo(((long) maxOps / SMALL_CHUNK_SIZE) * SMALL_CHUNK_SIZE));
+    }
+
+    public void testWaitForOpsToComplete() throws BrokenBarrierException, InterruptedException {
+        final int seqNo = randomIntBetween(0, 32);
+        final CyclicBarrier barrier = new CyclicBarrier(2);
+        final AtomicBoolean complete = new AtomicBoolean();
+        final Thread thread = new Thread(() -> {
+            try {
+                // sychronize starting with the test thread
+                barrier.await();
+                tracker.waitForOpsToComplete(seqNo);
+                complete.set(true);
+                // synchronize with the test thread checking if we are no longer waiting
+                barrier.await();
+            } catch (BrokenBarrierException | InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        thread.start();
+
+        // synchronize starting with the waiting thread
+        barrier.await();
+
+        final List<Integer> elements = IntStream.rangeClosed(0, seqNo).boxed().collect(Collectors.toList());
+        Randomness.shuffle(elements);
+        for (int i = 0; i < elements.size() - 1; i++) {
+            tracker.markSeqNoAsCompleted(elements.get(i));
+            assertFalse(complete.get());
+        }
+
+        tracker.markSeqNoAsCompleted(elements.get(elements.size() - 1));
+        // synchronize with the waiting thread to mark that it is complete
+        barrier.await();
+        assertTrue(complete.get());
+
+        thread.join();
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
index 1600d22efd7..6b7c77fdf5e 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexSearcherWrapperTests.java
@@ -122,7 +122,7 @@ public class IndexSearcherWrapperTests extends ESTestCase {
         };
         final ConcurrentHashMap<Object, TopDocs> cache = new ConcurrentHashMap<>();
         try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) {
-            try (final Engine.Searcher wrap = wrapper.wrap(engineSearcher)) {
+            try (Engine.Searcher wrap = wrapper.wrap(engineSearcher)) {
                 ElasticsearchDirectoryReader.addReaderCloseListener(wrap.getDirectoryReader(), reader -> {
                     cache.remove(reader.getCoreCacheKey());
                 });
@@ -205,7 +205,7 @@ public class IndexSearcherWrapperTests extends ESTestCase {
         private final String field;
         private final AtomicInteger closeCalls;
 
-        public FieldMaskingReader(String field, DirectoryReader in, AtomicInteger closeCalls) throws IOException {
+        FieldMaskingReader(String field, DirectoryReader in, AtomicInteger closeCalls) throws IOException {
             super(in, new SubReaderWrapper() {
                 @Override
                 public LeafReader wrap(LeafReader reader) {
@@ -237,7 +237,7 @@ public class IndexSearcherWrapperTests extends ESTestCase {
 
         private final boolean hideDelegate;
 
-        public BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException {
+        BrokenWrapper(DirectoryReader in, boolean hideDelegate) throws IOException {
             super(in, new SubReaderWrapper() {
                 @Override
                 public LeafReader wrap(LeafReader reader) {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
index ef938376940..97c96c8af12 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java
@@ -44,6 +44,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
@@ -56,6 +57,7 @@ import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.translog.Translog;
@@ -100,9 +102,10 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing, long seqNo,
-                                              ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+    private ParsedDocument testParsedDocument(String id, String type, String routing, long seqNo,
+                                              ParseContext.Document document, BytesReference source, XContentType xContentType,
+                                              Mapping mappingUpdate) {
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
         document.add(uidField);
@@ -111,7 +114,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         document.add(seqID.seqNoDocValue);
         document.add(seqID.primaryTerm);
         return new ParsedDocument(versionField, seqID, id, type, routing,
-                Collections.singletonList(document), source, mappingUpdate);
+                Collections.singletonList(document), source, xContentType, mappingUpdate);
     }
 
     public void testLockTryingToDelete() throws Exception {
@@ -145,7 +148,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
     public void testMarkAsInactiveTriggersSyncedFlush() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test")
             .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
-        client().prepareIndex("test", "test").setSource("{}").get();
+        client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
         ensureGreen("test");
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         indicesService.indexService(resolveIndex("test")).getShardOrNull(0).checkIdle(0);
@@ -162,14 +165,14 @@ public class IndexShardIT extends ESSingleNodeTestCase {
     public void testDurableFlagHasEffect() {
         createIndex("test");
         ensureGreen();
-        client().prepareIndex("test", "bar", "1").setSource("{}").get();
+        client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).get();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
         setDurability(shard, Translog.Durability.REQUEST);
         assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
         setDurability(shard, Translog.Durability.ASYNC);
-        client().prepareIndex("test", "bar", "2").setSource("{}").get();
+        client().prepareIndex("test", "bar", "2").setSource("{}", XContentType.JSON).get();
         assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
         setDurability(shard, Translog.Durability.REQUEST);
         client().prepareDelete("test", "bar", "1").get();
@@ -180,13 +183,13 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
         setDurability(shard, Translog.Durability.REQUEST);
         assertNoFailures(client().prepareBulk()
-            .add(client().prepareIndex("test", "bar", "3").setSource("{}"))
+            .add(client().prepareIndex("test", "bar", "3").setSource("{}", XContentType.JSON))
             .add(client().prepareDelete("test", "bar", "1")).get());
         assertFalse(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
 
         setDurability(shard, Translog.Durability.ASYNC);
         assertNoFailures(client().prepareBulk()
-            .add(client().prepareIndex("test", "bar", "4").setSource("{}"))
+            .add(client().prepareIndex("test", "bar", "4").setSource("{}", XContentType.JSON))
             .add(client().prepareDelete("test", "bar", "3")).get());
         setDurability(shard, Translog.Durability.REQUEST);
         assertTrue(ShardUtilsTests.getShardEngine(shard).getTranslog().syncNeeded());
@@ -217,7 +220,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
             .build();
         createIndex("test", idxSettings);
         ensureGreen("test");
-        client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
         client().admin().indices().prepareDelete("test").get();
@@ -229,7 +232,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         assertAcked(client().admin().indices().prepareCreate("test")
             .setSettings(SETTING_NUMBER_OF_SHARDS, 1, SETTING_NUMBER_OF_REPLICAS, 0));
         for (int i = 0; i < 50; i++) {
-            client().prepareIndex("test", "test").setSource("{}").get();
+            client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
         }
         ensureGreen("test");
         InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class);
@@ -265,7 +268,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString());
         createIndex(INDEX, sb);
         ensureGreen(INDEX);
-        client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex(INDEX, "bar", "1").setSource("{}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
 
         SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
         assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
@@ -322,21 +325,22 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
             .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
                 new ByteSizeValue(117 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        client().prepareIndex("test", "test", "0")
+            .setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertFalse(shard.shouldFlush());
         ParsedDocument doc = testParsedDocument(
-            "1",
             "1",
             "test",
             null,
             SequenceNumbersService.UNASSIGNED_SEQ_NO,
             new ParseContext.Document(),
-            new BytesArray(new byte[]{1}), null);
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc);
+            new BytesArray(new byte[]{1}), XContentType.JSON, null);
+        Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc);
         shard.index(index);
         assertTrue(shard.shouldFlush());
         assertEquals(2, shard.getEngine().getTranslog().totalOperations());
-        client().prepareIndex("test", "test", "2").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON)
+            .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertBusy(() -> { // this is async
             assertFalse(shard.shouldFlush());
         });
@@ -369,7 +373,8 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(
             IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
             new ByteSizeValue(117/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        client().prepareIndex("test", "test", "0").setSource("{}", XContentType.JSON)
+            .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertFalse(shard.shouldFlush());
         final AtomicBoolean running = new AtomicBoolean(true);
         final int numThreads = randomIntBetween(2, 4);
@@ -394,7 +399,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         barrier.await();
         FlushStats flushStats = shard.flushStats();
         long total = flushStats.getTotal();
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         assertBusy(() -> assertEquals(total + 1, shard.flushStats().getTotal()));
         running.set(false);
         for (int i = 0; i < threads.length; i++) {
@@ -409,9 +414,9 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get();
+        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
         client().prepareDelete("test", "test", "0").get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}", XContentType.JSON).setRefreshPolicy(IMMEDIATE).get();
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
         shard.close("simon says", false);
@@ -420,7 +425,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
         IndexingOperationListener listener = new IndexingOperationListener() {
 
             @Override
-            public void postIndex(Engine.Index index, Engine.IndexResult result) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
                 try {
                     assertNotNull(shardRef.get());
                     // this is all IMC needs to do - check current memory and refresh
@@ -434,7 +439,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
 
 
             @Override
-            public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 try {
                     assertNotNull(shardRef.get());
                     // this is all IMC needs to do - check current memory and refresh
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 036faa9b903..cb3abaea60a 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -29,10 +29,12 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
@@ -57,11 +59,15 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.fielddata.FieldDataStats;
@@ -73,6 +79,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -112,13 +119,16 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.common.lucene.Lucene.cleanLuceneIndex;
-import static org.elasticsearch.common.lucene.Lucene.readScoreDoc;
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -133,7 +143,7 @@ import static org.hamcrest.Matchers.nullValue;
 public class IndexShardTests extends IndexShardTestCase {
 
     public static ShardStateMetaData load(Logger logger, Path... shardPaths) throws IOException {
-        return ShardStateMetaData.FORMAT.loadLatestState(logger, shardPaths);
+        return ShardStateMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, shardPaths);
     }
 
     public static void write(ShardStateMetaData shardStateMetaData,
@@ -539,9 +549,9 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(shard);
     }
 
-    private ParsedDocument testParsedDocument(String uid, String id, String type, String routing,
+    private ParsedDocument testParsedDocument(String id, String type, String routing,
                                               ParseContext.Document document, BytesReference source, Mapping mappingUpdate) {
-        Field uidField = new Field("_uid", uid, UidFieldMapper.Defaults.FIELD_TYPE);
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", 0);
         SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
         document.add(uidField);
@@ -549,7 +559,8 @@ public class IndexShardTests extends IndexShardTestCase {
         document.add(seqID.seqNo);
         document.add(seqID.seqNoDocValue);
         document.add(seqID.primaryTerm);
-        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, mappingUpdate);
+        return new ParsedDocument(versionField, seqID, id, type, routing, Arrays.asList(document), source, XContentType.JSON,
+            mappingUpdate);
     }
 
     public void testIndexingOperationsListeners() throws IOException {
@@ -565,13 +576,13 @@ public class IndexShardTests extends IndexShardTestCase {
         shard.close("simon says", true);
         shard = reinitShard(shard, new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 preIndex.incrementAndGet();
                 return operation;
             }
 
             @Override
-            public void postIndex(Engine.Index index, Engine.IndexResult result) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
                 if (result.hasFailure() == false) {
                     if (result.isCreated()) {
                         postIndexCreate.incrementAndGet();
@@ -579,41 +590,41 @@ public class IndexShardTests extends IndexShardTestCase {
                         postIndexUpdate.incrementAndGet();
                     }
                 } else {
-                    postIndex(index, result.getFailure());
+                    postIndex(shardId, index, result.getFailure());
                 }
             }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
                 postIndexException.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 if (result.hasFailure() == false) {
                     postDelete.incrementAndGet();
                 } else {
-                    postDelete(delete, result.getFailure());
+                    postDelete(shardId, delete, result.getFailure());
                 }
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
                 postDeleteException.incrementAndGet();
 
             }
         });
         recoveryShardFromStore(shard);
 
-        ParsedDocument doc = testParsedDocument("1", "1", "test", null, new ParseContext.Document(),
+        ParsedDocument doc = testParsedDocument("1", "test", null, new ParseContext.Document(),
             new BytesArray(new byte[]{1}), null);
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc);
+        Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc);
         shard.index(index);
         assertEquals(1, preIndex.get());
         assertEquals(1, postIndexCreate.get());
@@ -632,7 +643,7 @@ public class IndexShardTests extends IndexShardTestCase {
         assertEquals(0, postDelete.get());
         assertEquals(0, postDeleteException.get());
 
-        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1"));
+        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", doc.uid()));
         shard.delete(delete);
 
         assertEquals(2, preIndex.get());
@@ -649,7 +660,7 @@ public class IndexShardTests extends IndexShardTestCase {
         try {
             shard.index(index);
             fail();
-        } catch (IllegalIndexShardStateException e) {
+        } catch (AlreadyClosedException e) {
 
         }
 
@@ -663,7 +674,7 @@ public class IndexShardTests extends IndexShardTestCase {
         try {
             shard.delete(delete);
             fail();
-        } catch (IllegalIndexShardStateException e) {
+        } catch (AlreadyClosedException e) {
 
         }
 
@@ -1136,24 +1147,24 @@ public class IndexShardTests extends IndexShardTestCase {
         final AtomicInteger postDelete = new AtomicInteger();
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 preIndex.incrementAndGet();
                 return operation;
             }
 
             @Override
-            public void postIndex(Engine.Index index, Engine.IndexResult result) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
                 postIndex.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 postDelete.incrementAndGet();
 
             }
@@ -1357,11 +1368,100 @@ public class IndexShardTests extends IndexShardTestCase {
         closeShards(sourceShard, targetShard);
     }
 
+    public void testDocStats() throws IOException {
+        IndexShard indexShard = null;
+        try {
+            indexShard = newStartedShard();
+            final long numDocs = randomIntBetween(2, 32); // at least two documents so we have docs to delete
+            // Delete at least numDocs/10 documents otherwise the number of deleted docs will be below 10%
+            // and forceMerge will refuse to expunge deletes
+            final long numDocsToDelete = randomIntBetween((int) Math.ceil(Math.nextUp(numDocs / 10.0)), Math.toIntExact(numDocs));
+            for (int i = 0; i < numDocs; i++) {
+                final String id = Integer.toString(i);
+                final ParsedDocument doc =
+                    testParsedDocument(id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null);
+                final Engine.Index index =
+                    new Engine.Index(
+                        new Term("_uid", doc.uid()),
+                        doc,
+                        SequenceNumbersService.UNASSIGNED_SEQ_NO,
+                        0,
+                        Versions.MATCH_ANY,
+                        VersionType.INTERNAL,
+                        PRIMARY,
+                        System.nanoTime(),
+                        -1,
+                        false);
+                final Engine.IndexResult result = indexShard.index(index);
+                assertThat(result.getVersion(), equalTo(1L));
+            }
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docsStats = indexShard.docStats();
+                assertThat(docsStats.getCount(), equalTo(numDocs));
+                assertThat(docsStats.getDeleted(), equalTo(0L));
+            }
+
+            final List<Integer> ids = randomSubsetOf(
+                Math.toIntExact(numDocsToDelete),
+                IntStream.range(0, Math.toIntExact(numDocs)).boxed().collect(Collectors.toList()));
+            for (final Integer i : ids) {
+                final String id = Integer.toString(i);
+                final ParsedDocument doc =
+                    testParsedDocument(id, "test", null, new ParseContext.Document(), new BytesArray("{}"), null);
+                final Engine.Index index =
+                    new Engine.Index(
+                        new Term("_uid", doc.uid()),
+                        doc,
+                        SequenceNumbersService.UNASSIGNED_SEQ_NO,
+                        0,
+                        Versions.MATCH_ANY,
+                        VersionType.INTERNAL,
+                        PRIMARY,
+                        System.nanoTime(),
+                        -1,
+                        false);
+                final Engine.IndexResult result = indexShard.index(index);
+                assertThat(result.getVersion(), equalTo(2L));
+            }
+
+            // flush the buffered deletes
+            final FlushRequest flushRequest = new FlushRequest();
+            flushRequest.force(false);
+            flushRequest.waitIfOngoing(false);
+            indexShard.flush(flushRequest);
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docStats = indexShard.docStats();
+                assertThat(docStats.getCount(), equalTo(numDocs));
+                // Lucene will delete a segment if all docs are deleted from it; this means that we lose the deletes when deleting all docs
+                assertThat(docStats.getDeleted(), equalTo(numDocsToDelete == numDocs ? 0 : numDocsToDelete));
+            }
+
+            // merge them away
+            final ForceMergeRequest forceMergeRequest = new ForceMergeRequest();
+            forceMergeRequest.onlyExpungeDeletes(randomBoolean());
+            forceMergeRequest.maxNumSegments(1);
+            indexShard.forceMerge(forceMergeRequest);
+
+            indexShard.refresh("test");
+            {
+                final DocsStats docStats = indexShard.docStats();
+                assertThat(docStats.getCount(), equalTo(numDocs));
+                assertThat(docStats.getDeleted(), equalTo(0L));
+            }
+        } finally {
+            closeShards(indexShard);
+        }
+    }
+
     /** A dummy repository for testing which just needs restore overridden */
     private abstract static class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
         private final String indexName;
 
-        public RestoreOnlyRepository(String indexName) {
+        RestoreOnlyRepository(String indexName) {
             super(Settings.EMPTY);
             this.indexName = indexName;
         }
@@ -1397,7 +1497,7 @@ public class IndexShardTests extends IndexShardTestCase {
         public RepositoryData getRepositoryData() {
             Map<IndexId, Set<SnapshotId>> map = new HashMap<>();
             map.put(new IndexId(indexName, "blah"), emptySet());
-            return new RepositoryData(Collections.emptyList(), map);
+            return new RepositoryData(EMPTY_REPO_GEN, Collections.emptyList(), map, Collections.emptyList());
         }
 
         @Override
@@ -1405,12 +1505,13 @@ public class IndexShardTests extends IndexShardTestCase {
         }
 
         @Override
-        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
+        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<IndexId> indices, long startTime, String failure, int totalShards,
+                                             List<SnapshotShardFailure> shardFailures, long repositoryStateId) {
             return null;
         }
 
         @Override
-        public void deleteSnapshot(SnapshotId snapshotId) {
+        public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
index c28e34d9ca4..2eb91a16d80 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
@@ -19,7 +19,10 @@
 package org.elasticsearch.index.shard;
 
 import org.apache.lucene.index.Term;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.InternalEngineTests;
+import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -29,6 +32,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.hamcrest.Matchers.is;
+
 public class IndexingOperationListenerTests extends ESTestCase{
 
     // this test also tests if calls are correct if one or more listeners throw exceptions
@@ -39,76 +44,83 @@ public class IndexingOperationListenerTests extends ESTestCase{
         AtomicInteger preDelete = new AtomicInteger();
         AtomicInteger postDelete = new AtomicInteger();
         AtomicInteger postDeleteException = new AtomicInteger();
+        ShardId randomShardId = new ShardId(new Index(randomAsciiOfLength(10), randomAsciiOfLength(10)), randomIntBetween(1, 10));
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
+                assertThat(shardId, is(randomShardId));
                 preIndex.incrementAndGet();
                 return operation;
             }
 
             @Override
-            public void postIndex(Engine.Index index, Engine.IndexResult result) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
+                assertThat(shardId, is(randomShardId));
                 if (result.hasFailure() == false) {
                     postIndex.incrementAndGet();
                 } else {
-                    postIndex(index, result.getFailure());
+                    postIndex(shardId, index, result.getFailure());
                 }
             }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
+                assertThat(shardId, is(randomShardId));
                 postIndexException.incrementAndGet();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
+                assertThat(shardId, is(randomShardId));
                 preDelete.incrementAndGet();
                 return delete;
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
+                assertThat(shardId, is(randomShardId));
                 if (result.hasFailure() == false) {
                     postDelete.incrementAndGet();
                 } else {
-                    postDelete(delete, result.getFailure());
+                    postDelete(shardId, delete, result.getFailure());
                 }
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
+                assertThat(shardId, is(randomShardId));
                 postDeleteException.incrementAndGet();
             }
         };
 
         IndexingOperationListener throwingListener = new IndexingOperationListener() {
             @Override
-            public Engine.Index preIndex(Engine.Index operation) {
+            public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postIndex(Engine.Index index, Engine.IndexResult result) {
+            public void postIndex(ShardId shardId, Engine.Index index, Engine.IndexResult result) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postIndex(Engine.Index index, Exception ex) {
+            public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
                 throw new RuntimeException();
             }
 
             @Override
-            public Engine.Delete preDelete(Engine.Delete delete) {
+            public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Engine.DeleteResult result) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResult result) {
                 throw new RuntimeException();
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Exception ex) {
+            public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
                 throw new RuntimeException();
             }
         };
@@ -120,10 +132,12 @@ public class IndexingOperationListenerTests extends ESTestCase{
             }
         }
         Collections.shuffle(indexingOperationListeners, random());
-        IndexingOperationListener.CompositeListener compositeListener = new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger);
-        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", "1"));
-        Engine.Index index = new Engine.Index(new Term("_uid", "1"), null);
-        compositeListener.postDelete(delete, new Engine.DeleteResult(1, SequenceNumbersService.UNASSIGNED_SEQ_NO, true));
+        IndexingOperationListener.CompositeListener compositeListener =
+            new IndexingOperationListener.CompositeListener(indexingOperationListeners, logger);
+        ParsedDocument doc = InternalEngineTests.createParsedDoc("1", "test", null);
+        Engine.Delete delete = new Engine.Delete("test", "1", new Term("_uid", doc.uid()));
+        Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc);
+        compositeListener.postDelete(randomShardId, delete, new Engine.DeleteResult(1, SequenceNumbersService.UNASSIGNED_SEQ_NO, true));
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -131,7 +145,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
         assertEquals(2, postDelete.get());
         assertEquals(0, postDeleteException.get());
 
-        compositeListener.postDelete(delete, new RuntimeException());
+        compositeListener.postDelete(randomShardId, delete, new RuntimeException());
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -139,7 +153,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.preDelete(delete);
+        compositeListener.preDelete(randomShardId, delete);
         assertEquals(0, preIndex.get());
         assertEquals(0, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -147,7 +161,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.postIndex(index, new Engine.IndexResult(0, SequenceNumbersService.UNASSIGNED_SEQ_NO, false));
+        compositeListener.postIndex(randomShardId, index, new Engine.IndexResult(0, SequenceNumbersService.UNASSIGNED_SEQ_NO, false));
         assertEquals(0, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(0, postIndexException.get());
@@ -155,7 +169,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.postIndex(index, new RuntimeException());
+        compositeListener.postIndex(randomShardId, index, new RuntimeException());
         assertEquals(0, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(2, postIndexException.get());
@@ -163,7 +177,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
         assertEquals(2, postDelete.get());
         assertEquals(2, postDeleteException.get());
 
-        compositeListener.preIndex(index);
+        compositeListener.preIndex(randomShardId, index);
         assertEquals(2, preIndex.get());
         assertEquals(2, postIndex.get());
         assertEquals(2, postIndexException.get());
diff --git a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
index f1515cd559b..fc8fc12e75d 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/NewPathForShardTests.java
@@ -76,7 +76,7 @@ public class NewPathForShardTests extends ESTestCase {
     /** Mock file system that fakes usable space for each FileStore */
     static class MockUsableSpaceFileSystemProvider extends FilterFileSystemProvider {
 
-        public MockUsableSpaceFileSystemProvider(FileSystem inner) {
+        MockUsableSpaceFileSystemProvider(FileSystem inner) {
             super("mockusablespace://", inner);
             final List<FileStore> fileStores = new ArrayList<>();
             fileStores.add(aFileStore);
@@ -99,7 +99,7 @@ public class NewPathForShardTests extends ESTestCase {
 
         private final String desc;
 
-        public MockFileStore(String desc) {
+        MockFileStore(String desc) {
             this.desc = desc;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 57e40a03295..846ca6be201 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.CodecService;
@@ -48,6 +49,7 @@ import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
@@ -116,9 +118,10 @@ public class RefreshListenersTests extends ESTestCase {
                 // we don't need to notify anybody in this test
             }
         };
+        TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger);
         EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null,
                 store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(),
-                iwc.getSimilarity(), new CodecService(null, logger), eventListener, new TranslogHandler(shardId.getIndexName(), logger),
+                iwc.getSimilarity(), new CodecService(null, logger), eventListener, translogHandler,
                 IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
                 TimeValue.timeValueMinutes(5), listeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
         engine = new InternalEngine(config);
@@ -131,7 +134,38 @@ public class RefreshListenersTests extends ESTestCase {
         terminate(threadPool);
     }
 
+    public void testBeforeRefresh() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult index = index("1");
+        DummyRefreshListener listener = new DummyRefreshListener();
+        assertFalse(listeners.addOrNotify(index.getTranslogLocation(), listener));
+        assertNull(listener.forcedRefresh.get());
+        assertEquals(1, listeners.pendingCount());
+        engine.refresh("I said so");
+        assertFalse(listener.forcedRefresh.get());
+        listener.assertNoError();
+        assertEquals(0, listeners.pendingCount());
+    }
+
+    public void testAfterRefresh() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult index = index("1");
+        engine.refresh("I said so");
+        if (randomBoolean()) {
+            index(randomFrom("1" /* same document */, "2" /* different document */));
+            if (randomBoolean()) {
+                engine.refresh("I said so");
+            }
+        }
+        DummyRefreshListener listener = new DummyRefreshListener();
+        assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener));
+        assertFalse(listener.forcedRefresh.get());
+        listener.assertNoError();
+        assertEquals(0, listeners.pendingCount());
+    }
+
     public void testTooMany() throws Exception {
+        assertEquals(0, listeners.pendingCount());
         assertFalse(listeners.refreshNeeded());
         Engine.IndexResult index = index("1");
 
@@ -148,6 +182,7 @@ public class RefreshListenersTests extends ESTestCase {
         for (DummyRefreshListener listener : nonForcedListeners) {
             assertNull("Called listener too early!", listener.forcedRefresh.get());
         }
+        assertEquals(maxListeners, listeners.pendingCount());
 
         // Add one more listener which should cause a refresh.
         DummyRefreshListener forcingListener = new DummyRefreshListener();
@@ -161,22 +196,45 @@ public class RefreshListenersTests extends ESTestCase {
             listener.assertNoError();
         }
         assertFalse(listeners.refreshNeeded());
+        assertEquals(0, listeners.pendingCount());
     }
 
-    public void testAfterRefresh() throws Exception {
-        Engine.IndexResult index = index("1");
+    public void testClose() throws Exception {
+        assertEquals(0, listeners.pendingCount());
+        Engine.IndexResult refreshedOperation = index("1");
         engine.refresh("I said so");
-        if (randomBoolean()) {
-            index(randomFrom("1" /* same document */, "2" /* different document */));
-            if (randomBoolean()) {
-                engine.refresh("I said so");
-            }
+        Engine.IndexResult unrefreshedOperation = index("1");
+        {
+            /* Closing flushed pending listeners as though they were refreshed. Since this can only happen when the index is closed and no
+             * longer useful there doesn't seem much point in sending the listener some kind of "I'm closed now, go away" enum value. */
+            DummyRefreshListener listener = new DummyRefreshListener();
+            assertFalse(listeners.addOrNotify(unrefreshedOperation.getTranslogLocation(), listener));
+            assertNull(listener.forcedRefresh.get());
+            listeners.close();
+            assertFalse(listener.forcedRefresh.get());
+            listener.assertNoError();
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
+        }
+        {
+            // If you add a listener for an already refreshed location then it'll just fire even if closed
+            DummyRefreshListener listener = new DummyRefreshListener();
+            assertTrue(listeners.addOrNotify(refreshedOperation.getTranslogLocation(), listener));
+            assertFalse(listener.forcedRefresh.get());
+            listener.assertNoError();
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
+        }
+        {
+            // But adding a listener to a non-refreshed location will fail
+            DummyRefreshListener listener = new DummyRefreshListener();
+            Exception e = expectThrows(IllegalStateException.class, () ->
+                listeners.addOrNotify(unrefreshedOperation.getTranslogLocation(), listener));
+            assertEquals("can't wait for refresh on a closed index", e.getMessage());
+            assertNull(listener.forcedRefresh.get());
+            assertFalse(listeners.refreshNeeded());
+            assertEquals(0, listeners.pendingCount());
         }
-
-        DummyRefreshListener listener = new DummyRefreshListener();
-        assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener));
-        assertFalse(listener.forcedRefresh.get());
-        listener.assertNoError();
     }
 
     /**
@@ -241,7 +299,7 @@ public class RefreshListenersTests extends ESTestCase {
                         }
                         listener.assertNoError();
 
-                        Engine.Get get = new Engine.Get(false, new Term("_uid", "test:"+threadId));
+                        Engine.Get get = new Engine.Get(false, new Term("_uid",  Uid.createUid("test", threadId)));
                         try (Engine.GetResult getResult = engine.get(get)) {
                             assertTrue("document not found", getResult.exists());
                             assertEquals(iteration, getResult.version());
@@ -263,16 +321,16 @@ public class RefreshListenersTests extends ESTestCase {
         refresher.cancel();
     }
 
-    private Engine.IndexResult index(String id) {
+    private Engine.IndexResult index(String id) throws IOException {
         return index(id, "test");
     }
 
-    private Engine.IndexResult index(String id, String testFieldValue) {
+    private Engine.IndexResult index(String id, String testFieldValue) throws IOException {
         String type = "test";
         String uid = type + ":" + id;
         Document document = new Document();
         document.add(new TextField("test", testFieldValue, Field.Store.YES));
-        Field uidField = new Field("_uid", type + ":" + id, UidFieldMapper.Defaults.FIELD_TYPE);
+        Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
         Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
         SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
         document.add(uidField);
@@ -281,8 +339,9 @@ public class RefreshListenersTests extends ESTestCase {
         document.add(seqID.seqNoDocValue);
         document.add(seqID.primaryTerm);
         BytesReference source = new BytesArray(new byte[] { 1 });
-        ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, null);
-        Engine.Index index = new Engine.Index(new Term("_uid", uid), doc);
+        ParsedDocument doc = new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, XContentType.JSON,
+            null);
+        Engine.Index index = new Engine.Index(new Term("_uid", doc.uid()), doc);
         return engine.index(index);
     }
 
@@ -290,13 +349,12 @@ public class RefreshListenersTests extends ESTestCase {
         /**
          * When the listener is called this captures it's only argument.
          */
-        AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
+        final AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
         private volatile Exception error;
 
         @Override
         public void accept(Boolean forcedRefresh) {
             try {
-                assertNotNull(forcedRefresh);
                 Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
                 assertNull("Listener called twice", oldValue);
             } catch (Exception e) {
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
new file mode 100644
index 00000000000..dd05c10af96
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardIdTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.test.ESTestCase;
+
+public class ShardIdTests extends ESTestCase {
+
+    public void testShardIdFromString() {
+        String indexName = randomAsciiOfLengthBetween(3,50);
+        int shardId = randomInt();
+        ShardId id = ShardId.fromString("["+indexName+"]["+shardId+"]");
+        assertEquals(indexName, id.getIndexName());
+        assertEquals(shardId, id.getId());
+        assertEquals(indexName, id.getIndex().getName());
+        assertEquals(IndexMetaData.INDEX_UUID_NA_VALUE, id.getIndex().getUUID());
+
+        id = ShardId.fromString("[some]weird[0]Name][-125]");
+        assertEquals("some]weird[0]Name", id.getIndexName());
+        assertEquals(-125, id.getId());
+        assertEquals("some]weird[0]Name", id.getIndex().getName());
+        assertEquals(IndexMetaData.INDEX_UUID_NA_VALUE, id.getIndex().getUUID());
+
+        String badId = indexName + "," + shardId; // missing separator
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ShardId.fromString(badId));
+        assertEquals("Unexpected shardId string format, expected [indexName][shardId] but got " + badId, ex.getMessage());
+
+        String badId2 = indexName + "][" + shardId + "]"; // missing opening bracket
+        ex = expectThrows(IllegalArgumentException.class,
+                () -> ShardId.fromString(badId2));
+
+        String badId3 = "[" + indexName + "][" + shardId; // missing closing bracket
+        ex = expectThrows(IllegalArgumentException.class,
+                () -> ShardId.fromString(badId3));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
index bcb29b66148..f3c9cd9dbfd 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.is;
 
 public class ShardPathTests extends ESTestCase {
     public void testLoadShardPath() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
@@ -54,7 +54,7 @@ public class ShardPathTests extends ESTestCase {
     }
 
     public void testFailLoadShardPathOnMultiState() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             final String indexUUID = "0xDEADBEEF";
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
@@ -71,7 +71,7 @@ public class ShardPathTests extends ESTestCase {
     }
 
     public void testFailLoadShardPathIndexUUIDMissmatch() throws IOException {
-        try (final NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
+        try (NodeEnvironment env = newNodeEnvironment(Settings.builder().build())) {
             Settings.Builder builder = Settings.builder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar")
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
             Settings settings = builder.build();
@@ -127,7 +127,7 @@ public class ShardPathTests extends ESTestCase {
             indexSettings = indexSettingsBuilder.build();
             nodeSettings = Settings.EMPTY;
         }
-        try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
+        try (NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
             ShardId shardId = new ShardId("foo", indexUUID, 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
index 74b61047ace..1fefbfceac3 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/SlicedInputStreamTests.java
@@ -121,7 +121,7 @@ public class SlicedInputStreamTests extends ESTestCase {
 
         public boolean closed = false;
 
-        public CheckClosedInputStream(InputStream in) {
+        CheckClosedInputStream(InputStream in) {
             super(in);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
index 9c4bcce6ff6..6fa32df2885 100644
--- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
@@ -61,7 +61,7 @@ public class IndexStoreTests extends ESTestCase {
         Settings settings = settingsBuilder.build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
         FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
-        try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
+        try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             switch (type) {
                 case NIOFS:
                     assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index 11b94c26fff..156c2132358 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -468,11 +468,11 @@ public class StoreTests extends ESTestCase {
         private final Directory dir;
         private final Random random;
 
-        public LuceneManagedDirectoryService(Random random) {
+        LuceneManagedDirectoryService(Random random) {
             this(random, true);
         }
 
-        public LuceneManagedDirectoryService(Random random, boolean preventDoubleWrite) {
+        LuceneManagedDirectoryService(Random random, boolean preventDoubleWrite) {
             super(new ShardId(INDEX_SETTINGS.getIndex(), 1), INDEX_SETTINGS);
             dir = StoreTests.newDirectory(random);
             this.random = random;
diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java
index 71ffffdcbff..2d699b2c457 100644
--- a/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java
@@ -31,7 +31,7 @@ public class CompletionsStatsTests extends ESTestCase {
 
     public void testSerialize() throws IOException {
         FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats();
-        CompletionStats stats = new CompletionStats(randomPositiveLong(), map == null ? null :
+        CompletionStats stats = new CompletionStats(randomNonNegativeLong(), map == null ? null :
             map);
         BytesStreamOutput out = new BytesStreamOutput();
         stats.writeTo(out);
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 41118d6efa2..33f6f8b8a25 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -23,6 +23,9 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.mockfile.FilterFileChannel;
 import org.apache.lucene.store.AlreadyClosedException;
@@ -31,7 +34,7 @@ import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -47,7 +50,15 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.Engine.Operation.Origin;
+import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog.Location;
@@ -68,6 +79,7 @@ import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -88,9 +100,11 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasToString;
 
 @LuceneTestCase.SuppressFileSystems("ExtrasFS")
 public class TranslogTests extends ESTestCase {
@@ -226,17 +240,28 @@ public class TranslogTests extends ESTestCase {
         assertThat(snapshot, SnapshotMatchers.equalsTo(ops));
         assertThat(snapshot.totalOperations(), equalTo(ops.size()));
 
+        final long seqNo = randomNonNegativeLong();
+        final long primaryTerm = randomNonNegativeLong();
+        final String reason = randomAsciiOfLength(16);
+        addToTranslogAndList(translog, ops, new Translog.NoOp(seqNo, primaryTerm, reason));
+
         snapshot = translog.newSnapshot();
 
         Translog.Index index = (Translog.Index) snapshot.next();
-        assertThat(index != null, equalTo(true));
+        assertNotNull(index);
         assertThat(BytesReference.toBytes(index.source()), equalTo(new byte[]{1}));
 
         Translog.Delete delete = (Translog.Delete) snapshot.next();
-        assertThat(delete != null, equalTo(true));
+        assertNotNull(delete);
         assertThat(delete.uid(), equalTo(newUid("2")));
 
-        assertThat(snapshot.next(), equalTo(null));
+        Translog.NoOp noOp = (Translog.NoOp) snapshot.next();
+        assertNotNull(noOp);
+        assertThat(noOp.seqNo(), equalTo(seqNo));
+        assertThat(noOp.primaryTerm(), equalTo(primaryTerm));
+        assertThat(noOp.reason(), equalTo(reason));
+
+        assertNull(snapshot.next());
 
         long firstId = translog.currentFileGeneration();
         translog.prepareCommit();
@@ -268,68 +293,104 @@ public class TranslogTests extends ESTestCase {
 
     public void testStats() throws IOException {
         final long firstOperationPosition = translog.getFirstOperationPosition();
-        TranslogStats stats = stats();
-        assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
-        long lastSize = stats.getTranslogSizeInBytes();
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
+        }
         assertThat((int) firstOperationPosition, greaterThan(CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC)));
-        assertThat(lastSize, equalTo(firstOperationPosition));
-        TranslogStats total = new TranslogStats();
         translog.add(new Translog.Index("test", "1", new byte[]{1}));
-        stats = stats();
-        total.add(stats);
-        assertThat(stats.estimatedNumberOfOperations(), equalTo(1L));
-        assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize));
-        lastSize = stats.getTranslogSizeInBytes();
+
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(1L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(97L));
+        }
 
         translog.add(new Translog.Delete(newUid("2")));
-        stats = stats();
-        total.add(stats);
-        assertThat(stats.estimatedNumberOfOperations(), equalTo(2L));
-        assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize));
-        lastSize = stats.getTranslogSizeInBytes();
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(2L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(139L));
+        }
 
         translog.add(new Translog.Delete(newUid("3")));
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(3L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(181L));
+        }
+
+        final long seqNo = 1;
+        final long primaryTerm = 1;
+        translog.add(new Translog.NoOp(seqNo, primaryTerm, randomAsciiOfLength(16)));
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(4L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(223L));
+        }
+
+        final long expectedSizeInBytes = 266L;
         translog.prepareCommit();
-        stats = stats();
-        total.add(stats);
-        assertThat(stats.estimatedNumberOfOperations(), equalTo(3L));
-        assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize));
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(4L));
+            assertThat(
+                stats.getTranslogSizeInBytes(),
+                equalTo(expectedSizeInBytes));
+        }
+
+        {
+            final TranslogStats stats = stats();
+            final BytesStreamOutput out = new BytesStreamOutput();
+            stats.writeTo(out);
+            final TranslogStats copy = new TranslogStats();
+            copy.readFrom(out.bytes().streamInput());
+
+            assertThat(copy.estimatedNumberOfOperations(), equalTo(4L));
+            assertThat(copy.getTranslogSizeInBytes(), equalTo(expectedSizeInBytes));
+
+            try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+                builder.startObject();
+                copy.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                builder.endObject();
+                assertThat(builder.string(), equalTo("{\"translog\":{\"operations\":4,\"size_in_bytes\":" + expectedSizeInBytes + "}}"));
+            }
+        }
 
         translog.commit();
-        stats = stats();
-        total.add(stats);
-        assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
-        assertThat(stats.getTranslogSizeInBytes(), equalTo(firstOperationPosition));
-        assertEquals(6, total.estimatedNumberOfOperations());
-        assertEquals(419, total.getTranslogSizeInBytes());
+        {
+            final TranslogStats stats = stats();
+            assertThat(stats.estimatedNumberOfOperations(), equalTo(0L));
+            assertThat(stats.getTranslogSizeInBytes(), equalTo(firstOperationPosition));
+        }
+    }
 
-        BytesStreamOutput out = new BytesStreamOutput();
-        total.writeTo(out);
-        TranslogStats copy = new TranslogStats();
-        copy.readFrom(out.bytes().streamInput());
-
-        assertEquals(6, copy.estimatedNumberOfOperations());
-        assertEquals(419, copy.getTranslogSizeInBytes());
-
-        try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
-            builder.startObject();
-            copy.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            assertEquals("{\"translog\":{\"operations\":6,\"size_in_bytes\":419}}", builder.string());
+    public void testTotalTests() {
+        final TranslogStats total = new TranslogStats();
+        final int n = randomIntBetween(0, 16);
+        final List<TranslogStats> statsList = new ArrayList<>(n);
+        for (int i = 0; i < n; i++) {
+            final TranslogStats stats = new TranslogStats(randomIntBetween(1, 4096), randomIntBetween(1, 1 << 20));
+            statsList.add(stats);
+            total.add(stats);
         }
 
-        try {
-            new TranslogStats(1, -1);
-            fail("must be positive");
-        } catch (IllegalArgumentException ex) {
-            //all well
-        }
-        try {
-            new TranslogStats(-1, 1);
-            fail("must be positive");
-        } catch (IllegalArgumentException ex) {
-            //all well
-        }
+        assertThat(
+            total.estimatedNumberOfOperations(),
+            equalTo(statsList.stream().mapToLong(TranslogStats::estimatedNumberOfOperations).sum()));
+        assertThat(
+            total.getTranslogSizeInBytes(),
+            equalTo(statsList.stream().mapToLong(TranslogStats::getTranslogSizeInBytes).sum()));
+    }
+
+    public void testNegativeNumberOfOperations() {
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new TranslogStats(-1, 1));
+        assertThat(e, hasToString(containsString("numberOfOperations must be >= 0")));
+    }
+
+    public void testNegativeSizeInBytes() {
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new TranslogStats(1, -1));
+        assertThat(e, hasToString(containsString("translogSizeInBytes must be >= 0")));
     }
 
     public void testSnapshot() throws IOException {
@@ -402,7 +463,7 @@ public class TranslogTests extends ESTestCase {
         final Translog.Operation operation;
         final Translog.Location location;
 
-        public LocationOperation(Translog.Operation operation, Translog.Location location) {
+        LocationOperation(Translog.Operation operation, Translog.Location location) {
             this.operation = operation;
             this.location = location;
         }
@@ -465,8 +526,15 @@ public class TranslogTests extends ESTestCase {
                     assertEquals(expDelOp.version(), delOp.version());
                     assertEquals(expDelOp.versionType(), delOp.versionType());
                     break;
+                case NO_OP:
+                    final Translog.NoOp noOp = (Translog.NoOp) op;
+                    final Translog.NoOp expectedNoOp = (Translog.NoOp) expectedOp;
+                    assertThat(noOp.seqNo(), equalTo(expectedNoOp.seqNo()));
+                    assertThat(noOp.primaryTerm(), equalTo(expectedNoOp.primaryTerm()));
+                    assertThat(noOp.reason(), equalTo(expectedNoOp.reason()));
+                    break;
                 default:
-                    throw new ElasticsearchException("unsupported opType");
+                    throw new AssertionError("unsupported operation type [" + op.opType() + "]");
             }
 
         }
@@ -559,8 +627,12 @@ public class TranslogTests extends ESTestCase {
         }
     }
 
-    private Term newUid(String id) {
-        return new Term("_uid", id);
+    private Term newUid(ParsedDocument doc) {
+        return new Term("_uid", doc.uid());
+    }
+
+    private Term newUid(String uid) {
+        return new Term("_uid", uid);
     }
 
     public void testVerifyTranslogIsNotDeleted() throws IOException {
@@ -607,7 +679,9 @@ public class TranslogTests extends ESTestCase {
                     while (run.get()) {
                         long id = idGenerator.incrementAndGet();
                         final Translog.Operation op;
-                        switch (Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))]) {
+                        final Translog.Operation.Type type =
+                            Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))];
+                        switch (type) {
                             case CREATE:
                             case INDEX:
                                 op = new Translog.Index("type", "" + id, new byte[]{(byte) id});
@@ -615,8 +689,11 @@ public class TranslogTests extends ESTestCase {
                             case DELETE:
                                 op = new Translog.Delete(newUid("" + id));
                                 break;
+                            case NO_OP:
+                                op = new Translog.NoOp(id, id, Long.toString(id));
+                                break;
                             default:
-                                throw new ElasticsearchException("unknown type");
+                                throw new AssertionError("unsupported operation type [" + type + "]");
                         }
                         Translog.Location location = translog.add(op);
                         Translog.Location existing = writtenOps.put(op, location);
@@ -864,7 +941,7 @@ public class TranslogTests extends ESTestCase {
         translog.add(new Translog.Index("test", "" + translogOperations, Integer.toString(translogOperations).getBytes(Charset.forName("UTF-8"))));
 
         final Checkpoint checkpoint = Checkpoint.read(translog.location().resolve(Translog.CHECKPOINT_FILE_NAME));
-        try (final TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) {
+        try (TranslogReader reader = translog.openReader(translog.location().resolve(Translog.getFilename(translog.currentFileGeneration())), checkpoint)) {
             assertEquals(lastSynced + 1, reader.totalOperations());
             Translog.Snapshot snapshot = reader.newSnapshot();
 
@@ -1137,7 +1214,7 @@ public class TranslogTests extends ESTestCase {
         try (Translog ignored = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) {
             fail("corrupted");
         } catch (IllegalStateException ex) {
-            assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=2353, numOps=55, translogFileGeneration=2, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration=0, globalCheckpoint=-2}");
+            assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=3123, numOps=55, translogFileGeneration=2, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration=0, globalCheckpoint=-2}");
         }
         Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
         try (Translog translog = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) {
@@ -1278,7 +1355,7 @@ public class TranslogTests extends ESTestCase {
         private final Exception[] threadExceptions;
         private final Translog translog;
 
-        public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Exception[] threadExceptions) {
+        TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Exception[] threadExceptions) {
             this.translog = translog;
             this.downLatch = downLatch;
             this.opsPerThread = opsPerThread;
@@ -1293,7 +1370,8 @@ public class TranslogTests extends ESTestCase {
                 downLatch.await();
                 for (int opCount = 0; opCount < opsPerThread; opCount++) {
                     Translog.Operation op;
-                    switch (randomFrom(Translog.Operation.Type.values())) {
+                    final Translog.Operation.Type type = randomFrom(Translog.Operation.Type.values());
+                    switch (type) {
                         case CREATE:
                         case INDEX:
                             op = new Translog.Index("test", threadId + "_" + opCount,
@@ -1307,8 +1385,11 @@ public class TranslogTests extends ESTestCase {
                                 1 + randomInt(100000),
                                 randomFrom(VersionType.values()));
                             break;
+                        case NO_OP:
+                            op = new Translog.NoOp(randomNonNegativeLong(), randomNonNegativeLong(), randomAsciiOfLength(16));
+                            break;
                         default:
-                            throw new ElasticsearchException("not supported op type");
+                            throw new AssertionError("unsupported operation type [" + type + "]");
                     }
 
                     Translog.Location loc = add(op);
@@ -1929,4 +2010,48 @@ public class TranslogTests extends ESTestCase {
     public static Translog.Location randomTranslogLocation() {
         return new Translog.Location(randomLong(), randomLong(), randomInt());
     }
+
+    public void testTranslogOpSerialization() throws Exception {
+        BytesReference B_1 = new BytesArray(new byte[]{1});
+        SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
+        assert Version.CURRENT.major <= 6 : "Using UNASSIGNED_SEQ_NO can be removed in 7.0, because 6.0+ nodes have actual sequence numbers";
+        long randomSeqNum = randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong();
+        long randomPrimaryTerm = randomBoolean() ? 0 : randomNonNegativeLong();
+        seqID.seqNo.setLongValue(randomSeqNum);
+        seqID.seqNoDocValue.setLongValue(randomSeqNum);
+        seqID.primaryTerm.setLongValue(randomPrimaryTerm);
+        Field uidField = new Field("_uid", Uid.createUid("test", "1"), UidFieldMapper.Defaults.FIELD_TYPE);
+        Field versionField = new NumericDocValuesField("_version", 1);
+        Document document = new Document();
+        document.add(new TextField("value", "test", Field.Store.YES));
+        document.add(uidField);
+        document.add(versionField);
+        document.add(seqID.seqNo);
+        document.add(seqID.seqNoDocValue);
+        document.add(seqID.primaryTerm);
+        ParsedDocument doc = new ParsedDocument(versionField, seqID, "1", "type", null, Arrays.asList(document), B_1, XContentType.JSON,
+            null);
+
+        Engine.Index eIndex = new Engine.Index(newUid(doc), doc, randomSeqNum, randomPrimaryTerm,
+                1, VersionType.INTERNAL, Origin.PRIMARY, 0, 0, false);
+        Engine.IndexResult eIndexResult = new Engine.IndexResult(1, randomSeqNum, true);
+        Translog.Index index = new Translog.Index(eIndex, eIndexResult);
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        index.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        Translog.Index serializedIndex = new Translog.Index(in);
+        assertEquals(index, serializedIndex);
+
+        Engine.Delete eDelete = new Engine.Delete(doc.type(), doc.id(), newUid(doc), randomSeqNum, randomPrimaryTerm,
+                2, VersionType.INTERNAL, Origin.PRIMARY, 0);
+        Engine.DeleteResult eDeleteResult = new Engine.DeleteResult(2, randomSeqNum, true);
+        Translog.Delete delete = new Translog.Delete(eDelete, eDeleteResult);
+
+        out = new BytesStreamOutput();
+        delete.writeTo(out);
+        in = out.bytes().streamInput();
+        Translog.Delete serializedDelete = new Translog.Delete(in);
+        assertEquals(delete, serializedDelete);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
index bb8943f19d0..4ef26b9357c 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MockEngineFactoryPlugin;
@@ -112,7 +113,7 @@ public class TruncateTranslogIT extends ESIntegTestCase {
             // Try running it before the shard is closed, it should flip out because it can't acquire the lock
             try {
                 logger.info("--> running truncate while index is open on [{}]", translogDir.toAbsolutePath());
-                ttc.execute(t, options, new HashMap<String, String>());
+                ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
                 fail("expected the truncate command to fail not being able to acquire the lock");
             } catch (Exception e) {
                 assertThat(e.getMessage(), containsString("Failed to lock shard's directory"));
@@ -160,7 +161,7 @@ public class TruncateTranslogIT extends ESIntegTestCase {
 
             OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString(), "-b");
             logger.info("--> running truncate translog command for [{}]", translogDir.toAbsolutePath());
-            ttc.execute(t, options, new HashMap<String, String>());
+            ttc.execute(t, options, null /* TODO: env should be real here, and ttc should actually use it... */);
             logger.info("--> output:\n{}", t.getOutput());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
index 8682d8127ae..43326116f15 100644
--- a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
@@ -257,21 +257,7 @@ public class IndexActionIT extends ESIntegTestCase {
             }
         );
         assertThat(e.getMessage(), containsString("failed to parse"));
-        assertThat(e.getRootCause().getMessage(), containsString("name cannot be empty string"));
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class); // uses index.version.created
-    }
-
-    public void testDocumentWithBlankFieldName2x() {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_4);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        assertAcked(prepareCreate("test1").setSettings(settings));
-        ensureGreen();
-
-        IndexResponse indexResponse = client().prepareIndex("test1", "type", "1").setSource("", "value1_2").execute().actionGet();
-        assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
+        assertThat(e.getRootCause().getMessage(),
+                containsString("object field starting or ending with a [.] makes object resolution ambiguous: []"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
index 82482b1f821..fe4ab2e363a 100644
--- a/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -57,9 +58,9 @@ public class DateMathIndexExpressionsIntegrationIT extends ESIntegTestCase {
         String dateMathExp1 = "<.marvel-{now/d}>";
         String dateMathExp2 = "<.marvel-{now/d-1d}>";
         String dateMathExp3 = "<.marvel-{now/d-2d}>";
-        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
-        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
-        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
+        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get();
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
@@ -116,9 +117,9 @@ public class DateMathIndexExpressionsIntegrationIT extends ESIntegTestCase {
         String dateMathExp1 = "<.marvel-{now/d}>";
         String dateMathExp2 = "<.marvel-{now/d-1d}>";
         String dateMathExp3 = "<.marvel-{now/d-2d}>";
-        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
-        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
-        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
+        client().prepareIndex(dateMathExp1, "type", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp2, "type", "2").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get();
         refresh();
 
         SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index e9a717f6636..fb524f27591 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
@@ -64,7 +65,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         // Shards that are currently throttled
         final Set<IndexShard> throttled = new HashSet<>();
 
-        public MockController(Settings settings) {
+        MockController(Settings settings) {
             super(Settings.builder()
                             .put("indices.memory.interval", "200h") // disable it
                             .put(settings)
@@ -423,7 +424,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
         for (int i = 0; i < 100; i++) {
-            client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}").get();
+            client().prepareIndex("test", "test", Integer.toString(i)).setSource("{\"foo\" : \"bar\"}", XContentType.JSON).get();
         }
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index 5d5584a156f..3c0674609c5 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -169,7 +169,6 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
         assertTrue(loader.loadedFromCache);
-        assertTrue(loader.loadedFromCache);
         assertEquals(1, cache.count());
         assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
         assertEquals(1, cache.numRegisteredCloseListeners());
@@ -186,7 +185,6 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
         assertTrue(loader.loadedFromCache);
-        assertTrue(loader.loadedFromCache);
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
@@ -215,7 +213,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
                 new ShardId("foo", "bar", 1));
             TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
             Loader secondLoader = new Loader(secondReader, 0);
-            
+
             BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
             assertEquals("foo", value1.streamInput().readString());
             BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
@@ -324,7 +322,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         private final int id;
         public boolean loadedFromCache = true;
 
-        public Loader(DirectoryReader reader, int id) {
+        Loader(DirectoryReader reader, int id) {
             super();
             this.reader = reader;
             this.id = id;
@@ -347,6 +345,74 @@ public class IndicesRequestCacheTests extends ESTestCase {
 
     }
 
+    public void testInvalidate() throws Exception {
+        ShardRequestCache requestCacheStats = new ShardRequestCache();
+        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
+            new ShardId("foo", "bar", 1));
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
+        AtomicBoolean indexShard = new AtomicBoolean(true);
+
+        // initial cache
+        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
+        Loader loader = new Loader(reader, 0);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(0, requestCacheStats.stats().getHitCount());
+        assertEquals(1, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertFalse(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+
+        // cache hit
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(1, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertTrue(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+        assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
+        assertEquals(1, cache.numRegisteredCloseListeners());
+
+        // load again after invalidate
+        entity = new TestEntity(requestCacheStats, indexShard);
+        loader = new Loader(reader, 0);
+        cache.invalidate(entity, reader,  termQuery.buildAsBytes());
+        value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(2, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertFalse(loader.loadedFromCache);
+        assertEquals(1, cache.count());
+        assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
+        assertEquals(1, cache.numRegisteredCloseListeners());
+
+        // release
+        if (randomBoolean()) {
+            reader.close();
+        } else {
+            indexShard.set(false); // closed shard but reader is still open
+            cache.clear(entity);
+        }
+        cache.cleanCache();
+        assertEquals(1, requestCacheStats.stats().getHitCount());
+        assertEquals(2, requestCacheStats.stats().getMissCount());
+        assertEquals(0, requestCacheStats.stats().getEvictions());
+        assertEquals(0, cache.count());
+        assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
+
+        IOUtils.close(reader, writer, dir, cache);
+        assertEquals(0, cache.numRegisteredCloseListeners());
+    }
+
     private class TestEntity extends AbstractIndexShardCacheEntity {
         private final AtomicBoolean standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index 6f379e48bae..61d325a64e8 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.Analysis;
@@ -136,6 +137,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(EnglishAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasReferencesAlias() throws IOException {
@@ -154,6 +159,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(GermanAnalyzer.class)));
         // analyzer types are bound early before we resolve aliases
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(StandardAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasDefault() throws IOException {
@@ -168,6 +177,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
         IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);
         assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
         assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasMoreThanOnce() throws IOException {
@@ -183,6 +194,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         AnalysisRegistry newRegistry = getNewRegistry(settings);
         IllegalStateException ise = expectThrows(IllegalStateException.class, () -> getIndexAnalyzers(newRegistry, settings));
         assertEquals("alias [default] is already used by [foobar]", ise.getMessage());
+        assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
+                "5.x; analyzer aliases can no longer be created on new indices.",
+                "setting [index.analysis.analyzer.foobar1.alias] is only allowed on index [test] because it was created before " +
+                        "5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testAnalyzerAliasNotAllowedPost5x() throws IOException {
@@ -292,7 +307,7 @@ public class AnalysisModuleTests extends ModuleTestCase {
 //        assertThat(dictionaryDecompounderAnalyze.tokenFilters().length, equalTo(1));
 //        assertThat(dictionaryDecompounderAnalyze.tokenFilters()[0], instanceOf(DictionaryCompoundWordTokenFilterFactory.class));
 
-        Set<?> wordList = Analysis.getWordSet(null, settings, "index.analysis.filter.dict_dec.word_list");
+        Set<?> wordList = Analysis.getWordSet(null, Version.CURRENT, settings, "index.analysis.filter.dict_dec.word_list");
         MatcherAssert.assertThat(wordList.size(), equalTo(6));
 //        MatcherAssert.assertThat(wordList, hasItems("donau", "dampf", "schiff", "spargel", "creme", "suppe"));
     }
@@ -305,9 +320,10 @@ public class AnalysisModuleTests extends ModuleTestCase {
         String[] words = new String[]{"donau", "dampf", "schiff", "spargel", "creme", "suppe"};
 
         Path wordListFile = generateWordList(words);
-        settings = Settings.builder().loadFromSource("index: \n  word_list_path: " + wordListFile.toAbsolutePath()).build();
+        settings = Settings.builder().loadFromSource("index: \n  word_list_path: " + wordListFile.toAbsolutePath(), XContentType.YAML)
+            .build();
 
-        Set<?> wordList = Analysis.getWordSet(env, settings, "index.word_list");
+        Set<?> wordList = Analysis.getWordSet(env, Version.CURRENT, settings, "index.word_list");
         MatcherAssert.assertThat(wordList.size(), equalTo(6));
 //        MatcherAssert.assertThat(wordList, hasItems(words));
         Files.delete(wordListFile);
@@ -353,6 +369,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("analyzer name must not start with '_'. got \"_invalid_name\""));
         }
+        assertWarnings("setting [index.analysis.analyzer.valid_name.alias] is only allowed on index [test] because it was " +
+                "created before 5.x; analyzer aliases can no longer be created on new indices.");
     }
 
     public void testDeprecatedPositionOffsetGap() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
index b79c7a31392..a3e6afc1ac6 100644
--- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
@@ -50,21 +50,25 @@ public class AnalyzeActionIT extends ESIntegTestCase {
             assertThat(token.getStartOffset(), equalTo(0));
             assertThat(token.getEndOffset(), equalTo(4));
             assertThat(token.getPosition(), equalTo(0));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(1);
             assertThat(token.getTerm(), equalTo("is"));
             assertThat(token.getStartOffset(), equalTo(5));
             assertThat(token.getEndOffset(), equalTo(7));
             assertThat(token.getPosition(), equalTo(1));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(2);
             assertThat(token.getTerm(), equalTo("a"));
             assertThat(token.getStartOffset(), equalTo(8));
             assertThat(token.getEndOffset(), equalTo(9));
             assertThat(token.getPosition(), equalTo(2));
+            assertThat(token.getPositionLength(), equalTo(1));
             token = analyzeResponse.getTokens().get(3);
             assertThat(token.getTerm(), equalTo("test"));
             assertThat(token.getStartOffset(), equalTo(10));
             assertThat(token.getEndOffset(), equalTo(14));
             assertThat(token.getPosition(), equalTo(3));
+            assertThat(token.getPositionLength(), equalTo(1));
         }
     }
 
@@ -104,7 +108,7 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1));
         assertThat(analyzeResponse.getTokens().get(0).getStartOffset(), equalTo(3));
         assertThat(analyzeResponse.getTokens().get(0).getEndOffset(), equalTo(9));
-
+        assertThat(analyzeResponse.getTokens().get(0).getPositionLength(), equalTo(1));
     }
 
     public void testAnalyzeWithCharFilters() throws Exception {
@@ -137,6 +141,54 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getTerm(), equalTo("fish"));
     }
 
+    public void testAnalyzeWithNonDefaultPostionLength() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+            .setSettings(Settings.builder().put(indexSettings())
+                .put("index.analysis.filter.syns.type", "synonym")
+                .putArray("index.analysis.filter.syns.synonyms", "wtf, what the fudge")
+                .put("index.analysis.analyzer.custom_syns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.custom_syns.filter", "lowercase", "syns")));
+        ensureGreen();
+
+        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("say what the fudge").setIndex("test").setAnalyzer("custom_syns").get();
+        assertThat(analyzeResponse.getTokens().size(), equalTo(5));
+
+        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
+        assertThat(token.getTerm(), equalTo("say"));
+        assertThat(token.getPosition(), equalTo(0));
+        assertThat(token.getStartOffset(), equalTo(0));
+        assertThat(token.getEndOffset(), equalTo(3));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(1);
+        assertThat(token.getTerm(), equalTo("what"));
+        assertThat(token.getPosition(), equalTo(1));
+        assertThat(token.getStartOffset(), equalTo(4));
+        assertThat(token.getEndOffset(), equalTo(8));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(2);
+        assertThat(token.getTerm(), equalTo("wtf"));
+        assertThat(token.getPosition(), equalTo(1));
+        assertThat(token.getStartOffset(), equalTo(4));
+        assertThat(token.getEndOffset(), equalTo(18));
+        assertThat(token.getPositionLength(), equalTo(3));
+
+        token = analyzeResponse.getTokens().get(3);
+        assertThat(token.getTerm(), equalTo("the"));
+        assertThat(token.getPosition(), equalTo(2));
+        assertThat(token.getStartOffset(), equalTo(9));
+        assertThat(token.getEndOffset(), equalTo(12));
+        assertThat(token.getPositionLength(), equalTo(1));
+
+        token = analyzeResponse.getTokens().get(4);
+        assertThat(token.getTerm(), equalTo("fudge"));
+        assertThat(token.getPosition(), equalTo(3));
+        assertThat(token.getStartOffset(), equalTo(13));
+        assertThat(token.getEndOffset(), equalTo(18));
+        assertThat(token.getPositionLength(), equalTo(1));
+    }
+
     public void testAnalyzerWithFieldOrTypeTests() throws Exception {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
         ensureGreen();
@@ -154,6 +206,7 @@ public class AnalyzeActionIT extends ESIntegTestCase {
             assertThat(token.getTerm(), equalTo("test"));
             assertThat(token.getStartOffset(), equalTo(10));
             assertThat(token.getEndOffset(), equalTo(14));
+            assertThat(token.getPositionLength(), equalTo(1));
         }
     }
 
@@ -200,13 +253,14 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(14));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.getTokens().get(5);
         assertThat(token.getTerm(), equalTo("second"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(19));
         assertThat(token.getEndOffset(), equalTo(25));
-
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testDetailAnalyze() throws Exception {
@@ -350,12 +404,14 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(14));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().analyzer().getTokens()[5];
         assertThat(token.getTerm(), equalTo("second"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(19));
         assertThat(token.getEndOffset(), equalTo(25));
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testDetailAnalyzeWithMultiValuesWithCustomAnalyzer() throws Exception {
@@ -395,12 +451,14 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenizer().getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubled"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         // tokenfilter(snowball)
         assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2));
@@ -412,12 +470,14 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubl"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         // tokenfilter(lowercase)
         assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("lowercase"));
@@ -428,14 +488,14 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(token.getPosition(), equalTo(3));
         assertThat(token.getStartOffset(), equalTo(10));
         assertThat(token.getEndOffset(), equalTo(15));
+        assertThat(token.getPositionLength(), equalTo(1));
 
         token = analyzeResponse.detail().tokenfilters()[0].getTokens()[5];
         assertThat(token.getTerm(), equalTo("troubl"));
         assertThat(token.getPosition(), equalTo(105));
         assertThat(token.getStartOffset(), equalTo(20));
         assertThat(token.getEndOffset(), equalTo(28));
-
-
+        assertThat(token.getPositionLength(), equalTo(1));
     }
 
     public void testNonExistTokenizer() {
@@ -468,16 +528,19 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("buzz"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(8));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("test"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1));
 
         // tokenfilter(lowercase)
         assertThat(analyzeResponse.detail().tokenfilters().length, equalTo(2));
@@ -487,16 +550,19 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getTerm(), equalTo("buzz"));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getStartOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getEndOffset(), equalTo(8));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getTerm(), equalTo("test"));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenfilters()[0].getTokens()[2].getPositionLength(), equalTo(1));
 
         // tokenfilter({"type": "stop", "stopwords": ["foo", "buzz"]})
         assertThat(analyzeResponse.detail().tokenfilters()[1].getName(), equalTo("_anonymous_tokenfilter_[1]"));
@@ -506,6 +572,7 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getStartOffset(), equalTo(9));
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getEndOffset(), equalTo(13));
         assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenfilters()[1].getTokens()[0].getPositionLength(), equalTo(1));
     }
 
 
@@ -533,6 +600,7 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getTerm(), equalTo("jeff qit fish"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(15));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
     }
 
 
@@ -556,16 +624,19 @@ public class AnalyzeActionIT extends ESIntegTestCase {
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getStartOffset(), equalTo(0));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getEndOffset(), equalTo(2));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPosition(), equalTo(0));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[0].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getTerm(), equalTo("oo"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getStartOffset(), equalTo(1));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getEndOffset(), equalTo(3));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPosition(), equalTo(1));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[1].getPositionLength(), equalTo(1));
 
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getTerm(), equalTo("od"));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getStartOffset(), equalTo(2));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getEndOffset(), equalTo(4));
         assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPosition(), equalTo(2));
+        assertThat(analyzeResponse.detail().tokenizer().getTokens()[2].getPositionLength(), equalTo(1));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index d1d266a42a4..470772d511c 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -52,6 +52,7 @@ import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedShard;
@@ -61,11 +62,12 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService;
@@ -112,7 +114,7 @@ public class ClusterStateChanges extends AbstractComponent {
     private final TransportClusterRerouteAction transportClusterRerouteAction;
     private final TransportCreateIndexAction transportCreateIndexAction;
 
-    public ClusterStateChanges() {
+    public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool threadPool) {
         super(Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build());
 
         allocationService = new AllocationService(settings, new AllocationDeciders(settings,
@@ -128,7 +130,6 @@ public class ClusterStateChanges extends AbstractComponent {
         IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
         DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterSettings);
         Environment environment = new Environment(settings);
-        ThreadPool threadPool = null; // it's not used
         Transport transport = null; // it's not used
 
         // mocks
@@ -155,8 +156,9 @@ public class ClusterStateChanges extends AbstractComponent {
 
         // services
         TransportService transportService = new TransportService(settings, transport, threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, clusterSettings);
-        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, null, null) {
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings);
+        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null) {
             // metaData upgrader should do nothing
             @Override
             public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
@@ -167,10 +169,10 @@ public class ClusterStateChanges extends AbstractComponent {
             metaDataIndexUpgradeService, indicesService);
         MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService);
         MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
-            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService);
+            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, threadPool);
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
             allocationService, new AliasValidator(settings), environment,
-            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool);
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry);
 
         transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool,
             indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations);
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 4b21b8820c7..37f6c989229 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -47,6 +47,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -74,7 +75,30 @@ import static org.mockito.Mockito.when;
 
 public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndicesClusterStateServiceTestCase {
 
-    private final ClusterStateChanges cluster = new ClusterStateChanges();
+    private ThreadPool threadPool;
+    private ClusterStateChanges cluster;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+        cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        terminate(threadPool);
+    }
+
+    /**
+     * needed due to random usage of {@link IndexMetaData#INDEX_SHADOW_REPLICAS_SETTING}. removed once
+     * shadow replicas are removed.
+     */
+    @Override
+    protected boolean enableWarningsCheck() {
+        return false;
+    }
 
     public void testRandomClusterStateUpdates() {
         // we have an IndicesClusterStateService per node in the cluster
@@ -368,7 +392,8 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         final MockIndicesService indicesService = indicesServiceSupplier.get();
         final Settings settings = Settings.builder().put("node.name", discoveryNode.getName()).build();
         final TransportService transportService = new TransportService(settings, null, threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR,
+            boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), null);
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(settings, clusterService,
             transportService, null);
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index d974ea348cf..50979cf611d 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationComman
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.engine.Engine;
@@ -57,7 +58,7 @@ public class FlushIT extends ESIntegTestCase {
         final int numIters = scaledRandomIntBetween(10, 30);
         for (int i = 0; i < numIters; i++) {
             for (int j = 0; j < 10; j++) {
-                client().prepareIndex("test", "test").setSource("{}").get();
+                client().prepareIndex("test", "test").setSource("{}", XContentType.JSON).get();
             }
             final CountDownLatch latch = new CountDownLatch(10);
             final CopyOnWriteArrayList<Throwable> errors = new CopyOnWriteArrayList<>();
@@ -164,7 +165,7 @@ public class FlushIT extends ESIntegTestCase {
             @Override
             public void run() {
                 while (stop.get() == false) {
-                    client().prepareIndex().setIndex("test").setType("doc").setSource("{}").get();
+                    client().prepareIndex().setIndex("test").setType("doc").setSource("{}", XContentType.JSON).get();
                     numDocs.incrementAndGet();
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
index c35df81cfd6..59784456e98 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
@@ -42,7 +43,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testModificationPreventsFlushing() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -54,7 +55,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
         assertEquals("exactly one active shard", 1, activeShards.size());
         Map<String, Engine.CommitId> commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId);
         assertEquals("exactly one commit id", 1, commitIds.size());
-        client().prepareIndex("test", "test", "2").setSource("{}").get();
+        client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get();
         String syncId = UUIDs.base64UUID();
         SyncedFlushUtil.LatchedListener<ShardsSyncedFlushResult> listener = new SyncedFlushUtil.LatchedListener<>();
         flushService.sendSyncRequests(syncId, activeShards, state, commitIds, shardId, shardRoutingTable.size(), listener);
@@ -86,7 +87,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testSingleShardSuccess() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -106,7 +107,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -161,7 +162,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testFailAfterIntermediateCommit() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
@@ -174,7 +175,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
         Map<String, Engine.CommitId> commitIds = SyncedFlushUtil.sendPreSyncRequests(flushService, activeShards, state, shardId);
         assertEquals("exactly one commit id", 1, commitIds.size());
         if (randomBoolean()) {
-            client().prepareIndex("test", "test", "2").setSource("{}").get();
+            client().prepareIndex("test", "test", "2").setSource("{}", XContentType.JSON).get();
         }
         client().admin().indices().prepareFlush("test").setForce(true).get();
         String syncId = UUIDs.base64UUID();
@@ -194,7 +195,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
     public void testFailWhenCommitIsMissing() throws InterruptedException {
         createIndex("test");
-        client().prepareIndex("test", "test", "1").setSource("{}").get();
+        client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
 
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index ed371eca895..484f6e5db76 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -40,6 +41,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.indices.breaker.BreakerSettings;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.CircuitBreakerStats;
@@ -48,6 +50,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.junit.After;
 import org.junit.Before;
 
@@ -152,8 +155,11 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
         // execute a search that loads field data (sorting on the "test" field)
         // again, this time it should trip the breaker
         SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
-        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [100/100b]"));
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [100/100b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
         int breaks = 0;
@@ -173,7 +179,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
 
         // Create an index where the mappings have a field data filter
         assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " +
-                "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}"));
+                "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", XContentType.JSON));
 
         ensureGreen("ramtest");
 
@@ -200,9 +206,12 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
 
         // execute a search that loads field data (sorting on the "test" field)
         // again, this time it should trip the breaker
-        assertFailures(client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC),
-                RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [100/100b]"));
+        SearchRequestBuilder searchRequest = client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [100/100b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
         int breaks = 0;
@@ -217,6 +226,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
      * Test that a breaker correctly redistributes to a different breaker, in
      * this case, the fielddata breaker borrows space from the request breaker
      */
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.search:TRACE")
     public void testParentChecking() throws Exception {
         if (noopBreakerUsed()) {
             logger.info("--> noop breakers used, skipping test");
@@ -245,14 +255,20 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
             client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
             fail("should have thrown an exception");
         } catch (Exception e) {
-            String errMsg = "[fielddata] Data too large, data for [test] would be larger than limit of [10/10b]";
-            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
-                e.toString(), containsString(errMsg));
+            String errMsg = "CircuitBreakingException[[fielddata] Data too large, data for [test] would be";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [10/10b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
         }
 
-        assertFailures(client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC),
-                RestStatus.INTERNAL_SERVER_ERROR,
-                containsString("Data too large, data for [test] would be larger than limit of [10/10b]"));
+        // execute a search that loads field data (sorting on the "test" field)
+        // again, this time it should trip the breaker
+        SearchRequestBuilder searchRequest = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
+
+        String errMsg = "Data too large, data for [test] would be";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
+        errMsg = "which is larger than the limit of [10/10b]";
+        assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
 
         reset();
 
@@ -316,9 +332,8 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
             fail("aggregation should have tripped the breaker");
         } catch (Exception e) {
             String errMsg = "CircuitBreakingException[[request] Data too large";
-            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
-                e.toString(), containsString(errMsg));
-            errMsg = "would be larger than limit of [10/10b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [10/10b]]";
             assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException", e.toString(), containsString(errMsg));
         }
     }
@@ -354,9 +369,11 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
             assertTrue("there should be shard failures", resp.getFailedShards() > 0);
             fail("aggregation should have tripped the breaker");
         } catch (Exception e) {
-            String errMsg = "CircuitBreakingException[[request] " +
-                    "Data too large, data for [<agg [my_terms]>] would be larger than limit of [100/100b]]";
-            assertThat("Exception: " + e.toString() + " should contain a CircuitBreakingException",
+            String errMsg = "CircuitBreakingException[[request] Data too large, data for [<agg [my_terms]>] would be";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
+                    e.toString(), containsString(errMsg));
+            errMsg = "which is larger than the limit of [100/100b]]";
+            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
                     e.toString(), containsString(errMsg));
         }
     }
@@ -465,7 +482,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
             IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i));
-            indexRequest.source("field", "value", "num", i);
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i);
             bulkRequest.add(indexRequest);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/EvilPeerRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/EvilPeerRecoveryIT.java
new file mode 100644
index 00000000000..913742dcbf1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/EvilPeerRecoveryIT.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.recovery;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.plugins.AnalysisPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class EvilPeerRecoveryIT extends ESIntegTestCase {
+
+    private static AtomicReference<CountDownLatch> indexLatch = new AtomicReference<>();
+    private static AtomicReference<CountDownLatch> waitForOpsToCompleteLatch = new AtomicReference<>();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(LatchAnalysisPlugin.class);
+    }
+
+    public static class LatchAnalysisPlugin extends Plugin implements AnalysisPlugin {
+
+        @Override
+        public Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+            return Collections.singletonMap("latch_analyzer", (a, b, c, d) -> new LatchAnalyzerProvider());
+        }
+
+    }
+
+    static class LatchAnalyzerProvider implements AnalyzerProvider<LatchAnalyzer> {
+
+        @Override
+        public String name() {
+            return "latch_analyzer";
+        }
+
+        @Override
+        public AnalyzerScope scope() {
+            return AnalyzerScope.INDICES;
+        }
+
+        @Override
+        public LatchAnalyzer get() {
+            return new LatchAnalyzer();
+        }
+
+    }
+
+    static class LatchAnalyzer extends Analyzer {
+
+        @Override
+        protected TokenStreamComponents createComponents(final String fieldName) {
+            return new TokenStreamComponents(new LatchTokenizer());
+        }
+
+    }
+
+    static class LatchTokenizer extends Tokenizer {
+
+        @Override
+        public final boolean incrementToken() throws IOException {
+            try {
+                if (indexLatch.get() != null) {
+                    // latch that all exected operations are in the engine
+                    indexLatch.get().countDown();
+                }
+
+                if (waitForOpsToCompleteLatch.get() != null) {
+                    // latch that waits for the replica to restart and allows recovery to proceed
+                    waitForOpsToCompleteLatch.get().await();
+                }
+
+            } catch (final InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            return false;
+        }
+
+    }
+
+    /*
+     * This tests that sequence-number-based recoveries wait for in-flight operations to complete. The trick here is simple. We latch some
+     * in-flight operations inside the engine after sequence numbers are assigned. While these operations are latched, we restart a replica.
+     * Sequence-number-based recovery on this replica has to wait until these in-flight operations complete to proceed. We verify at the end
+     * of recovery that a file-based recovery was not completed, and that the expected number of operations was replayed via the translog.
+     */
+    @TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.get:TRACE," +
+        "org.elasticsearch.discovery:TRACE," +
+        "org.elasticsearch.cluster.service:TRACE,org.elasticsearch.indices.recovery:TRACE," +
+        "org.elasticsearch.indices.cluster:TRACE,org.elasticsearch.index.shard:TRACE")
+    @AwaitsFix(bugUrl =
+        "boaz is looking into failures: https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+java9-periodic/1545")
+    public void testRecoveryWaitsForOps() throws Exception {
+        final int docs = randomIntBetween(1, 64);
+        try {
+            internalCluster().startMasterOnlyNode();
+            final String primaryNode = internalCluster().startDataOnlyNode(nodeSettings(0));
+
+            // prepare mapping that uses our latch analyzer
+            final XContentBuilder mapping = jsonBuilder();
+            mapping.startObject();
+            {
+                mapping.startObject("type");
+                {
+                    mapping.startObject("properties");
+                    {
+                        mapping.startObject("foo");
+                        {
+                            mapping.field("type", "text");
+                            mapping.field("analyzer", "latch_analyzer");
+                            mapping.endObject();
+                        }
+                        mapping.endObject();
+                    }
+                    mapping.endObject();
+                }
+                mapping.endObject();
+            }
+
+            // create the index with our mapping
+            client()
+                .admin()
+                .indices()
+                .prepareCreate("index")
+                .addMapping("type", mapping)
+                .setSettings(Settings.builder().put("number_of_shards", 1))
+                .get();
+
+            // start the replica node; we do this after creating the index so we can control which node is holds the primary shard
+            final String replicaNode = internalCluster().startDataOnlyNode(nodeSettings(1));
+            ensureGreen();
+
+            // index some documents so that the replica will attempt a sequence-number-based recovery upon restart
+            for (int foo = 0; foo < docs; foo++) {
+                index(randomFrom(primaryNode, replicaNode), foo);
+            }
+
+            if (randomBoolean()) {
+                client().admin().indices().flush(new FlushRequest()).get();
+            }
+
+            // start some in-flight operations that will get latched in the engine
+            final List<Thread> threads = new ArrayList<>();
+            final int latchedDocs = internalCluster().getInstance(ThreadPool.class, replicaNode).info(ThreadPool.Names.BULK).getMax();
+            indexLatch.set(new CountDownLatch(latchedDocs));
+            waitForOpsToCompleteLatch.set(new CountDownLatch(1));
+            for (int i = docs; i < docs + latchedDocs; i++) {
+                final int foo = i;
+                // we have to index through the primary since we are going to restart the replica
+                final Thread thread = new Thread(() -> index(primaryNode, foo));
+                threads.add(thread);
+                thread.start();
+            }
+
+            // latch until all operations are inside the engine
+            indexLatch.get().await();
+
+            internalCluster().restartNode(replicaNode, new InternalTestCluster.RestartCallback());
+
+            final Index index = resolveIndex("index");
+
+            // wait until recovery starts
+            assertBusy(() -> {
+                    final IndicesService primaryService = internalCluster().getInstance(IndicesService.class, primaryNode);
+                    assertThat(primaryService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
+                    final IndicesService replicaService = internalCluster().getInstance(IndicesService.class, replicaNode);
+                    assertThat(replicaService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
+                }
+            );
+
+            // unlatch the operations that are latched inside the engine
+            waitForOpsToCompleteLatch.get().countDown();
+
+            for (final Thread thread : threads) {
+                thread.join();
+            }
+
+            // recovery should complete successfully
+            ensureGreen();
+
+            // verify that a sequence-number-based recovery was completed
+            final org.elasticsearch.action.admin.indices.recovery.RecoveryResponse response =
+                client().admin().indices().prepareRecoveries("index").get();
+            final List<RecoveryState> states = response.shardRecoveryStates().get("index");
+            for (final RecoveryState state : states) {
+                if (state.getTargetNode().getName().equals(replicaNode)) {
+                    assertThat(state.getTranslog().recoveredOperations(), equalTo(latchedDocs));
+                    assertThat(state.getIndex().recoveredFilesPercent(), equalTo(0f));
+                }
+            }
+        } finally {
+            internalCluster().close();
+        }
+
+    }
+
+    private void index(final String node, final int foo) {
+        client(node).prepareIndex("index", "type").setSource("{\"foo\":\"" + Integer.toString(foo) + "\"}").get();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
index 95067992eed..8313160046b 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexPrimaryRelocationIT.java
@@ -44,7 +44,7 @@ public class IndexPrimaryRelocationIT extends ESIntegTestCase {
 
     private static final int RELOCATION_COUNT = 25;
 
-    @TestLogging("_root:DEBUG,org.elasticsearch.action.delete:TRACE,org.elasticsearch.action.index:TRACE,index.shard:TRACE,org.elasticsearch.cluster.service:TRACE")
+    @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
     public void testPrimaryRelocationWhileIndexing() throws Exception {
         internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3));
         client().admin().indices().prepareCreate("test")
@@ -97,4 +97,5 @@ public class IndexPrimaryRelocationIT extends ESIntegTestCase {
         ElasticsearchAssertions.assertHitCount(client().prepareSearch("test")// extra paranoia ;)
             .setQuery(QueryBuilders.termQuery("auto", true)).get(), numAutoGenDocs.get());
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 4e3c516aa72..5a7f983b056 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -31,8 +31,8 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource;
 import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
@@ -42,6 +42,8 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.store.Store;
@@ -55,12 +57,12 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.store.MockFSDirectoryService;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -73,6 +75,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -573,7 +577,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         int numDocs = scaledRandomIntBetween(25, 250);
         for (int i = 0; i < numDocs; i++) {
-            requests.add(client().prepareIndex(indexName, "type").setSource("{}"));
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
         }
         indexRandom(true, requests);
         ensureSearchable(indexName);
@@ -632,7 +636,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         private final String recoveryActionToBlock;
         private final CountDownLatch requestBlocked;
 
-        public RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) {
+        RecoveryActionBlocker(boolean dropRequests, String recoveryActionToBlock, Transport delegate, CountDownLatch requestBlocked) {
             super(delegate);
             this.dropRequests = dropRequests;
             this.recoveryActionToBlock = recoveryActionToBlock;
@@ -653,4 +657,128 @@ public class IndexRecoveryIT extends ESIntegTestCase {
             super.sendRequest(connection, requestId, action, request, options);
         }
     }
+
+    /**
+     * Tests scenario where recovery target successfully sends recovery request to source but then the channel gets closed while
+     * the source is working on the recovery process.
+     */
+    @TestLogging("_root:DEBUG,org.elasticsearch.indices.recovery:TRACE")
+    public void testDisconnectsDuringRecovery() throws Exception {
+        boolean primaryRelocation = randomBoolean();
+        final String indexName = "test";
+        final Settings nodeSettings = Settings.builder()
+            .put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(0, 100)))
+            .build();
+        TimeValue disconnectAfterDelay = TimeValue.timeValueMillis(randomIntBetween(0, 100));
+        // start a master node
+        String masterNodeName = internalCluster().startMasterOnlyNode(nodeSettings);
+
+        final String blueNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
+        final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
+
+        client().admin().indices().prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "blue")
+                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            ).get();
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        int numDocs = scaledRandomIntBetween(25, 250);
+        for (int i = 0; i < numDocs; i++) {
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
+        }
+        indexRandom(true, requests);
+        ensureSearchable(indexName);
+        assertHitCount(client().prepareSearch(indexName).get(), numDocs);
+
+        MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterNodeName);
+        MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName);
+        MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName);
+
+        redMockTransportService.addDelegate(blueMockTransportService, new MockTransportService.DelegateTransport(redMockTransportService.original()) {
+            private final AtomicInteger count = new AtomicInteger();
+
+            @Override
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
+                logger.info("--> sending request {} on {}", action, connection.getNode());
+                if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action) && count.incrementAndGet() == 1) {
+                    // ensures that it's considered as valid recovery attempt by source
+                    try {
+                        awaitBusy(() -> client(blueNodeName).admin().cluster().prepareState().setLocal(true).get()
+                            .getState().getRoutingTable().index("test").shard(0).getAllInitializingShards().isEmpty() == false);
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                    super.sendRequest(connection, requestId, action, request, options);
+                    try {
+                        Thread.sleep(disconnectAfterDelay.millis());
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+                    throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulation disconnect after successfully sending " + action + " request");
+                } else {
+                    super.sendRequest(connection, requestId, action, request, options);
+                }
+            }
+        });
+
+        final AtomicBoolean finalized = new AtomicBoolean();
+        blueMockTransportService.addDelegate(redMockTransportService, new MockTransportService.DelegateTransport(blueMockTransportService.original()) {
+            @Override
+            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                       TransportRequestOptions options) throws IOException {
+                logger.info("--> sending request {} on {}", action, connection.getNode());
+                if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
+                    finalized.set(true);
+                }
+                super.sendRequest(connection, requestId, action, request, options);
+            }
+        });
+
+        for (MockTransportService mockTransportService : Arrays.asList(redMockTransportService, blueMockTransportService)) {
+            mockTransportService.addDelegate(masterTransportService, new MockTransportService.DelegateTransport(mockTransportService.original()) {
+                @Override
+                protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request,
+                                           TransportRequestOptions options) throws IOException {
+                    logger.info("--> sending request {} on {}", action, connection.getNode());
+                    if ((primaryRelocation && finalized.get()) == false) {
+                        assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME);
+                    }
+                    super.sendRequest(connection, requestId, action, request, options);
+                }
+            });
+        }
+
+        if (primaryRelocation) {
+            logger.info("--> starting primary relocation recovery from blue to red");
+            client().admin().indices().prepareUpdateSettings(indexName).setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "red")
+            ).get();
+
+            ensureGreen(); // also waits for relocation / recovery to complete
+            // if a primary relocation fails after the source shard has been marked as relocated, both source and target are failed. If the
+            // source shard is moved back to started because the target fails first, it's possible that there is a cluster state where the
+            // shard is marked as started again (and ensureGreen returns), but while applying the cluster state the primary is failed and
+            // will be reallocated. The cluster will thus become green, then red, then green again. Triggering a refresh here before
+            // searching helps, as in contrast to search actions, refresh waits for the closed shard to be reallocated.
+            client().admin().indices().prepareRefresh(indexName).get();
+        } else {
+            logger.info("--> starting replica recovery from blue to red");
+            client().admin().indices().prepareUpdateSettings(indexName).setSettings(
+                Settings.builder()
+                    .put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "color", "red,blue")
+                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            ).get();
+
+            ensureGreen();
+        }
+
+        for (int i = 0; i < 10; i++) {
+            assertHitCount(client().prepareSearch(indexName).get(), numDocs);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
new file mode 100644
index 00000000000..bbe4d8ed12e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.recovery;
+
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class PeerRecoveryTargetServiceTests extends IndexShardTestCase {
+
+    public void testGetStartingSeqNo() throws Exception {
+        IndexShard replica = newShard(false);
+        RecoveryTarget recoveryTarget = new RecoveryTarget(replica, null, null, null);
+        try {
+            recoveryEmptyReplica(replica);
+            int docs = randomIntBetween(1, 10);
+            final String index = replica.shardId().getIndexName();
+            long seqNo = 0;
+            for (int i = 0; i < docs; i++) {
+                Engine.Index indexOp = replica.prepareIndexOnReplica(
+                    SourceToParse.source(SourceToParse.Origin.REPLICA, index, "type", "doc_" + i, new BytesArray("{}"), XContentType.JSON),
+                    seqNo++, 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
+                replica.index(indexOp);
+                if (rarely()) {
+                    // insert a gap
+                    seqNo++;
+                }
+            }
+
+            final long maxSeqNo = replica.seqNoStats().getMaxSeqNo();
+            final long localCheckpoint = replica.getLocalCheckpoint();
+
+            assertThat(PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget), equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO));
+
+            replica.updateGlobalCheckpointOnReplica(maxSeqNo - 1);
+            replica.getTranslog().sync();
+
+            // commit is enough, global checkpoint is below max *committed* which is NO_OPS_PERFORMED
+            assertThat(PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget), equalTo(0L));
+
+            replica.flush(new FlushRequest());
+
+            // commit is still not good enough, global checkpoint is below max
+            assertThat(PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget), equalTo(SequenceNumbersService.UNASSIGNED_SEQ_NO));
+
+            replica.updateGlobalCheckpointOnReplica(maxSeqNo);
+            replica.getTranslog().sync();
+            // commit is enough, global checkpoint is below max
+            assertThat(PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget), equalTo(localCheckpoint + 1));
+        } finally {
+            closeShards(replica);
+            recoveryTarget.decRef();
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index f109bc4e378..3858b6647f9 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -20,6 +20,7 @@ package org.elasticsearch.indices.recovery;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.CorruptIndexException;
@@ -27,6 +28,7 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -35,12 +37,26 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.SegmentsStats;
+import org.elasticsearch.index.mapper.Mapping;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SeqNoFieldMapper;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardRelocatedException;
 import org.elasticsearch.index.shard.IndexShardState;
@@ -57,6 +73,7 @@ import org.elasticsearch.test.IndexSettingsModule;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -65,7 +82,9 @@ import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -79,13 +98,17 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            null, randomBoolean(), randomLong());
+        final StartRecoveryRequest request = new StartRecoveryRequest(
+            shardId,
+            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+            null,
+            randomBoolean(),
+            randomNonNegativeLong(),
+            randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
         Store store = newStore(createTempDir());
         RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {},
-            recoverySettings.getChunkSize().bytesAsInt(), logger);
+            recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY);
         Directory dir = store.directory();
         RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
         int numDocs = randomIntBetween(10, 100);
@@ -127,19 +150,91 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         IOUtils.close(reader, store, targetStore);
     }
 
+    public void testSendSnapshotSendsOps() throws IOException {
+        final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
+        final int fileChunkSizeInBytes = recoverySettings.getChunkSize().bytesAsInt();
+        final long startingSeqNo = randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomIntBetween(0, 16);
+        final StartRecoveryRequest request = new StartRecoveryRequest(
+            shardId,
+            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+            null,
+            randomBoolean(),
+            randomNonNegativeLong(),
+            randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
+        final IndexShard shard = mock(IndexShard.class);
+        when(shard.state()).thenReturn(IndexShardState.STARTED);
+        final RecoveryTargetHandler recoveryTarget = mock(RecoveryTargetHandler.class);
+        final RecoverySourceHandler handler =
+            new RecoverySourceHandler(shard, recoveryTarget, request, () -> 0L, e -> () -> {}, fileChunkSizeInBytes, Settings.EMPTY);
+        final List<Translog.Operation> operations = new ArrayList<>();
+        final int initialNumberOfDocs = randomIntBetween(16, 64);
+        for (int i = 0; i < initialNumberOfDocs; i++) {
+            final Engine.Index index = getIndex(Integer.toString(i));
+            operations.add(new Translog.Index(index, new Engine.IndexResult(1, SequenceNumbersService.UNASSIGNED_SEQ_NO, true)));
+        }
+        final int numberOfDocsWithValidSequenceNumbers = randomIntBetween(16, 64);
+        for (int i = initialNumberOfDocs; i < initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers; i++) {
+            final Engine.Index index = getIndex(Integer.toString(i));
+            operations.add(new Translog.Index(index, new Engine.IndexResult(1, i - initialNumberOfDocs, true)));
+        }
+        operations.add(null);
+        int totalOperations = handler.sendSnapshot(startingSeqNo, new Translog.Snapshot() {
+            private int counter = 0;
+
+            @Override
+            public int totalOperations() {
+                return operations.size() - 1;
+            }
+
+            @Override
+            public Translog.Operation next() throws IOException {
+                return operations.get(counter++);
+            }
+        });
+        if (startingSeqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO) {
+            assertThat(totalOperations, equalTo(initialNumberOfDocs + numberOfDocsWithValidSequenceNumbers));
+        } else {
+            assertThat(totalOperations, equalTo(Math.toIntExact(numberOfDocsWithValidSequenceNumbers - startingSeqNo)));
+        }
+    }
+
+    private Engine.Index getIndex(final String id) {
+        final String type = "test";
+        final ParseContext.Document document = new ParseContext.Document();
+        document.add(new TextField("test", "test", Field.Store.YES));
+        final Field uidField = new Field("_uid", Uid.createUid(type, id), UidFieldMapper.Defaults.FIELD_TYPE);
+        final Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
+        final SeqNoFieldMapper.SequenceID seqID = SeqNoFieldMapper.SequenceID.emptySeqID();
+        document.add(uidField);
+        document.add(versionField);
+        document.add(seqID.seqNo);
+        document.add(seqID.seqNoDocValue);
+        document.add(seqID.primaryTerm);
+        final BytesReference source = new BytesArray(new byte[] { 1 });
+        final ParsedDocument doc =
+            new ParsedDocument(versionField, seqID, id, type, null, Arrays.asList(document), source, XContentType.JSON, null);
+        return new Engine.Index(new Term("_uid", doc.uid()), doc);
+    }
+
     public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        final StartRecoveryRequest request =
+            new StartRecoveryRequest(
+                shardId,
                 new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            null, randomBoolean(), randomLong());
+                null,
+                randomBoolean(),
+                randomNonNegativeLong(),
+                randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
         AtomicBoolean failedEngine = new AtomicBoolean(false);
         RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {},
-            recoverySettings.getChunkSize().bytesAsInt(), logger) {
+            recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {
             @Override
             protected void failEngine(IOException cause) {
                 assertFalse(failedEngine.get());
@@ -195,15 +290,20 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
+        final StartRecoveryRequest request =
+            new StartRecoveryRequest(
+                shardId,
                 new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
                 new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            null, randomBoolean(), randomLong());
+                null,
+                randomBoolean(),
+                randomNonNegativeLong(),
+                randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
         AtomicBoolean failedEngine = new AtomicBoolean(false);
         RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {},
-            recoverySettings.getChunkSize().bytesAsInt(), logger) {
+            recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {
             @Override
             protected void failEngine(IOException cause) {
                 assertFalse(failedEngine.get());
@@ -254,18 +354,39 @@ public class RecoverySourceHandlerTests extends ESTestCase {
 
     public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Completed() throws IOException {
         final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            null, false, randomLong());
-        IndexShard shard = mock(IndexShard.class);
-        Translog.View translogView = mock(Translog.View.class);
+        final boolean attemptSequenceNumberBasedRecovery = randomBoolean();
+        final boolean isTranslogReadyForSequenceNumberBasedRecovery = attemptSequenceNumberBasedRecovery && randomBoolean();
+        final StartRecoveryRequest request =
+            new StartRecoveryRequest(
+                shardId,
+                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+                null,
+                false,
+                randomNonNegativeLong(),
+                attemptSequenceNumberBasedRecovery ? randomNonNegativeLong() : SequenceNumbersService.UNASSIGNED_SEQ_NO);
+        final IndexShard shard = mock(IndexShard.class);
+        when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
+        when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class));
+        final Translog.View translogView = mock(Translog.View.class);
         when(shard.acquireTranslogView()).thenReturn(translogView);
         when(shard.state()).thenReturn(IndexShardState.RELOCATED);
-        AtomicBoolean phase1Called = new AtomicBoolean();
-        AtomicBoolean phase2Called = new AtomicBoolean();
-        RecoverySourceHandler handler = new RecoverySourceHandler(shard, null, request, () -> 0L, e -> () -> {},
-            recoverySettings.getChunkSize().bytesAsInt(), logger) {
+        final AtomicBoolean phase1Called = new AtomicBoolean();
+        final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean();
+        final AtomicBoolean phase2Called = new AtomicBoolean();
+        final RecoverySourceHandler handler = new RecoverySourceHandler(
+            shard,
+            mock(RecoveryTargetHandler.class),
+            request,
+            () -> 0L,
+            e -> () -> {},
+            recoverySettings.getChunkSize().bytesAsInt(),
+            Settings.EMPTY) {
+
+            @Override
+            boolean isTranslogReadyForSequenceNumberBasedRecovery(final Translog.View translogView) {
+                return isTranslogReadyForSequenceNumberBasedRecovery;
+            }
 
             @Override
             public void phase1(final IndexCommit snapshot, final Translog.View translogView) {
@@ -273,29 +394,47 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             }
 
             @Override
-            public void phase2(Translog.Snapshot snapshot) {
+            void prepareTargetForTranslog(final int totalTranslogOps, final long maxUnsafeAutoIdTimestamp) throws IOException {
+                prepareTargetForTranslogCalled.set(true);
+            }
+
+            @Override
+            void phase2(long startingSeqNo, Translog.Snapshot snapshot) throws IOException {
                 phase2Called.set(true);
             }
+
         };
-        expectThrows(IndexShardRelocatedException.class, () -> handler.recoverToTarget());
-        assertTrue(phase1Called.get());
+        expectThrows(IndexShardRelocatedException.class, handler::recoverToTarget);
+        // phase1 should only be attempted if we are not doing a sequence-number-based recovery
+        assertThat(phase1Called.get(), equalTo(!isTranslogReadyForSequenceNumberBasedRecovery));
+        assertTrue(prepareTargetForTranslogCalled.get());
         assertFalse(phase2Called.get());
     }
 
     public void testWaitForClusterStateOnPrimaryRelocation() throws IOException, InterruptedException {
         final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
-        StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-            null, true, randomLong());
-        AtomicBoolean phase1Called = new AtomicBoolean();
-        AtomicBoolean phase2Called = new AtomicBoolean();
-        AtomicBoolean ensureClusterStateVersionCalled = new AtomicBoolean();
-        AtomicBoolean recoveriesDelayed = new AtomicBoolean();
-        AtomicBoolean relocated = new AtomicBoolean();
+        final boolean attemptSequenceNumberBasedRecovery = randomBoolean();
+        final boolean isTranslogReadyForSequenceNumberBasedRecovery = attemptSequenceNumberBasedRecovery && randomBoolean();
+        final StartRecoveryRequest request =
+            new StartRecoveryRequest(
+                shardId,
+                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
+                null,
+                true,
+                randomNonNegativeLong(),
+                attemptSequenceNumberBasedRecovery ? randomNonNegativeLong(): SequenceNumbersService.UNASSIGNED_SEQ_NO);
+        final AtomicBoolean phase1Called = new AtomicBoolean();
+        final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean();
+        final AtomicBoolean phase2Called = new AtomicBoolean();
+        final AtomicBoolean ensureClusterStateVersionCalled = new AtomicBoolean();
+        final AtomicBoolean recoveriesDelayed = new AtomicBoolean();
+        final AtomicBoolean relocated = new AtomicBoolean();
 
-        IndexShard shard = mock(IndexShard.class);
-        Translog.View translogView = mock(Translog.View.class);
+        final IndexShard shard = mock(IndexShard.class);
+        when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
+        when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class));
+        final Translog.View translogView = mock(Translog.View.class);
         when(shard.acquireTranslogView()).thenReturn(translogView);
         when(shard.state()).then(i -> relocated.get() ? IndexShardState.RELOCATED : IndexShardState.STARTED);
         doAnswer(i -> {
@@ -304,16 +443,17 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             return null;
         }).when(shard).relocated(any(String.class));
 
-        RecoveryTargetHandler targetHandler = mock(RecoveryTargetHandler.class);
-
         final Supplier<Long> currentClusterStateVersionSupplier = () -> {
             assertFalse(ensureClusterStateVersionCalled.get());
             assertTrue(recoveriesDelayed.get());
             ensureClusterStateVersionCalled.set(true);
             return 0L;
         };
+
         final Function<String, Releasable> delayNewRecoveries = s -> {
-            assertTrue(phase1Called.get());
+            // phase1 should only be attempted if we are not doing a sequence-number-based recovery
+            assertThat(phase1Called.get(), equalTo(!isTranslogReadyForSequenceNumberBasedRecovery));
+            assertTrue(prepareTargetForTranslogCalled.get());
             assertTrue(phase2Called.get());
 
             assertFalse(recoveriesDelayed.get());
@@ -324,8 +464,19 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             };
         };
 
-        RecoverySourceHandler handler = new RecoverySourceHandler(shard, targetHandler, request, currentClusterStateVersionSupplier,
-            delayNewRecoveries, recoverySettings.getChunkSize().bytesAsInt(), logger) {
+        final RecoverySourceHandler handler = new RecoverySourceHandler(
+            shard,
+            mock(RecoveryTargetHandler.class),
+            request,
+            currentClusterStateVersionSupplier,
+            delayNewRecoveries,
+            recoverySettings.getChunkSize().bytesAsInt(),
+            Settings.EMPTY) {
+
+            @Override
+            boolean isTranslogReadyForSequenceNumberBasedRecovery(final Translog.View translogView) {
+                return isTranslogReadyForSequenceNumberBasedRecovery;
+            }
 
             @Override
             public void phase1(final IndexCommit snapshot, final Translog.View translogView) {
@@ -333,13 +484,22 @@ public class RecoverySourceHandlerTests extends ESTestCase {
             }
 
             @Override
-            public void phase2(Translog.Snapshot snapshot) {
+            void prepareTargetForTranslog(final int totalTranslogOps, final long maxUnsafeAutoIdTimestamp) throws IOException {
+                prepareTargetForTranslogCalled.set(true);
+            }
+
+            @Override
+            void phase2(long startingSeqNo, Translog.Snapshot snapshot) throws IOException {
                 phase2Called.set(true);
             }
+
         };
+
         handler.recoverToTarget();
         assertTrue(ensureClusterStateVersionCalled.get());
-        assertTrue(phase1Called.get());
+        // phase1 should only be attempted if we are not doing a sequence-number-based recovery
+        assertThat(phase1Called.get(), equalTo(!isTranslogReadyForSequenceNumberBasedRecovery));
+        assertTrue(prepareTargetForTranslogCalled.get());
         assertTrue(phase2Called.get());
         assertTrue(relocated.get());
         assertFalse(recoveriesDelayed.get());
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
index 691b043e86a..c176b7c3d29 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/StartRecoveryRequestTests.java
@@ -23,6 +23,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.index.seqno.SequenceNumbers;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.test.ESTestCase;
@@ -33,36 +35,44 @@ import java.io.ByteArrayOutputStream;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
 import static org.hamcrest.Matchers.equalTo;
 
 public class StartRecoveryRequestTests extends ESTestCase {
+
     public void testSerialization() throws Exception {
-        Version targetNodeVersion = randomVersion(random());
-        StartRecoveryRequest outRequest = new StartRecoveryRequest(
+        final Version targetNodeVersion = randomVersion(random());
+        final StartRecoveryRequest outRequest = new StartRecoveryRequest(
                 new ShardId("test", "_na_", 0),
                 new DiscoveryNode("a", buildNewFakeTransportAddress(), emptyMap(), emptySet(), targetNodeVersion),
                 new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), targetNodeVersion),
                 Store.MetadataSnapshot.EMPTY,
                 randomBoolean(),
-                1L
-        );
-        ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
-        OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
+                randomNonNegativeLong(),
+                randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
+
+        final ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
+        final OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
         out.setVersion(targetNodeVersion);
         outRequest.writeTo(out);
 
-        ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
+        final ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
         InputStreamStreamInput in = new InputStreamStreamInput(inBuffer);
         in.setVersion(targetNodeVersion);
-        StartRecoveryRequest inRequest = new StartRecoveryRequest();
+        final StartRecoveryRequest inRequest = new StartRecoveryRequest();
         inRequest.readFrom(in);
 
         assertThat(outRequest.shardId(), equalTo(inRequest.shardId()));
         assertThat(outRequest.sourceNode(), equalTo(inRequest.sourceNode()));
         assertThat(outRequest.targetNode(), equalTo(inRequest.targetNode()));
         assertThat(outRequest.metadataSnapshot().asMap(), equalTo(inRequest.metadataSnapshot().asMap()));
-        assertThat(outRequest.recoveryId(), equalTo(inRequest.recoveryId()));
         assertThat(outRequest.isPrimaryRelocation(), equalTo(inRequest.isPrimaryRelocation()));
+        assertThat(outRequest.recoveryId(), equalTo(inRequest.recoveryId()));
+        if (targetNodeVersion.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+            assertThat(outRequest.startingSeqNo(), equalTo(inRequest.startingSeqNo()));
+        } else {
+            assertThat(SequenceNumbersService.UNASSIGNED_SEQ_NO, equalTo(inRequest.startingSeqNo()));
+        }
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index 3cdea34a69b..f7b9cb306f3 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -244,7 +244,7 @@ public class UpdateSettingsIT extends ESIntegTestCase {
         public boolean sawUpdateMaxThreadCount;
         public boolean sawUpdateAutoThrottle;
 
-        public MockAppender(final String name) throws IllegalAccessException {
+        MockAppender(final String name) throws IllegalAccessException {
             super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 3d9e66755eb..91673630c3d 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -21,21 +21,26 @@ package org.elasticsearch.indices.stats;
 
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MergePolicyConfig;
@@ -54,14 +59,27 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.Random;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -722,8 +740,8 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         ensureGreen();
 
-        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
-        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
+        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
+        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
         refresh();
 
         client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet();
@@ -769,8 +787,8 @@ public class IndexStatsIT extends ESIntegTestCase {
                         "{ \"properties\": { \"bar\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}"));
         ensureGreen();
 
-        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
-        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
+        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
+        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get();
         refresh();
 
         IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
@@ -1068,4 +1086,103 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), equalTo(0L));
     }
 
+    /**
+     * Test that we can safely concurrently index and get stats. This test was inspired by a serialization issue that arose due to a race
+     * getting doc stats during heavy indexing. The race could lead to deleted docs being negative which would then be serialized as a
+     * variable-length long. Since serialization of negative longs using a variable-length format was unsupported
+     * ({@link org.elasticsearch.common.io.stream.StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test
+     * that we can continue to get stats while indexing.
+     */
+    public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierException, InterruptedException, ExecutionException {
+        final AtomicInteger idGenerator = new AtomicInteger();
+        final int numberOfIndexingThreads = Runtime.getRuntime().availableProcessors();
+        final int numberOfStatsThreads = 4 * numberOfIndexingThreads;
+        final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfIndexingThreads + numberOfStatsThreads);
+        final AtomicBoolean stop = new AtomicBoolean();
+        final List<Thread> threads = new ArrayList<>(numberOfIndexingThreads + numberOfIndexingThreads);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicBoolean failed = new AtomicBoolean();
+        final AtomicReference<List<ShardOperationFailedException>> shardFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
+        final AtomicReference<List<Exception>> executionFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
+
+        // increasing the number of shards increases the number of chances any one stats request will hit a race
+        final CreateIndexRequest createIndexRequest =
+            new CreateIndexRequest("test", Settings.builder().put("index.number_of_shards", 10).build());
+        client().admin().indices().create(createIndexRequest).get();
+
+        // start threads that will index concurrently with stats requests
+        for (int i = 0; i < numberOfIndexingThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    failed.set(true);
+                    executionFailures.get().add(e);
+                    latch.countDown();
+                }
+                while (!stop.get()) {
+                    final String id = Integer.toString(idGenerator.incrementAndGet());
+                    final IndexResponse response =
+                        client()
+                            .prepareIndex("test", "type", id)
+                            .setSource("{}", XContentType.JSON)
+                            .get();
+                    assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED));
+                }
+            });
+            thread.setName("indexing-" + i);
+            threads.add(thread);
+            thread.start();
+        }
+
+        // start threads that will get stats concurrently with indexing
+        for (int i = 0; i < numberOfStatsThreads; i++) {
+            final Thread thread = new Thread(() -> {
+                try {
+                    barrier.await();
+                } catch (final BrokenBarrierException | InterruptedException e) {
+                    failed.set(true);
+                    executionFailures.get().add(e);
+                    latch.countDown();
+                }
+                final IndicesStatsRequest request = new IndicesStatsRequest();
+                request.all();
+                request.indices(new String[0]);
+                while (!stop.get()) {
+                    try {
+                        final IndicesStatsResponse response = client().admin().indices().stats(request).get();
+                        if (response.getFailedShards() > 0) {
+                            failed.set(true);
+                            shardFailures.get().addAll(Arrays.asList(response.getShardFailures()));
+                            latch.countDown();
+                        }
+                    } catch (final ExecutionException | InterruptedException e) {
+                        failed.set(true);
+                        executionFailures.get().add(e);
+                        latch.countDown();
+                    }
+                }
+            });
+            thread.setName("stats-" + i);
+            threads.add(thread);
+            thread.start();
+        }
+
+        // release the hounds
+        barrier.await();
+
+        // wait for a failure, or for fifteen seconds to elapse
+        latch.await(15, TimeUnit.SECONDS);
+
+        // stop all threads and wait for them to complete
+        stop.set(true);
+        for (final Thread thread : threads) {
+            thread.join();
+        }
+
+        assertThat(shardFailures.get(), emptyCollectionOf(ShardOperationFailedException.class));
+        assertThat(executionFailures.get(), emptyCollectionOf(Exception.class));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index 29845e5396d..59c29a5b6a0 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -23,10 +23,9 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -45,7 +44,7 @@ import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.PeerRecoverySourceService;
+import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -56,8 +55,6 @@ import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
 import org.elasticsearch.test.disruption.SingleNodeDisruption;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
@@ -398,9 +395,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         // disable relocations when we do this, to make sure the shards are not relocated from node2
         // due to rebalancing, and delete its content
         client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)).get();
-        internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        internalCluster().getInstance(ClusterService.class, nonMasterNode).submitStateUpdateTask("test", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
                 for (int i = 0; i < numShards; i++) {
                     indexRoutingTableBuilder.addIndexShard(
@@ -409,14 +406,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
                                     .build()
                     );
                 }
-                return ClusterState.builder(currentState)
+                return newState(ClusterState.builder(currentState)
                         .routingTable(RoutingTable.builder().add(indexRoutingTableBuilder).build())
-                        .build();
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                        .build());
             }
 
             @Override
@@ -480,18 +472,13 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
 
         @Override
         public void receivedRequest(long requestId, String action) {
-            if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) {
+            if (action.equals(PeerRecoveryTargetService.Actions.FILES_INFO)) {
+                logger.info("received: {}, relocation starts", action);
+                beginRelocationLatch.countDown();
+            } else if (action.equals(IndicesStore.ACTION_SHARD_EXISTS)) {
                 receivedShardExistsRequestLatch.countDown();
                 logger.info("received: {}, relocation done", action);
             }
         }
-
-        @Override
-        public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
-            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
-                logger.info("sent: {}, relocation starts", action);
-                beginRelocationLatch.countDown();
-            }
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 57777453976..cb6b8b752bc 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -30,14 +30,15 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.InvalidAliasNameException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
-
 import org.junit.After;
 
 import java.io.IOException;
@@ -155,18 +156,37 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         client().admin().indices().preparePutTemplate("template_1")
                 .setPatterns(Collections.singletonList("te*"))
                 .setOrder(0)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field1").field("type", "text").field("store", true).endObject()
-                        .startObject("field2").field("type", "text").field("store", true).endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .execute().actionGet();
 
         client().admin().indices().preparePutTemplate("template_2")
                 .setPatterns(Collections.singletonList("test*"))
                 .setOrder(1)
-                .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
-                        .startObject("field2").field("type", "text").field("store", "no").endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("field2")
+                                    .field("type", "text")
+                                    .field("store", false)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
                 .execute().actionGet();
 
         logger.info("--> explicitly delete template_1");
@@ -306,7 +326,6 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 "get template with " + Arrays.toString(names));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/8802")
     public void testBrokenMapping() throws Exception {
         // clean all templates setup by the framework.
         client().admin().indices().prepareDeleteTemplate("*").get();
@@ -318,7 +337,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         MapperParsingException e = expectThrows( MapperParsingException.class,
             () -> client().admin().indices().preparePutTemplate("template_1")
                 .setPatterns(Collections.singletonList("te*"))
-                .addMapping("type1", "abcde")
+                .addMapping("type1", "{\"foo\": \"abcde\"}", XContentType.JSON)
                 .get());
         assertThat(e.getMessage(), containsString("Failed to parse mapping "));
 
@@ -355,7 +374,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
 
         client().admin().indices().preparePutTemplate("template_with_aliases")
                 .setPatterns(Collections.singletonList("te*"))
-                .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}")
+                .addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON)
                 .addAlias(new Alias("simple_alias"))
                 .addAlias(new Alias("templated_alias-{index}"))
                 .addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}"))
@@ -406,7 +425,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
 
     public void testIndexTemplateWithAliasesInSource() {
         client().admin().indices().preparePutTemplate("template_1")
-                .setSource("{\n" +
+                .setSource(new BytesArray("{\n" +
                         "    \"template\" : \"*\",\n" +
                         "    \"aliases\" : {\n" +
                         "        \"my_alias\" : {\n" +
@@ -417,7 +436,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                         "            }\n" +
                         "        }\n" +
                         "    }\n" +
-                        "}").get();
+                        "}"), XContentType.JSON).get();
 
 
         assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2"));
@@ -618,8 +637,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setOrder(0)
                 .addAlias(new Alias("alias4").filter(termQuery("field", "value"))).get();
 
-        client().prepareIndex("a1", "test", "test").setSource("{}").get();
-        BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get();
+        client().prepareIndex("a1", "test", "test").setSource("{}", XContentType.JSON).get();
+        BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("a2"));
@@ -627,8 +646,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         assertThat(response.getItems()[0].getId(), equalTo("test"));
         assertThat(response.getItems()[0].getVersion(), equalTo(1L));
 
-        client().prepareIndex("b1", "test", "test").setSource("{}").get();
-        response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get();
+        client().prepareIndex("b1", "test", "test").setSource("{}", XContentType.JSON).get();
+        response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("b2"));
@@ -636,8 +655,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         assertThat(response.getItems()[0].getId(), equalTo("test"));
         assertThat(response.getItems()[0].getVersion(), equalTo(1L));
 
-        client().prepareIndex("c1", "test", "test").setSource("{}").get();
-        response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get();
+        client().prepareIndex("c1", "test", "test").setSource("{}", XContentType.JSON).get();
+        response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getIndex(), equalTo("c2"));
@@ -653,9 +672,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         // So the aliases defined in the index template for this index will not fail
         // even though the fields in the alias fields don't exist yet and indexing into
         // an index that doesn't exist yet will succeed
-        client().prepareIndex("d1", "test", "test").setSource("{}").get();
+        client().prepareIndex("d1", "test", "test").setSource("{}", XContentType.JSON).get();
 
-        response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get();
+        response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}", XContentType.JSON)).get();
         assertThat(response.hasFailures(), is(false));
         assertThat(response.getItems()[0].isFailed(), equalTo(false));
         assertThat(response.getItems()[0].getId(), equalTo("test"));
@@ -685,7 +704,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                     "                }\n" +
                     "            }\n" +
                     "         }\n" +
-                    "    }\n")
+                    "    }\n", XContentType.JSON)
             .get();
 
         // put template using custom_1 analyzer
@@ -764,4 +783,63 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         assertEquals("value1", searchResponse.getHits().getAt(0).field("field1").value().toString());
         assertNull(searchResponse.getHits().getAt(0).field("field2"));
     }
+
+    public void testPartitionedTemplate() throws Exception {
+        // clean all templates setup by the framework.
+        client().admin().indices().prepareDeleteTemplate("*").get();
+
+        // check get all templates on an empty index.
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
+        assertThat(response.getIndexTemplates(), empty());
+
+        // provide more partitions than shards
+        IllegalArgumentException eBadSettings = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutTemplate("template_1")
+                .setPatterns(Collections.singletonList("te*"))
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", "5")
+                    .put("index.routing_partition_size", "6"))
+                .get());
+        assertThat(eBadSettings.getMessage(), containsString("partition size [6] should be a positive number "
+                + "less than the number of shards [5]"));
+
+        // provide an invalid mapping for a partitioned index
+        IllegalArgumentException eBadMapping = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutTemplate("template_2")
+                .setPatterns(Collections.singletonList("te*"))
+                .addMapping("type", "{\"type\":{\"_routing\":{\"required\":false}}}")
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", "6")
+                    .put("index.routing_partition_size", "3"))
+                .get());
+        assertThat(eBadMapping.getMessage(), containsString("must have routing required for partitioned index"));
+
+        // no templates yet
+        response = client().admin().indices().prepareGetTemplates().get();
+        assertEquals(0, response.getIndexTemplates().size());
+
+        // a valid configuration that only provides the partition size
+        assertAcked(client().admin().indices().preparePutTemplate("just_partitions")
+            .setPatterns(Collections.singletonList("te*"))
+            .setSettings(Settings.builder()
+                .put("index.routing_partition_size", "6"))
+            .get());
+
+        // create an index with too few shards
+        IllegalArgumentException eBadIndex = expectThrows(IllegalArgumentException.class,
+                () -> prepareCreate("test_bad", Settings.builder()
+            .put("index.number_of_shards", 5))
+            .get());
+
+        assertThat(eBadIndex.getMessage(), containsString("partition size [6] should be a positive number "
+                + "less than the number of shards [5]"));
+
+        // finally, create a valid index
+        prepareCreate("test_good", Settings.builder()
+            .put("index.number_of_shards", 7))
+            .get();
+
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get();
+        assertEquals("6", getSettingsResponse.getIndexToSettings().get("test_good").getAsMap().get("index.routing_partition_size"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index f2d9aaa5170..eacc93a96fd 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -31,6 +31,8 @@ import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
 
@@ -105,12 +107,17 @@ public class ConfigurationUtilsTests extends ESTestCase {
         assertThat(result.get(0), sameInstance(processor));
         assertThat(result.get(1), sameInstance(processor));
 
-        config.add(Collections.singletonMap("unknown_processor", emptyConfig));
+        Map<String, Object> unknownTaggedConfig = new HashMap<>();
+        unknownTaggedConfig.put("tag", "my_unknown");
+        config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig));
         try {
             ConfigurationUtils.readProcessorConfigs(config, registry);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
+            assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown")));
+            assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor")));
+            assertThat(e.getHeader("property_name"), is(nullValue()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index 14335bde076..2b59f0d421c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -36,8 +36,10 @@ import org.elasticsearch.action.ingest.SimulateDocumentBaseResult;
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -81,7 +83,7 @@ public class IngestClientIT extends ESIntegTestCase {
             .endObject()
             .endArray()
             .endObject().bytes();
-        client().admin().cluster().preparePutPipeline("_id", pipelineSource)
+        client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON)
                 .get();
         GetPipelineResponse getResponse = client().admin().cluster().prepareGetPipeline("_id")
                 .get();
@@ -104,10 +106,10 @@ public class IngestClientIT extends ESIntegTestCase {
             .endObject().bytes();
         SimulatePipelineResponse response;
         if (randomBoolean()) {
-            response = client().admin().cluster().prepareSimulatePipeline(bytes)
+            response = client().admin().cluster().prepareSimulatePipeline(bytes, XContentType.JSON)
                 .setId("_id").get();
         } else {
-            SimulatePipelineRequest request = new SimulatePipelineRequest(bytes);
+            SimulatePipelineRequest request = new SimulatePipelineRequest(bytes, XContentType.JSON);
             request.setId("_id");
             response = client().admin().cluster().simulatePipeline(request).get();
         }
@@ -137,14 +139,14 @@ public class IngestClientIT extends ESIntegTestCase {
             .endObject()
             .endArray()
             .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         client().admin().cluster().putPipeline(putPipelineRequest).get();
 
         int numRequests = scaledRandomIntBetween(32, 128);
         BulkRequest bulkRequest = new BulkRequest();
         for (int i = 0; i < numRequests; i++) {
             IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id");
-            indexRequest.source("field", "value", "fail", i % 2 == 0);
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
             bulkRequest.add(indexRequest);
         }
 
@@ -177,7 +179,7 @@ public class IngestClientIT extends ESIntegTestCase {
             .endObject()
             .endArray()
             .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         client().admin().cluster().putPipeline(putPipelineRequest).get();
 
         GetPipelineRequest getPipelineRequest = new GetPipelineRequest("_id");
@@ -219,7 +221,7 @@ public class IngestClientIT extends ESIntegTestCase {
                 .endObject()
                 .endArray()
                 .endObject().bytes();
-        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source);
+        PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
         try {
             client().admin().cluster().putPipeline(putPipelineRequest).get();
         } catch (ExecutionException e) {
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
index 8d36598b4d9..518b775d7f8 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
@@ -42,10 +42,10 @@ public class IngestMetadataTests extends ESTestCase {
 
     public void testFromXContent() throws IOException {
         PipelineConfiguration pipeline = new PipelineConfiguration(
-            "1",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "1", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         PipelineConfiguration pipeline2 = new PipelineConfiguration(
-            "2",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}")
+            "2", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field1\", \"value\": \"_value1\"}}]}"), XContentType.JSON
         );
         Map<String, PipelineConfiguration> map = new HashMap<>();
         map.put(pipeline.getId(), pipeline);
@@ -58,7 +58,7 @@ public class IngestMetadataTests extends ESTestCase {
         builder.endObject();
         XContentBuilder shuffled = shuffleXContent(builder);
         final XContentParser parser = createParser(shuffled);
-        MetaData.Custom custom = ingestMetadata.fromXContent(parser);
+        MetaData.Custom custom = IngestMetadata.fromXContent(parser);
         assertTrue(custom instanceof IngestMetadata);
         IngestMetadata m = (IngestMetadata) custom;
         assertEquals(2, m.getPipelines().size());
@@ -72,14 +72,14 @@ public class IngestMetadataTests extends ESTestCase {
         BytesReference pipelineConfig = new BytesArray("{}");
 
         Map<String, PipelineConfiguration> pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig));
-        pipelines.put("2", new PipelineConfiguration("2", pipelineConfig));
+        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", pipelineConfig, XContentType.JSON));
         IngestMetadata ingestMetadata1 = new IngestMetadata(pipelines);
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig));
-        pipelines.put("3", new PipelineConfiguration("3", pipelineConfig));
-        pipelines.put("4", new PipelineConfiguration("4", pipelineConfig));
+        pipelines.put("1", new PipelineConfiguration("1", pipelineConfig, XContentType.JSON));
+        pipelines.put("3", new PipelineConfiguration("3", pipelineConfig, XContentType.JSON));
+        pipelines.put("4", new PipelineConfiguration("4", pipelineConfig, XContentType.JSON));
         IngestMetadata ingestMetadata2 = new IngestMetadata(pipelines);
 
         IngestMetadata.IngestMetadataDiff diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata2.diff(ingestMetadata1);
@@ -92,13 +92,13 @@ public class IngestMetadataTests extends ESTestCase {
         IngestMetadata endResult = (IngestMetadata) diff.apply(ingestMetadata2);
         assertThat(endResult, not(equalTo(ingestMetadata1)));
         assertThat(endResult.getPipelines().size(), equalTo(3));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig)));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("3"), equalTo(new PipelineConfiguration("3", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("4"), equalTo(new PipelineConfiguration("4", pipelineConfig, XContentType.JSON)));
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}")));
-        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}")));
+        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{}"), XContentType.JSON));
         IngestMetadata ingestMetadata3 = new IngestMetadata(pipelines);
 
         diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata3.diff(ingestMetadata1);
@@ -108,12 +108,12 @@ public class IngestMetadataTests extends ESTestCase {
         endResult = (IngestMetadata) diff.apply(ingestMetadata3);
         assertThat(endResult, equalTo(ingestMetadata1));
         assertThat(endResult.getPipelines().size(), equalTo(2));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig)));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", pipelineConfig, XContentType.JSON)));
 
         pipelines = new HashMap<>();
-        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}")));
-        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}")));
+        pipelines.put("1", new PipelineConfiguration("1", new BytesArray("{}"), XContentType.JSON));
+        pipelines.put("2", new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON));
         IngestMetadata ingestMetadata4 = new IngestMetadata(pipelines);
 
         diff = (IngestMetadata.IngestMetadataDiff) ingestMetadata4.diff(ingestMetadata1);
@@ -123,7 +123,8 @@ public class IngestMetadataTests extends ESTestCase {
         endResult = (IngestMetadata) diff.apply(ingestMetadata4);
         assertThat(endResult, not(equalTo(ingestMetadata1)));
         assertThat(endResult.getPipelines().size(), equalTo(2));
-        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig)));
-        assertThat(endResult.getPipelines().get("2"), equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"))));
+        assertThat(endResult.getPipelines().get("1"), equalTo(new PipelineConfiguration("1", pipelineConfig, XContentType.JSON)));
+        assertThat(endResult.getPipelines().get("2"),
+            equalTo(new PipelineConfiguration("2", new BytesArray("{\"key\" : \"value\"}"), XContentType.JSON)));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index 02fe0d03c77..1a8e6e873e0 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -22,7 +22,8 @@ package org.elasticsearch.ingest;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -69,7 +70,7 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
         ensureStableCluster(2, node2);
 
         try {
-            client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+            client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
             fail("exception expected");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Processor type [test] is not installed on node"));
@@ -82,7 +83,7 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
         internalCluster().startNode();
 
         try {
-            client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+            client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("No processor type exists with name [test]"));
@@ -95,7 +96,7 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
         installPlugin = true;
         String node1 = internalCluster().startNode();
 
-        WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource).get();
+        WritePipelineResponse response = client().admin().cluster().preparePutPipeline("_id", pipelineSource, XContentType.JSON).get();
         assertThat(response.isAcknowledged(), is(true));
         Pipeline pipeline = internalCluster().getInstance(NodeService.class, node1).getIngestService().getPipelineStore().get("_id");
         assertThat(pipeline, notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
new file mode 100644
index 00000000000..f41d01b32c8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ContextParser;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public class PipelineConfigurationTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        PipelineConfiguration configuration = new PipelineConfiguration("1",
+            new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON);
+        assertEquals(XContentType.JSON, configuration.getXContentType());
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        configuration.writeTo(out);
+        StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes);
+        PipelineConfiguration serialized = PipelineConfiguration.readFrom(in);
+        assertEquals(XContentType.JSON, serialized.getXContentType());
+        assertEquals("{}", serialized.getConfig().utf8ToString());
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("ATECe30AAAA=");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PipelineConfiguration configuration = PipelineConfiguration.readFrom(in);
+            assertEquals(XContentType.JSON, configuration.getXContentType());
+            assertEquals("{}", configuration.getConfig().utf8ToString());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                configuration.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
+    public void testParser() throws IOException {
+        ContextParser<Void, PipelineConfiguration> parser = PipelineConfiguration.getParser();
+        XContentType xContentType = randomFrom(XContentType.values());
+        final BytesReference bytes;
+        try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
+            new PipelineConfiguration("1", new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
+                .toXContent(builder, ToXContent.EMPTY_PARAMS);
+            bytes = builder.bytes();
+        }
+
+        XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes);
+        PipelineConfiguration parsed = parser.parse(xContentParser, null);
+        assertEquals(xContentType, parsed.getXContentType());
+        assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType()));
+        assertEquals("1", parsed.getId());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index c00e9254ab2..44c8e78bef7 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -25,8 +25,10 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.CustomTypeSafeMatcher;
@@ -273,7 +275,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
                 }
             } else {
                 IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
-                indexRequest.source("field1", "value1");
+                indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
                 request = indexRequest;
                 numIndexRequests++;
             }
@@ -303,7 +305,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         int numRequest = scaledRandomIntBetween(8, 64);
         for (int i = 0; i < numRequest; i++) {
             IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline(pipelineId);
-            indexRequest.source("field1", "value1");
+            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
             bulkRequest.add(indexRequest);
         }
 
@@ -331,8 +333,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, null, new CompoundProcessor(mock(Processor.class))));
 
         Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
-        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
-        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
 
         @SuppressWarnings("unchecked")
@@ -361,14 +363,14 @@ public class PipelineExecutionServiceTests extends ESTestCase {
     // issue: https://github.com/elastic/elasticsearch/issues/18126
     public void testUpdatingStatsWhenRemovingPipelineWorks() throws Exception {
         Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
-        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
-        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"), XContentType.JSON));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
         assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id1"));
         assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id2"));
 
         configurationMap = new HashMap<>();
-        configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}")));
+        configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}"), XContentType.JSON));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
         assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id1")));
         assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id2")));
@@ -382,7 +384,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         private final IngestDocument ingestDocument;
 
-        public IngestDocumentMatcher(String index, String type, String id, Map<String, Object> source) {
+        IngestDocumentMatcher(String index, String type, String id, Map<String, Object> source) {
             this.ingestDocument = new IngestDocument(index, type, id, null, null, source);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 4e90f5346d2..e91bb7e2e6e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -102,7 +103,7 @@ public class PipelineStoreTests extends ESTestCase {
         assertThat(store.pipelines.size(), is(0));
 
         PipelineConfiguration pipeline = new PipelineConfiguration(
-            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", pipeline));
         clusterState = ClusterState.builder(clusterState)
@@ -123,7 +124,7 @@ public class PipelineStoreTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
 
         // add a new pipeline:
-        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"));
+        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -134,7 +135,8 @@ public class PipelineStoreTests extends ESTestCase {
         assertThat(pipeline.getProcessors().size(), equalTo(0));
 
         // overwrite existing pipeline:
-        putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"));
+        putRequest =
+            new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"), XContentType.JSON);
         previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -151,7 +153,8 @@ public class PipelineStoreTests extends ESTestCase {
         assertThat(pipeline, nullValue());
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
 
-        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"));
+        PutPipelineRequest putRequest =
+            new PutPipelineRequest(id, new BytesArray("{\"description\": \"empty processors\"}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         try {
@@ -166,7 +169,7 @@ public class PipelineStoreTests extends ESTestCase {
 
     public void testDelete() {
         PipelineConfiguration config = new PipelineConfiguration(
-            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}")
+            "_id",new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON
         );
         IngestMetadata ingestMetadata = new IngestMetadata(Collections.singletonMap("_id", config));
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
@@ -192,13 +195,84 @@ public class PipelineStoreTests extends ESTestCase {
         }
     }
 
+    public void testDeleteUsingWildcard() {
+        HashMap<String, PipelineConfiguration> pipelines = new HashMap<>();
+        BytesArray definition = new BytesArray(
+            "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"
+        );
+        pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON));
+        pipelines.put("p2", new PipelineConfiguration("p2", definition, XContentType.JSON));
+        pipelines.put("q1", new PipelineConfiguration("q1", definition, XContentType.JSON));
+        IngestMetadata ingestMetadata = new IngestMetadata(pipelines);
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousClusterState = clusterState;
+        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder()
+            .putCustom(IngestMetadata.TYPE, ingestMetadata)).build();
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), notNullValue());
+        assertThat(store.get("p2"), notNullValue());
+        assertThat(store.get("q1"), notNullValue());
+
+        // Delete pipeline matching wildcard
+        DeletePipelineRequest deleteRequest = new DeletePipelineRequest("p*");
+        previousClusterState = clusterState;
+        clusterState = store.innerDelete(deleteRequest, clusterState);
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), nullValue());
+        assertThat(store.get("p2"), nullValue());
+        assertThat(store.get("q1"), notNullValue());
+
+        // Exception if we used name which does not exist
+        try {
+            store.innerDelete(new DeletePipelineRequest("unknown"), clusterState);
+            fail("exception expected");
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("pipeline [unknown] is missing"));
+        }
+
+        // match all wildcard works on last remaining pipeline
+        DeletePipelineRequest matchAllDeleteRequest = new DeletePipelineRequest("*");
+        previousClusterState = clusterState;
+        clusterState = store.innerDelete(matchAllDeleteRequest, clusterState);
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), nullValue());
+        assertThat(store.get("p2"), nullValue());
+        assertThat(store.get("q1"), nullValue());
+
+        // match all wildcard does not throw exception if none match
+        store.innerDelete(matchAllDeleteRequest, clusterState);
+    }
+
+    public void testDeleteWithExistingUnmatchedPipelines() {
+        HashMap<String, PipelineConfiguration> pipelines = new HashMap<>();
+        BytesArray definition = new BytesArray(
+            "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"
+        );
+        pipelines.put("p1", new PipelineConfiguration("p1", definition, XContentType.JSON));
+        IngestMetadata ingestMetadata = new IngestMetadata(pipelines);
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousClusterState = clusterState;
+        clusterState = ClusterState.builder(clusterState).metaData(MetaData.builder()
+            .putCustom(IngestMetadata.TYPE, ingestMetadata)).build();
+        store.innerUpdatePipelines(previousClusterState, clusterState);
+        assertThat(store.get("p1"), notNullValue());
+
+        DeletePipelineRequest deleteRequest = new DeletePipelineRequest("z*");
+        try {
+            store.innerDelete(deleteRequest, clusterState);
+            fail("exception expected");
+        } catch (ResourceNotFoundException e) {
+            assertThat(e.getMessage(), equalTo("pipeline [z*] is missing"));
+        }
+    }
+
     public void testGetPipelines() {
         Map<String, PipelineConfiguration> configs = new HashMap<>();
         configs.put("_id1", new PipelineConfiguration(
-            "_id1", new BytesArray("{\"processors\": []}")
+            "_id1", new BytesArray("{\"processors\": []}"), XContentType.JSON
         ));
         configs.put("_id2", new PipelineConfiguration(
-            "_id2", new BytesArray("{\"processors\": []}")
+            "_id2", new BytesArray("{\"processors\": []}"), XContentType.JSON
         ));
 
         assertThat(store.innerGetPipelines(null, "_id1").isEmpty(), is(true));
@@ -240,7 +314,7 @@ public class PipelineStoreTests extends ESTestCase {
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
 
         PutPipelineRequest putRequest = new PutPipelineRequest(id,
-                new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
+                new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
         ClusterState previousClusterState = clusterState;
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(previousClusterState, clusterState);
@@ -261,7 +335,8 @@ public class PipelineStoreTests extends ESTestCase {
 
     public void testValidate() throws Exception {
         PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
-                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"));
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"),
+            XContentType.JSON);
 
         DiscoveryNode node1 = new DiscoveryNode("_node_id1", buildNewFakeTransportAddress(),
                 emptyMap(), emptySet(), Version.CURRENT);
@@ -284,7 +359,7 @@ public class PipelineStoreTests extends ESTestCase {
 
     public void testValidateNoIngestInfo() throws Exception {
         PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
-                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
         try {
             store.validatePipeline(Collections.emptyMap(), putRequest);
             fail("exception expected");
diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
index 5c499eae4d8..3f0eac7762f 100644
--- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
+++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -143,7 +144,7 @@ public class SimpleMgetIT extends ESIntegTestCase {
                 .field("excluded", "should not be seen")
                 .endObject().bytes();
         for (int i = 0; i < 100; i++) {
-            client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef).get();
+            client().prepareIndex("test", "type", Integer.toString(i)).setSource(sourceBytesRef, XContentType.JSON).get();
         }
 
         MultiGetRequestBuilder request = client().prepareMultiGet();
diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
index 14f7151e40d..ab31a0b229b 100644
--- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
@@ -45,7 +45,7 @@ public class FsProbeTests extends ESTestCase {
         try (NodeEnvironment env = newNodeEnvironment()) {
             FsProbe probe = new FsProbe(Settings.EMPTY, env);
 
-            FsInfo stats = probe.stats(null);
+            FsInfo stats = probe.stats(null, null);
             assertNotNull(stats);
             assertThat(stats.getTimestamp(), greaterThan(0L));
 
diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
index 9bd4ab3b90f..5b110c928c3 100644
--- a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
@@ -41,7 +41,7 @@ public class OsProbeTests extends ESTestCase {
 
     public void testOsInfo() {
         int allocatedProcessors = randomIntBetween(1, Runtime.getRuntime().availableProcessors());
-        long refreshInterval = randomBoolean() ? -1 : randomPositiveLong();
+        long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
         OsInfo info = probe.osInfo(refreshInterval, allocatedProcessors);
         assertNotNull(info);
         assertEquals(refreshInterval, info.getRefreshInterval());
@@ -195,7 +195,7 @@ public class OsProbeTests extends ESTestCase {
             }
 
             @Override
-            protected boolean areCgroupStatsAvailable() {
+            boolean areCgroupStatsAvailable() {
                 return areCgroupStatsAvailable;
             }
 
diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
index 8334f71e86a..9f8b60a55a9 100644
--- a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java
@@ -38,11 +38,11 @@ public class OsStatsTests extends ESTestCase {
         OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
         OsStats.Cgroup cgroup = new OsStats.Cgroup(
             randomAsciiOfLength(8),
-            randomPositiveLong(),
+            randomNonNegativeLong(),
             randomAsciiOfLength(8),
-            randomPositiveLong(),
-            randomPositiveLong(),
-            new OsStats.Cgroup.CpuStat(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()));
+            randomNonNegativeLong(),
+            randomNonNegativeLong(),
+            new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
         OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup);
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
diff --git a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
index f8fc6a675cf..4a44c518051 100644
--- a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java
@@ -36,7 +36,7 @@ public class ProcessProbeTests extends ESTestCase {
     private final ProcessProbe probe = ProcessProbe.getInstance();
 
     public void testProcessInfo() {
-        long refreshInterval = randomPositiveLong();
+        long refreshInterval = randomNonNegativeLong();
         ProcessInfo info = probe.processInfo(refreshInterval);
         assertNotNull(info);
         assertEquals(refreshInterval, info.getRefreshInterval());
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
index 2dc95f8e9f1..21c1811616d 100644
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
@@ -21,34 +21,46 @@ package org.elasticsearch.node.internal;
 
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class InternalSettingsPreparerTests extends ESTestCase {
 
+    Path homeDir;
     Settings baseEnvSettings;
 
     @Before
     public void createBaseEnvSettings() {
+        homeDir = createTempDir();
         baseEnvSettings = Settings.builder()
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(Environment.PATH_HOME_SETTING.getKey(), homeDir)
             .build();
     }
 
     @After
     public void clearBaseEnvSettings() {
+        homeDir = null;
         baseEnvSettings = null;
     }
 
@@ -160,4 +172,28 @@ public class InternalSettingsPreparerTests extends ESTestCase {
             assertTrue(e.getMessage(), e.getMessage().contains(".properties"));
         }
     }
+
+    public void testSecureSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("foo", "secret");
+        Settings input = Settings.builder().put(baseEnvSettings).setSecureSettings(secureSettings).build();
+        Environment env = InternalSettingsPreparer.prepareEnvironment(input, null);
+        Setting<SecureString> fakeSetting = SecureSetting.secureString("foo", null, false);
+        assertEquals("secret", fakeSetting.get(env.settings()).toString());
+    }
+
+    public void testDefaultProperties() throws Exception {
+        Map<String, String> props = Collections.singletonMap("default.setting", "foo");
+        Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null, props);
+        assertEquals("foo", env.settings().get("setting"));
+    }
+
+    public void testDefaultPropertiesOverride() throws Exception {
+        Path configDir = homeDir.resolve("config");
+        Files.createDirectories(configDir);
+        Files.write(configDir.resolve("elasticsearch.yml"), Collections.singletonList("setting: bar"), StandardCharsets.UTF_8);
+        Map<String, String> props = Collections.singletonMap("default.setting", "foo");
+        Environment env = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null, props);
+        assertEquals("bar", env.settings().get("setting"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 59077367904..9c36a7a649a 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -112,13 +112,13 @@ public class NodeInfoStreamingTests extends ESTestCase {
         if (randomBoolean()) {
             int availableProcessors = randomIntBetween(1, 64);
             int allocatedProcessors = randomIntBetween(1, availableProcessors);
-            long refreshInterval = randomBoolean() ? -1 : randomPositiveLong();
+            long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong();
             String name = randomAsciiOfLengthBetween(3, 10);
             String arch = randomAsciiOfLengthBetween(3, 10);
             String version = randomAsciiOfLengthBetween(3, 10);
             osInfo = new OsInfo(refreshInterval, availableProcessors, allocatedProcessors, name, arch, version);
         }
-        ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomPositiveLong());
+        ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomNonNegativeLong());
         JvmInfo jvm = randomBoolean() ? null : JvmInfo.jvmInfo();
         ThreadPoolInfo threadPoolInfo = null;
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index 5b1073beffa..ca7290f5f7a 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -24,10 +24,12 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.replication.ESIndexLevelReplicationTestCase;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveriesCollection;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.PeerRecoveryTargetService;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -54,10 +56,10 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
             final RecoveriesCollection collection = new RecoveriesCollection(logger, threadPool, v -> {});
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             try (RecoveriesCollection.RecoveryRef status = collection.getRecovery(recoveryId)) {
-                final long lastSeenTime = status.status().lastAccessTime();
+                final long lastSeenTime = status.target().lastAccessTime();
                 assertBusy(() -> {
                     try (RecoveriesCollection.RecoveryRef currentStatus = collection.getRecovery(recoveryId)) {
-                        assertThat("access time failed to update", lastSeenTime, lessThan(currentStatus.status().lastAccessTime()));
+                        assertThat("access time failed to update", lastSeenTime, lessThan(currentStatus.target().lastAccessTime()));
                     }
                 });
             } finally {
@@ -100,7 +102,7 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             final long recoveryId2 = startRecovery(collection, shards.getPrimaryNode(), shards.addReplica());
             try (RecoveriesCollection.RecoveryRef recoveryRef = collection.getRecovery(recoveryId)) {
-                ShardId shardId = recoveryRef.status().shardId();
+                ShardId shardId = recoveryRef.target().shardId();
                 assertTrue("failed to cancel recoveries", collection.cancelRecoveriesForShard(shardId, "test"));
                 assertThat("all recoveries should be cancelled", collection.size(), equalTo(0));
             } finally {
@@ -118,30 +120,30 @@ public class RecoveriesCollectionTests extends ESIndexLevelReplicationTestCase {
             final RecoveriesCollection collection = new RecoveriesCollection(logger, threadPool, v -> {});
             IndexShard shard = shards.addReplica();
             final long recoveryId = startRecovery(collection, shards.getPrimaryNode(), shard);
-            try (RecoveriesCollection.RecoveryRef recovery = collection.getRecovery(recoveryId)) {
-                final int currentAsTarget = shard.recoveryStats().currentAsTarget();
-                final int referencesToStore = recovery.status().store().refCount();
-                String tempFileName = recovery.status().getTempNameForFile("foobar");
-                collection.resetRecovery(recoveryId, recovery.status().shardId());
-                try (RecoveriesCollection.RecoveryRef resetRecovery = collection.getRecovery(recoveryId)) {
-                    assertNotSame(recovery.status(), resetRecovery);
-                    assertSame(recovery.status().CancellableThreads(), resetRecovery.status().CancellableThreads());
-                    assertSame(recovery.status().indexShard(), resetRecovery.status().indexShard());
-                    assertSame(recovery.status().store(), resetRecovery.status().store());
-                    assertEquals(referencesToStore + 1, resetRecovery.status().store().refCount());
-                    assertEquals(currentAsTarget+1, shard.recoveryStats().currentAsTarget()); // we blink for a short moment...
-                    recovery.close();
-                    expectThrows(ElasticsearchException.class, () -> recovery.status().store());
-                    assertEquals(referencesToStore, resetRecovery.status().store().refCount());
-                    String resetTempFileName = resetRecovery.status().getTempNameForFile("foobar");
-                    assertNotEquals(tempFileName, resetTempFileName);
-                }
-                assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
-            }
-            try (RecoveriesCollection.RecoveryRef resetRecovery = collection.getRecovery(recoveryId)) {
+            RecoveryTarget recoveryTarget = collection.getRecoveryTarget(recoveryId);
+            final int currentAsTarget = shard.recoveryStats().currentAsTarget();
+            final int referencesToStore = recoveryTarget.store().refCount();
+            IndexShard indexShard = recoveryTarget.indexShard();
+            Store store = recoveryTarget.store();
+            String tempFileName = recoveryTarget.getTempNameForFile("foobar");
+            RecoveryTarget resetRecovery = collection.resetRecovery(recoveryId, TimeValue.timeValueMinutes(60));
+            final long resetRecoveryId = resetRecovery.recoveryId();
+            assertNotSame(recoveryTarget, resetRecovery);
+            assertNotSame(recoveryTarget.cancellableThreads(), resetRecovery.cancellableThreads());
+            assertSame(indexShard, resetRecovery.indexShard());
+            assertSame(store, resetRecovery.store());
+            assertEquals(referencesToStore, resetRecovery.store().refCount());
+            assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
+            assertEquals(recoveryTarget.refCount(), 0);
+            expectThrows(ElasticsearchException.class, () -> recoveryTarget.store());
+            expectThrows(ElasticsearchException.class, () -> recoveryTarget.indexShard());
+            String resetTempFileName = resetRecovery.getTempNameForFile("foobar");
+            assertNotEquals(tempFileName, resetTempFileName);
+            assertEquals(currentAsTarget, shard.recoveryStats().currentAsTarget());
+            try (RecoveriesCollection.RecoveryRef newRecoveryRef = collection.getRecovery(resetRecoveryId)) {
                 shards.recoverReplica(shard, (s, n) -> {
-                    assertSame(s, resetRecovery.status().indexShard());
-                    return resetRecovery.status();
+                    assertSame(s, newRecoveryRef.target().indexShard());
+                    return newRecoveryRef.target();
                 }, false);
             }
             shards.assertAllEqual(numDocs);
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index bbd1ad56bd0..f32688aafff 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -42,6 +42,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.seqno.SeqNoStats;
 import org.elasticsearch.index.seqno.SequenceNumbersService;
@@ -184,7 +185,7 @@ public class RelocationIT extends ESIntegTestCase {
         assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().totalHits(), equalTo(20L));
     }
 
-    @TestLogging("org.elasticsearch.action.index:TRACE,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
+    @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
     public void testRelocationWhileIndexingRandom() throws Exception {
         int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
         int numberOfReplicas = randomBoolean() ? 0 : 1;
@@ -281,7 +282,7 @@ public class RelocationIT extends ESIntegTestCase {
         }
     }
 
-    @TestLogging("org.elasticsearch.action.index:TRACE,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
+    @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
     public void testRelocationWhileRefreshing() throws Exception {
         int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
         int numberOfReplicas = randomBoolean() ? 0 : 1;
@@ -334,12 +335,12 @@ public class RelocationIT extends ESIntegTestCase {
 
             List<IndexRequestBuilder> builders1 = new ArrayList<>();
             for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
-                builders1.add(client().prepareIndex("test", "type").setSource("{}"));
+                builders1.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
             }
 
             List<IndexRequestBuilder> builders2 = new ArrayList<>();
             for (int numDocs = randomIntBetween(10, 30); numDocs > 0; numDocs--) {
-                builders2.add(client().prepareIndex("test", "type").setSource("{}"));
+                builders2.add(client().prepareIndex("test", "type").setSource("{}", XContentType.JSON));
             }
 
             logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
@@ -392,7 +393,7 @@ public class RelocationIT extends ESIntegTestCase {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         int numDocs = scaledRandomIntBetween(25, 250);
         for (int i = 0; i < numDocs; i++) {
-            requests.add(client().prepareIndex(indexName, "type").setSource("{}"));
+            requests.add(client().prepareIndex(indexName, "type").setSource("{}", XContentType.JSON));
         }
         indexRandom(true, requests);
         assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").setWaitForGreenStatus().get().isTimedOut());
@@ -452,7 +453,7 @@ public class RelocationIT extends ESIntegTestCase {
         }
     }
 
-    @TestLogging("org.elasticsearch.action.index:TRACE,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
+    @TestLogging("org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.search:TRACE")
     public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException {
         int halfNodes = randomIntBetween(1, 3);
         Settings[] nodeSettings = Stream.concat(
@@ -516,7 +517,7 @@ public class RelocationIT extends ESIntegTestCase {
 
         private final CountDownLatch corruptionCount;
 
-        public RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) {
+        RecoveryCorruption(Transport transport, CountDownLatch corruptionCount) {
             super(transport);
             this.corruptionCount = corruptionCount;
         }
diff --git a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
index e860cfc9ebd..bf8bce8ae63 100644
--- a/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/SimpleRecoveryIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.client.Requests.flushRequest;
@@ -52,12 +53,12 @@ public class SimpleRecoveryIT extends ESIntegTestCase {
 
         NumShards numShards = getNumShards("test");
 
-        client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"))).actionGet();
+        client().index(indexRequest("test").type("type1").id("1").source(source("1", "test"), XContentType.JSON)).actionGet();
         FlushResponse flushResponse = client().admin().indices().flush(flushRequest("test")).actionGet();
         assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards));
         assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
         assertThat(flushResponse.getFailedShards(), equalTo(0));
-        client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"))).actionGet();
+        client().index(indexRequest("test").type("type1").id("2").source(source("2", "test"), XContentType.JSON)).actionGet();
         RefreshResponse refreshResponse = client().admin().indices().refresh(refreshRequest("test")).actionGet();
         assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards));
         assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
diff --git a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
index c1ac1abfdb5..f9c620e3b9e 100644
--- a/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/RepositoryDataTests.java
@@ -37,6 +37,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.repositories.RepositoryData.EMPTY_REPO_GEN;
 import static org.hamcrest.Matchers.greaterThan;
 
 /**
@@ -54,18 +55,18 @@ public class RepositoryDataTests extends ESTestCase {
     public void testXContent() throws IOException {
         RepositoryData repositoryData = generateRandomRepoData();
         XContentBuilder builder = JsonXContent.contentBuilder();
-        repositoryData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        repositoryData.snapshotsToXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
-        assertEquals(repositoryData, RepositoryData.fromXContent(parser));
+        long gen = (long) randomIntBetween(0, 500);
+        RepositoryData fromXContent = RepositoryData.snapshotsFromXContent(parser, gen);
+        assertEquals(repositoryData, fromXContent);
+        assertEquals(gen, fromXContent.getGenId());
     }
 
     public void testAddSnapshots() {
         RepositoryData repositoryData = generateRandomRepoData();
         // test that adding the same snapshot id to the repository data throws an exception
-        final SnapshotId snapshotId = repositoryData.getSnapshotIds().get(0);
         Map<String, IndexId> indexIdMap = repositoryData.getIndices();
-        expectThrows(IllegalArgumentException.class,
-            () -> repositoryData.addSnapshot(new SnapshotId(snapshotId.getName(), snapshotId.getUUID()), Collections.emptyList()));
         // test that adding a snapshot and its indices works
         SnapshotId newSnapshot = new SnapshotId(randomAsciiOfLength(7), UUIDs.randomBase64UUID());
         List<IndexId> indices = new ArrayList<>();
@@ -91,6 +92,7 @@ public class RepositoryDataTests extends ESTestCase {
                 assertEquals(snapshotIds.size(), 1); // if it was a new index, only the new snapshot should be in its set
             }
         }
+        assertEquals(repositoryData.getGenId(), newRepoData.getGenId());
     }
 
     public void testInitIndices() {
@@ -99,7 +101,7 @@ public class RepositoryDataTests extends ESTestCase {
         for (int i = 0; i < numSnapshots; i++) {
             snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
         }
-        RepositoryData repositoryData = new RepositoryData(snapshotIds, Collections.emptyMap());
+        RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), Collections.emptyList());
         // test that initializing indices works
         Map<IndexId, Set<SnapshotId>> indices = randomIndices(snapshotIds);
         RepositoryData newRepoData = repositoryData.initIndices(indices);
@@ -140,7 +142,7 @@ public class RepositoryDataTests extends ESTestCase {
 
     public static RepositoryData generateRandomRepoData(final List<SnapshotId> origSnapshotIds) {
         List<SnapshotId> snapshotIds = randomSnapshots(origSnapshotIds);
-        return new RepositoryData(snapshotIds, randomIndices(snapshotIds));
+        return new RepositoryData(EMPTY_REPO_GEN, snapshotIds, randomIndices(snapshotIds), Collections.emptyList());
     }
 
     private static List<SnapshotId> randomSnapshots(final List<SnapshotId> origSnapshotIds) {
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index 07e6aa0f16c..6e538e721a4 100644
--- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -24,12 +24,15 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryData;
+import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -93,9 +96,8 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
 
-        List<SnapshotId> snapshotIds = repository.getSnapshots().stream()
-                                                             .sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
-                                                             .collect(Collectors.toList());
+        List<SnapshotId> snapshotIds = repository.getRepositoryData().getSnapshotIds().stream()
+            .sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
         assertThat(snapshotIds, equalTo(originalSnapshots));
     }
 
@@ -103,28 +105,24 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         final BlobStoreRepository repository = setupRepo();
 
         // write to and read from a index file with no entries
-        assertThat(repository.getSnapshots().size(), equalTo(0));
+        assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0));
         final RepositoryData emptyData = RepositoryData.EMPTY;
-        repository.writeIndexGen(emptyData);
-        final RepositoryData readData = repository.getRepositoryData();
-        assertEquals(readData, emptyData);
-        assertEquals(readData.getIndices().size(), 0);
-        assertEquals(readData.getSnapshotIds().size(), 0);
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        RepositoryData repoData = repository.getRepositoryData();
+        assertEquals(repoData, emptyData);
+        assertEquals(repoData.getIndices().size(), 0);
+        assertEquals(repoData.getSnapshotIds().size(), 0);
+        assertEquals(0L, repoData.getGenId());
 
         // write to and read from an index file with snapshots but no indices
-        final int numSnapshots = randomIntBetween(1, 20);
-        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
-        for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
-        }
-        RepositoryData repositoryData = new RepositoryData(snapshotIds, Collections.emptyMap());
-        repository.writeIndexGen(repositoryData);
-        assertEquals(repository.getRepositoryData(), repositoryData);
+        repoData = addRandomSnapshotsToRepoData(repoData, false);
+        repository.writeIndexGen(repoData, repoData.getGenId());
+        assertEquals(repoData, repository.getRepositoryData());
 
         // write to and read from a index file with random repository data
-        repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
-        assertThat(repository.getRepositoryData(), equalTo(repositoryData));
+        repoData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
+        repository.writeIndexGen(repoData, repoData.getGenId());
+        assertEquals(repoData, repository.getRepositoryData());
     }
 
     public void testIndexGenerationalFiles() throws Exception {
@@ -132,26 +130,65 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
 
         // write to index generational file
         RepositoryData repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertThat(repository.getRepositoryData(), equalTo(repositoryData));
         assertThat(repository.latestIndexBlobId(), equalTo(0L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L));
 
         // adding more and writing to a new index generational file
-        repositoryData = generateRandomRepoData();
-        repository.writeIndexGen(repositoryData);
+        repositoryData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertEquals(repository.getRepositoryData(), repositoryData);
         assertThat(repository.latestIndexBlobId(), equalTo(1L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L));
 
         // removing a snapshot and writing to a new index generational file
-        repositoryData = repositoryData.removeSnapshot(repositoryData.getSnapshotIds().get(0));
-        repository.writeIndexGen(repositoryData);
+        repositoryData = repository.getRepositoryData().removeSnapshot(repositoryData.getSnapshotIds().get(0));
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
         assertEquals(repository.getRepositoryData(), repositoryData);
         assertThat(repository.latestIndexBlobId(), equalTo(2L));
         assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L));
     }
 
+    public void testRepositoryDataConcurrentModificationNotAllowed() throws IOException {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to index generational file
+        RepositoryData repositoryData = generateRandomRepoData();
+        repository.writeIndexGen(repositoryData, repositoryData.getGenId());
+
+        // write repo data again to index generational file, errors because we already wrote to the
+        // N+1 generation from which this repository data instance was created
+        expectThrows(RepositoryException.class, () -> repository.writeIndexGen(repositoryData, repositoryData.getGenId()));
+    }
+
+    public void testReadAndWriteIncompatibleSnapshots() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to and read from incompatible snapshots file with no entries
+        assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
+        RepositoryData emptyData = RepositoryData.EMPTY;
+        repository.writeIndexGen(emptyData, emptyData.getGenId());
+        repository.writeIncompatibleSnapshots(emptyData);
+        RepositoryData readData = repository.getRepositoryData();
+        assertEquals(emptyData, readData);
+        assertEquals(0, readData.getIndices().size());
+        assertEquals(0, readData.getSnapshotIds().size());
+
+        // write to and read from incompatible snapshots with some number of entries
+        final int numSnapshots = randomIntBetween(1, 20);
+        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        RepositoryData repositoryData = new RepositoryData(readData.getGenId(), Collections.emptyList(), Collections.emptyMap(),
+                                                              snapshotIds);
+        repository.blobContainer().deleteBlob("incompatible-snapshots");
+        repository.writeIncompatibleSnapshots(repositoryData);
+        readData = repository.getRepositoryData();
+        assertEquals(repositoryData.getIncompatibleSnapshotIds(), readData.getIncompatibleSnapshotIds());
+    }
+
     private BlobStoreRepository setupRepo() {
         final Client client = client();
         final Path location = ESIntegTestCase.randomRepoPath(node().settings());
@@ -170,4 +207,18 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         return repository;
     }
 
+    private RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boolean inclIndices) {
+        int numSnapshots = randomIntBetween(1, 20);
+        for (int i = 0; i < numSnapshots; i++) {
+            SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
+            int numIndices = inclIndices ? randomIntBetween(0, 20) : 0;
+            List<IndexId> indexIds = new ArrayList<>(numIndices);
+            for (int j = 0; j < numIndices; j++) {
+                indexIds.add(new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+            }
+            repoData = repoData.addSnapshot(snapshotId, indexIds);
+        }
+        return repoData;
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java b/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java
index ef67006c204..80ee4dd6d0d 100644
--- a/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BaseRestHandlerTests.java
@@ -52,7 +52,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         params.put("consumed", randomAsciiOfLength(8));
         params.put("unconsumed", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
             expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
@@ -74,7 +74,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("consumed", randomAsciiOfLength(8));
         params.put("unconsumed-first", randomAsciiOfLength(8));
         params.put("unconsumed-second", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
             expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
@@ -108,7 +108,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("tokenzier", randomAsciiOfLength(8));
         params.put("very_close_to_parametre", randomAsciiOfLength(8));
         params.put("very_far_from_every_consumed_parameter", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         final IllegalArgumentException e =
             expectThrows(IllegalArgumentException.class, () -> handler.handleRequest(request, channel, mock(NodeClient.class)));
@@ -142,7 +142,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         params.put("consumed", randomAsciiOfLength(8));
         params.put("response_param", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         handler.handleRequest(request, channel, mock(NodeClient.class));
         assertTrue(executed.get());
@@ -160,9 +160,9 @@ public class BaseRestHandlerTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         params.put("format", randomAsciiOfLength(8));
         params.put("filter_path", randomAsciiOfLength(8));
-        params.put("pretty", randomAsciiOfLength(8));
-        params.put("human", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        params.put("pretty", randomFrom("true", "false", "", null));
+        params.put("human", null);
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         handler.handleRequest(request, channel, mock(NodeClient.class));
         assertTrue(executed.get());
@@ -196,7 +196,7 @@ public class BaseRestHandlerTests extends ESTestCase {
         params.put("bytes", randomAsciiOfLength(8));
         params.put("size", randomAsciiOfLength(8));
         params.put("time", randomAsciiOfLength(8));
-        RestRequest request = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
         RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
         handler.handleRequest(request, channel, mock(NodeClient.class));
         assertTrue(executed.get());
diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
index b9f97156df3..431bc152bd9 100644
--- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
@@ -20,11 +20,19 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
@@ -36,23 +44,18 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
+import static org.elasticsearch.ElasticsearchExceptionTests.assertDeepEquals;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class BytesRestResponseTests extends ESTestCase {
 
     class UnknownException extends Exception {
-
-        public UnknownException(final String message, final Throwable cause) {
+        UnknownException(final String message, final Throwable cause) {
             super(message, cause);
         }
-
     }
 
     public void testWithHeaders() throws Exception {
@@ -60,6 +63,7 @@ public class BytesRestResponseTests extends ESTestCase {
         RestChannel channel = randomBoolean() ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request);
 
         BytesRestResponse response = new BytesRestResponse(channel, new WithHeadersException());
+        assertEquals(2, response.getHeaders().size());
         assertThat(response.getHeaders().get("n1"), notNullValue());
         assertThat(response.getHeaders().get("n1"), contains("v11", "v12"));
         assertThat(response.getHeaders().get("n2"), notNullValue());
@@ -161,7 +165,7 @@ public class BytesRestResponseTests extends ESTestCase {
 
     public void testResponseWhenPathContainsEncodingError() throws IOException {
         final String path = "%a";
-        final RestRequest request = new RestRequest(Collections.emptyMap(), path) {
+        final RestRequest request = new RestRequest(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, Collections.emptyMap()) {
             @Override
             public Method method() {
                 return null;
@@ -181,16 +185,6 @@ public class BytesRestResponseTests extends ESTestCase {
             public BytesReference content() {
                 return null;
             }
-
-            @Override
-            public String header(String name) {
-                return null;
-            }
-
-            @Override
-            public Iterable<Map.Entry<String, String>> headers() {
-                return null;
-            }
         };
         final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath()));
         final RestChannel channel = new DetailedExceptionRestChannel(request);
@@ -214,12 +208,151 @@ public class BytesRestResponseTests extends ESTestCase {
         assertThat(content, containsString("\"status\":" + 500));
     }
 
+    public void testErrorToAndFromXContent() throws IOException {
+        final boolean detailed = randomBoolean();
+
+        Exception original;
+        ElasticsearchException cause = null;
+        String reason;
+        String type = "exception";
+        RestStatus status = RestStatus.INTERNAL_SERVER_ERROR;
+        boolean addHeadersOrMetadata = false;
+
+        switch (randomIntBetween(0, 5)) {
+            case 0:
+                original = new ElasticsearchException("ElasticsearchException without cause");
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    reason = "ElasticsearchException without cause";
+                } else {
+                    reason = "ElasticsearchException[ElasticsearchException without cause]";
+                }
+                break;
+            case 1:
+                original = new ElasticsearchException("ElasticsearchException with a cause", new FileNotFoundException("missing"));
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "exception";
+                    reason = "ElasticsearchException with a cause";
+                    cause = new ElasticsearchException("Elasticsearch exception [type=file_not_found_exception, reason=missing]");
+                } else {
+                    reason = "ElasticsearchException[ElasticsearchException with a cause]";
+                }
+                break;
+            case 2:
+                original = new ResourceNotFoundException("ElasticsearchException with custom status");
+                status = RestStatus.NOT_FOUND;
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "resource_not_found_exception";
+                    reason = "ElasticsearchException with custom status";
+                } else {
+                    reason = "ResourceNotFoundException[ElasticsearchException with custom status]";
+                }
+                break;
+            case 3:
+                TransportAddress address = buildNewFakeTransportAddress();
+                original = new RemoteTransportException("remote", address, "action",
+                        new ResourceAlreadyExistsException("ElasticsearchWrapperException with a cause that has a custom status"));
+                status = RestStatus.BAD_REQUEST;
+                if (detailed) {
+                    type = "resource_already_exists_exception";
+                    reason = "ElasticsearchWrapperException with a cause that has a custom status";
+                } else {
+                    reason = "RemoteTransportException[[remote][" + address.toString() + "][action]]";
+                }
+                break;
+            case 4:
+                original = new RemoteTransportException("ElasticsearchWrapperException with a cause that has a special treatment",
+                        new IllegalArgumentException("wrong"));
+                status = RestStatus.BAD_REQUEST;
+                if (detailed) {
+                    type = "illegal_argument_exception";
+                    reason = "wrong";
+                } else {
+                    reason = "RemoteTransportException[[ElasticsearchWrapperException with a cause that has a special treatment]]";
+                }
+                break;
+            case 5:
+                status = randomFrom(RestStatus.values());
+                original = new ElasticsearchStatusException("ElasticsearchStatusException with random status", status);
+                if (detailed) {
+                    addHeadersOrMetadata = randomBoolean();
+                    type = "status_exception";
+                    reason = "ElasticsearchStatusException with random status";
+                } else {
+                    reason = "ElasticsearchStatusException[ElasticsearchStatusException with random status]";
+                }
+                break;
+            default:
+                throw new UnsupportedOperationException("Failed to generate random exception");
+        }
+
+        String message = "Elasticsearch exception [type=" + type + ", reason=" + reason + "]";
+        ElasticsearchStatusException expected = new ElasticsearchStatusException(message, status, cause);
+
+        if (addHeadersOrMetadata) {
+            ElasticsearchException originalException = ((ElasticsearchException) original);
+            if (randomBoolean()) {
+                originalException.addHeader("foo", "bar", "baz");
+                expected.addHeader("foo", "bar", "baz");
+            }
+            if (randomBoolean()) {
+                originalException.addMetadata("es.metadata_0", "0");
+                expected.addMetadata("es.metadata_0", "0");
+            }
+            if (randomBoolean()) {
+                String resourceType = randomAsciiOfLength(5);
+                String resourceId = randomAsciiOfLength(5);
+                originalException.setResources(resourceType, resourceId);
+                expected.setResources(resourceType, resourceId);
+            }
+            if (randomBoolean()) {
+                originalException.setIndex("_index");
+                expected.setIndex("_index");
+            }
+        }
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        Map<String, String> params = Collections.singletonMap("format", xContentType.mediaType());
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
+        RestChannel channel = detailed ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request);
+
+        BytesRestResponse response = new BytesRestResponse(channel, original);
+
+        ElasticsearchException parsedError;
+        try (XContentParser parser = createParser(xContentType.xContent(), response.content())) {
+            parsedError = BytesRestResponse.errorFromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+
+        assertEquals(expected.status(), parsedError.status());
+        assertDeepEquals(expected, parsedError);
+    }
+
+    public void testNoErrorFromXContent() throws IOException {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
+                builder.startObject();
+                builder.field("status", randomFrom(RestStatus.values()).getStatus());
+                builder.endObject();
+
+                try (XContentParser parser = createParser(builder.contentType().xContent(), builder.bytes())) {
+                    BytesRestResponse.errorFromXContent(parser);
+                }
+            }
+        });
+        assertEquals("Failed to parse elasticsearch status exception: no exception was found", e.getMessage());
+    }
+
     public static class WithHeadersException extends ElasticsearchException {
 
         WithHeadersException() {
             super("");
             this.addHeader("n1", "v11", "v12");
             this.addHeader("n2", "v21", "v22");
+            this.addMetadata("es.test", "value1", "value2");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
index 7f46139a269..3c31b5b7f4d 100644
--- a/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
@@ -138,7 +138,7 @@ public class DeprecationRestHandlerTests extends ESTestCase {
         /**
          * Create a generator for characters [32, 126].
          */
-        public ASCIIHeaderGenerator() {
+        ASCIIHeaderGenerator() {
             super(asciiFromTo(32, 126));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 228561089af..50f9afd5c28 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -23,17 +23,34 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.UnaryOperator;
 
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.junit.Before;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
@@ -43,37 +60,65 @@ import static org.mockito.Mockito.verify;
 
 public class RestControllerTests extends ESTestCase {
 
+    private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20);
+    private CircuitBreaker inFlightRequestsBreaker;
+    private RestController restController;
+    private HierarchyCircuitBreakerService circuitBreakerService;
+
+    @Before
+    public void setup() {
+        Settings settings = Settings.EMPTY;
+        circuitBreakerService = new HierarchyCircuitBreakerService(
+            Settings.builder()
+                .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), BREAKER_LIMIT)
+                .build(),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        // we can do this here only because we know that we don't adjust breaker settings dynamically in the test
+        inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+
+        HttpServerTransport httpServerTransport = new TestHttpServerTransport();
+        restController = new RestController(settings, Collections.emptySet(), null, null, circuitBreakerService);
+        restController.registerHandler(RestRequest.Method.GET, "/",
+            (request, channel, client) -> channel.sendResponse(
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
+        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
+            throw new IllegalArgumentException("test error");
+        });
+
+        httpServerTransport.start();
+    }
+
+
     public void testApplyRelevantHeaders() throws Exception {
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
         Set<String> headers = new HashSet<>(Arrays.asList("header.1", "header.2"));
-        final RestController restController = new RestController(Settings.EMPTY, headers, null);
-        restController.registerHandler(RestRequest.Method.GET, "/",
-            (RestRequest request, RestChannel channel, NodeClient client) -> {
+        final RestController restController = new RestController(Settings.EMPTY, headers, null, null, circuitBreakerService);
+        threadContext.putHeader("header.3", "true");
+        Map<String, List<String>> restHeaders = new HashMap<>();
+        restHeaders.put("header.1", Collections.singletonList("true"));
+        restHeaders.put("header.2", Collections.singletonList("true"));
+        restHeaders.put("header.3", Collections.singletonList("false"));
+        restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).withHeaders(restHeaders).build(), null, null,
+                threadContext, (RestRequest request, RestChannel channel, NodeClient client) -> {
                 assertEquals("true", threadContext.getHeader("header.1"));
                 assertEquals("true", threadContext.getHeader("header.2"));
                 assertNull(threadContext.getHeader("header.3"));
             });
-        threadContext.putHeader("header.3", "true");
-        Map<String, String> restHeaders = new HashMap<>();
-        restHeaders.put("header.1", "true");
-        restHeaders.put("header.2", "true");
-        restHeaders.put("header.3", "false");
-        restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, null, threadContext);
         assertNull(threadContext.getHeader("header.1"));
         assertNull(threadContext.getHeader("header.2"));
         assertEquals("true", threadContext.getHeader("header.3"));
     }
 
     public void testCanTripCircuitBreaker() throws Exception {
-        RestController controller = new RestController(Settings.EMPTY, Collections.emptySet(), null);
+        RestController controller = new RestController(Settings.EMPTY, Collections.emptySet(), null, null, circuitBreakerService);
         // trip circuit breaker by default
         controller.registerHandler(RestRequest.Method.GET, "/trip", new FakeRestHandler(true));
         controller.registerHandler(RestRequest.Method.GET, "/do-not-trip", new FakeRestHandler(false));
 
-        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/trip").build()));
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/trip").build()));
         // assume trip even on unknown paths
-        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/unknown-path").build()));
-        assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/do-not-trip").build()));
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/unknown-path").build()));
+        assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder(xContentRegistry()).withPath("/do-not-trip").build()));
     }
 
     public void testRegisterAsDeprecatedHandler() {
@@ -125,10 +170,10 @@ public class RestControllerTests extends ESTestCase {
             assertSame(handler, h);
             return (RestRequest request, RestChannel channel, NodeClient client) -> wrapperCalled.set(true);
         };
-        final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper);
-        restController.registerHandler(RestRequest.Method.GET, "/", handler);
+        final RestController restController = new RestController(Settings.EMPTY, Collections.emptySet(), wrapper, null,
+            circuitBreakerService);
         final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-        restController.dispatchRequest(new FakeRestRequest.Builder().build(), null, null, threadContext);
+        restController.dispatchRequest(new FakeRestRequest.Builder(xContentRegistry()).build(), null, null, threadContext, handler);
         assertTrue(wrapperCalled.get());
         assertFalse(handlerCalled.get());
     }
@@ -153,4 +198,210 @@ public class RestControllerTests extends ESTestCase {
             return canTripCircuitBreaker;
         }
     }
+
+    public void testDispatchRequestAddsAndFreesBytesOnSuccess() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAsciiOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.OK);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestAddsAndFreesBytesOnError() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAsciiOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.BAD_REQUEST);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestAddsAndFreesBytesOnlyOnceOnError() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt();
+        String content = randomAsciiOfLength(contentLength);
+        // we will produce an error in the rest handler and one more when sending the error response
+        TestRestRequest request = new TestRestRequest("/error", content, XContentType.JSON);
+        ExceptionThrowingChannel channel = new ExceptionThrowingChannel(request, true);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(0, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequestLimitsBytes() {
+        int contentLength = BREAKER_LIMIT.bytesAsInt() + 1;
+        String content = randomAsciiOfLength(contentLength);
+        TestRestRequest request = new TestRestRequest("/", content, XContentType.JSON);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.SERVICE_UNAVAILABLE);
+
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+
+        assertEquals(1, inFlightRequestsBreaker.getTrippedCount());
+        assertEquals(0, inFlightRequestsBreaker.getUsed());
+    }
+
+    public void testDispatchRequiresContentTypeForRequestsWithContent() {
+        String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        TestRestRequest request = new TestRestRequest("/", content, null);
+        AssertingChannel channel = new AssertingChannel(request, true, RestStatus.NOT_ACCEPTABLE);
+        restController = new RestController(
+            Settings.builder().put(HttpTransportSettings.SETTING_HTTP_CONTENT_TYPE_REQUIRED.getKey(), true).build(),
+            Collections.emptySet(), null, null, circuitBreakerService);
+        restController.registerHandler(RestRequest.Method.GET, "/",
+            (r, c, client) -> c.sendResponse(
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
+
+        assertFalse(channel.sendResponseCalled.get());
+        restController.dispatchRequest(request, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.sendResponseCalled.get());
+    }
+
+    public void testDispatchDoesNotRequireContentTypeForRequestsWithoutContent() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+
+        assertFalse(channel.sendResponseCalled.get());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.sendResponseCalled.get());
+    }
+
+    public void testDispatchWorksWithPlainText() {
+        String content = randomAsciiOfLengthBetween(1, BREAKER_LIMIT.bytesAsInt());
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray(content), null).withPath("/foo")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("text/plain"))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+        restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+                channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
+            }
+
+            @Override
+            public boolean supportsPlainText() {
+                return true;
+            }
+        });
+
+        assertFalse(channel.sendResponseCalled.get());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.sendResponseCalled.get());
+        assertWarnings("Plain text request bodies are deprecated. Use request parameters or body in a supported format.");
+    }
+
+    public void testDispatchWorksWithAutoDetection() {
+        FakeRestRequest fakeRestRequest = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
+            .withContent(new BytesArray("{}"), null).withPath("/")
+            .withHeaders(Collections.singletonMap("Content-Type", Collections.singletonList("application/x-www-form-urlencoded"))).build();
+        AssertingChannel channel = new AssertingChannel(fakeRestRequest, true, RestStatus.OK);
+
+        assertFalse(channel.sendResponseCalled.get());
+        restController.dispatchRequest(fakeRestRequest, channel, new ThreadContext(Settings.EMPTY));
+        assertTrue(channel.sendResponseCalled.get());
+        assertWarnings("Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header.");
+    }
+
+    private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
+        HttpServerTransport {
+
+        TestHttpServerTransport() {
+            super(Settings.EMPTY);
+        }
+
+        @Override
+        protected void doStart() {
+        }
+
+        @Override
+        protected void doStop() {
+        }
+
+        @Override
+        protected void doClose() {
+        }
+
+        @Override
+        public BoundTransportAddress boundAddress() {
+            TransportAddress transportAddress = buildNewFakeTransportAddress();
+            return new BoundTransportAddress(new TransportAddress[] {transportAddress} ,transportAddress);
+        }
+
+        @Override
+        public HttpInfo info() {
+            return null;
+        }
+
+        @Override
+        public HttpStats stats() {
+            return null;
+        }
+    }
+
+    private static final class AssertingChannel extends AbstractRestChannel {
+        private final RestStatus expectedStatus;
+        private final AtomicBoolean sendResponseCalled = new AtomicBoolean(false);
+
+        protected AssertingChannel(RestRequest request, boolean detailedErrorsEnabled, RestStatus expectedStatus) {
+            super(request, detailedErrorsEnabled);
+            this.expectedStatus = expectedStatus;
+        }
+
+        @Override
+        public void sendResponse(RestResponse response) {
+            assertEquals(expectedStatus, response.status());
+            sendResponseCalled.set(true);
+        }
+    }
+
+    private static final class ExceptionThrowingChannel extends AbstractRestChannel {
+
+        protected ExceptionThrowingChannel(RestRequest request, boolean detailedErrorsEnabled) {
+            super(request, detailedErrorsEnabled);
+        }
+
+        @Override
+        public void sendResponse(RestResponse response) {
+            throw new IllegalStateException("always throwing an exception for testing");
+        }
+    }
+
+    private static final class TestRestRequest extends RestRequest {
+
+        private final BytesReference content;
+
+        private TestRestRequest(String path, String content, XContentType xContentType) {
+            super(NamedXContentRegistry.EMPTY, Collections.emptyMap(), path, xContentType == null ?
+                Collections.emptyMap() : Collections.singletonMap("Content-Type", Collections.singletonList(xContentType.mediaType())));
+            this.content = new BytesArray(content);
+        }
+
+        @Override
+        public Method method() {
+            return Method.GET;
+        }
+
+        @Override
+        public String uri() {
+            return null;
+        }
+
+        @Override
+        public boolean hasContent() {
+            return true;
+        }
+
+        @Override
+        public BytesReference content() {
+            return content;
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java
index 0511b3919b7..bddcd39160a 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestRequestTests.java
@@ -23,11 +23,16 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static java.util.Collections.emptyMap;
@@ -53,10 +58,13 @@ public class RestRequestTests extends ESTestCase {
     }
 
     public void testContentOrSourceParam() throws IOException {
-        assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam());
-        assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam());
-        assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam());
-        assertEquals(new BytesArray("stuff"), new ContentRestRequest("", singletonMap("source", "stuff")).contentOrSourceParam());
+        assertEquals(BytesArray.EMPTY, new ContentRestRequest("", emptyMap()).contentOrSourceParam().v2());
+        assertEquals(new BytesArray("stuff"), new ContentRestRequest("stuff", emptyMap()).contentOrSourceParam().v2());
+        assertEquals(new BytesArray("stuff"),
+            new ContentRestRequest("stuff", singletonMap("source", "stuff2")).contentOrSourceParam().v2());
+        assertEquals(new BytesArray("{\"foo\": \"stuff\"}"),
+            new ContentRestRequest("", singletonMap("source", "{\"foo\": \"stuff\"}")).contentOrSourceParam().v2());
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter.");
     }
 
     public void testHasContentOrSourceParam() throws IOException {
@@ -73,6 +81,7 @@ public class RestRequestTests extends ESTestCase {
         assertEquals(emptyMap(), new ContentRestRequest("{}", emptyMap()).contentOrSourceParamParser().map());
         assertEquals(emptyMap(), new ContentRestRequest("{}", singletonMap("source", "stuff2")).contentOrSourceParamParser().map());
         assertEquals(emptyMap(), new ContentRestRequest("", singletonMap("source", "{}")).contentOrSourceParamParser().map());
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter.");
     }
 
     public void testWithContentOrSourceParamParserOrNull() throws IOException {
@@ -82,12 +91,58 @@ public class RestRequestTests extends ESTestCase {
                 assertEquals(emptyMap(), parser.map()));
         new ContentRestRequest("", singletonMap("source", "{}")).withContentOrSourceParamParserOrNull(parser ->
                 assertEquals(emptyMap(), parser.map()));
+        assertWarnings("Deprecated use of the [source] parameter without the [source_content_type] parameter.");
+    }
+
+    public void testContentTypeParsing() {
+        for (XContentType xContentType : XContentType.values()) {
+            Map<String, List<String>> map = new HashMap<>();
+            map.put("Content-Type", Collections.singletonList(xContentType.mediaType()));
+            ContentRestRequest restRequest = new ContentRestRequest("", Collections.emptyMap(), map);
+            assertEquals(xContentType, restRequest.getXContentType());
+
+            map = new HashMap<>();
+            map.put("Content-Type", Collections.singletonList(xContentType.mediaTypeWithoutParameters()));
+            restRequest = new ContentRestRequest("", Collections.emptyMap(), map);
+            assertEquals(xContentType, restRequest.getXContentType());
+        }
+    }
+
+    public void testPlainTextSupport() {
+        ContentRestRequest restRequest = new ContentRestRequest(randomAsciiOfLengthBetween(1, 30), Collections.emptyMap(),
+            Collections.singletonMap("Content-Type",
+                Collections.singletonList(randomFrom("text/plain", "text/plain; charset=utf-8", "text/plain;charset=utf-8"))));
+        assertNull(restRequest.getXContentType());
+    }
+
+    public void testMalformedContentTypeHeader() {
+        final String type = randomFrom("text", "text/:ain; charset=utf-8", "text/plain\";charset=utf-8", ":", "/", "t:/plain");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(),
+            Collections.singletonMap("Content-Type", Collections.singletonList(type))));
+        assertEquals("invalid Content-Type header [" + type + "]", e.getMessage());
+    }
+
+    public void testNoContentTypeHeader() {
+        ContentRestRequest contentRestRequest = new ContentRestRequest("", Collections.emptyMap(), Collections.emptyMap());
+        assertNull(contentRestRequest.getXContentType());
+    }
+
+    public void testMultipleContentTypeHeaders() {
+        List<String> headers = new ArrayList<>(randomUnique(() -> randomAsciiOfLengthBetween(1, 16), randomIntBetween(2, 10)));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new ContentRestRequest("", Collections.emptyMap(),
+            Collections.singletonMap("Content-Type", headers)));
+        assertEquals("only one Content-Type header should be provided", e.getMessage());
     }
 
     private static final class ContentRestRequest extends RestRequest {
         private final BytesArray content;
-        public ContentRestRequest(String content, Map<String, String> params) {
-            super(params, "not used by this test");
+
+        ContentRestRequest(String content, Map<String, String> params) {
+            this(content, params, Collections.singletonMap("Content-Type", Collections.singletonList("application/json")));
+        }
+
+        ContentRestRequest(String content, Map<String, String> params, Map<String, List<String>> headers) {
+            super(NamedXContentRegistry.EMPTY, params, "not used by this test", headers);
             this.content = new BytesArray(content);
         }
 
@@ -95,7 +150,7 @@ public class RestRequestTests extends ESTestCase {
         public boolean hasContent() {
             return Strings.hasLength(content);
         }
-        
+
         @Override
         public BytesReference content() {
             return content;
@@ -110,15 +165,5 @@ public class RestRequestTests extends ESTestCase {
         public Method method() {
             throw new UnsupportedOperationException("Not used by this test");
         }
-
-        @Override
-        public Iterable<Entry<String, String>> headers() {
-            throw new UnsupportedOperationException("Not used by this test");
-        }
-
-        @Override
-        public String header(String name) {
-            throw new UnsupportedOperationException("Not used by this test");
-        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
index 449f5852cfa..8a78cce8259 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/RestMainActionTests.java
@@ -83,7 +83,7 @@ public class RestMainActionTests extends ESTestCase {
         if (prettyPrint == false) {
             params.put("pretty", String.valueOf(prettyPrint));
         }
-        RestRequest restRequest = new FakeRestRequest.Builder().withParams(params).build();
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withParams(params).build();
 
         BytesRestResponse response = RestMainAction.convertMainResponse(mainResponse, restRequest, builder);
         assertNotNull(response);
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
index 812bc7a85a8..ba478331ca0 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsActionTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster;
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -43,14 +44,14 @@ public class RestNodesStatsActionTests extends ESTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        action = new RestNodesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null));
+        action = new RestNodesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null));
     }
 
     public void testUnrecognizedMetric() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         final String metric = randomAsciiOfLength(64);
         params.put("metric", metric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -60,7 +61,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
     public void testUnrecognizedMetricDidYouMean() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         params.put("metric", "os,transprot,unrecognized");
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -75,7 +76,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         final String metric = randomSubsetOf(1, RestNodesStatsAction.METRICS.keySet()).get(0);
         params.put("metric", "_all," + metric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -87,7 +88,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
         params.put("metric", "indices");
         final String indexMetric = randomAsciiOfLength(64);
         params.put("index_metric", indexMetric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -98,7 +99,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         params.put("metric", "indices");
         params.put("index_metric", "indexing,stroe,unrecognized");
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -116,7 +117,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
         params.put("metric", randomSubsetOf(1, metrics).get(0));
         final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0);
         params.put("index_metric", indexMetric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -131,7 +132,7 @@ public class RestNodesStatsActionTests extends ESTestCase {
         params.put("metric", "_all");
         final String indexMetric = randomSubsetOf(1, RestNodesStatsAction.FLAGS.keySet()).get(0);
         params.put("index_metric", indexMetric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_nodes/stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_nodes/stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
index d7d5d12e91b..958b9e5222f 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestAnalyzeActionTests.java
@@ -19,11 +19,11 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.test.ESTestCase;
@@ -46,7 +46,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
 
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
 
-        RestAnalyzeAction.buildFromContent(content, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY));
+        RestAnalyzeAction.buildFromContent(content, analyzeRequest);
 
         assertThat(analyzeRequest.text().length, equalTo(1));
         assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
@@ -79,7 +79,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
 
         AnalyzeRequest analyzeRequest = new AnalyzeRequest("for test");
 
-        RestAnalyzeAction.buildFromContent(content, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY));
+        RestAnalyzeAction.buildFromContent(content, analyzeRequest);
 
         assertThat(analyzeRequest.text().length, equalTo(1));
         assertThat(analyzeRequest.text(), equalTo(new String[]{"THIS IS A TEST"}));
@@ -93,7 +93,8 @@ public class RestAnalyzeActionTests extends ESTestCase {
 
     public void testParseXContentForAnalyzeRequestWithInvalidJsonThrowsException() throws Exception {
         RestAnalyzeAction action = new RestAnalyzeAction(Settings.EMPTY, mock(RestController.class));
-        RestRequest request = new FakeRestRequest.Builder().withContent(new BytesArray("{invalid_json}")).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> action.handleRequest(request, null, null));
         assertThat(e.getMessage(), equalTo("Failed to parse request body"));
     }
@@ -106,7 +107,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                 .field("unknown", "keyword")
             .endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)));
+                () -> RestAnalyzeAction.buildFromContent(invalidContent, analyzeRequest));
         assertThat(e.getMessage(), startsWith("Unknown parameter [unknown]"));
     }
 
@@ -117,7 +118,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                 .field("explain", "fals")
             .endObject());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest, new ParseFieldMatcher(Settings.EMPTY)));
+            () -> RestAnalyzeAction.buildFromContent(invalidExplain, analyzeRequest));
         assertThat(e.getMessage(), startsWith("explain must be either 'true' or 'false'"));
     }
 
@@ -130,7 +131,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                         .array("filters", "lowercase")
                     .endObject());
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                    new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                    new AnalyzeRequest("for test")));
             assertThat(e.getMessage(), startsWith("Unknown parameter [filters]"));
         }
 
@@ -142,7 +143,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                         .array("token_filters", "lowercase")
                     .endObject());
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                    new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                    new AnalyzeRequest("for test")));
             assertThat(e.getMessage(), startsWith("Unknown parameter [token_filters]"));
         }
 
@@ -154,7 +155,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                         .array("char_filters", "lowercase")
                     .endObject());
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                    new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                    new AnalyzeRequest("for test")));
             assertThat(e.getMessage(), startsWith("Unknown parameter [char_filters]"));
         }
 
@@ -166,7 +167,7 @@ public class RestAnalyzeActionTests extends ESTestCase {
                         .array("token_filter", "lowercase")
                     .endObject());
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestAnalyzeAction.buildFromContent(parser,
-                    new AnalyzeRequest("for test"), new ParseFieldMatcher(Settings.EMPTY)));
+                    new AnalyzeRequest("for test")));
             assertThat(e.getMessage(), startsWith("Unknown parameter [token_filter]"));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
index bf7be8f09d2..0aa6e497836 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsActionTests.java
@@ -41,14 +41,14 @@ public class RestIndicesStatsActionTests extends ESTestCase {
     @Override
     public void setUp() throws Exception {
         super.setUp();
-        action = new RestIndicesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null));
+        action = new RestIndicesStatsAction(Settings.EMPTY, new RestController(Settings.EMPTY, Collections.emptySet(), null, null, null));
     }
 
     public void testUnrecognizedMetric() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         final String metric = randomAsciiOfLength(64);
         params.put("metric", metric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -58,7 +58,7 @@ public class RestIndicesStatsActionTests extends ESTestCase {
     public void testUnrecognizedMetricDidYouMean() throws IOException {
         final HashMap<String, String> params = new HashMap<>();
         params.put("metric", "request_cache,fieldata,unrecognized");
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
@@ -73,7 +73,7 @@ public class RestIndicesStatsActionTests extends ESTestCase {
         final HashMap<String, String> params = new HashMap<>();
         final String metric = randomSubsetOf(1, RestIndicesStatsAction.METRICS.keySet()).get(0);
         params.put("metric", "_all," + metric);
-        final RestRequest request = new FakeRestRequest.Builder().withPath("/_stats").withParams(params).build();
+        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withPath("/_stats").withParams(params).build();
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> action.prepareRequest(request, mock(NodeClient.class)));
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
index 2a22541bbc7..ae66664b456 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -74,7 +74,7 @@ public class RestIndicesActionTests extends ESTestCase {
 
     public void testBuildTable() {
         final Settings settings = Settings.EMPTY;
-        final RestController restController = new RestController(settings, Collections.emptySet(), null);
+        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null);
         final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings));
 
         // build a (semi-)random table
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index fa93e4a80d3..232d5ec3d32 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -50,8 +50,8 @@ public class RestRecoveryActionTests extends ESTestCase {
 
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
-        final RestController restController = new RestController(settings, Collections.emptySet(), null);
-        final RestRecoveryAction action = new RestRecoveryAction(settings, restController, restController);
+        final RestController restController = new RestController(settings, Collections.emptySet(), null, null, null);
+        final RestRecoveryAction action = new RestRecoveryAction(settings, restController);
         final int totalShards = randomIntBetween(1, 32);
         final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2));
         final int failedShards = totalShards - successfulShards;
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
index ba3ec55ab2a..4f7bc1bedd9 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestTableTests.java
@@ -104,42 +104,42 @@ public class RestTableTests extends ESTestCase {
     }
 
     public void testThatWeUseTheAcceptHeaderJson() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_JSON),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_JSON)),
                 APPLICATION_JSON,
                 JSON_TABLE_BODY);
     }
 
     public void testThatWeUseTheAcceptHeaderYaml() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, APPLICATION_YAML),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_YAML)),
                 APPLICATION_YAML,
                 YAML_TABLE_BODY);
     }
 
     public void testThatWeUseTheAcceptHeaderSmile() throws Exception {
-        assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_SMILE),
+        assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_SMILE)),
                 APPLICATION_SMILE);
     }
 
     public void testThatWeUseTheAcceptHeaderCbor() throws Exception {
-        assertResponseContentType(Collections.singletonMap(ACCEPT, APPLICATION_CBOR),
+        assertResponseContentType(Collections.singletonMap(ACCEPT, Collections.singletonList(APPLICATION_CBOR)),
                 APPLICATION_CBOR);
     }
 
     public void testThatWeUseTheAcceptHeaderText() throws Exception {
-        assertResponse(Collections.singletonMap(ACCEPT, TEXT_PLAIN),
+        assertResponse(Collections.singletonMap(ACCEPT, Collections.singletonList(TEXT_PLAIN)),
                 TEXT_PLAIN,
                 TEXT_TABLE_BODY);
     }
 
     public void testIgnoreContentType() throws Exception {
-        assertResponse(Collections.singletonMap(CONTENT_TYPE, APPLICATION_JSON),
+        assertResponse(Collections.singletonMap(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)),
                 TEXT_PLAIN,
                 TEXT_TABLE_BODY);
     }
 
     public void testThatDisplayHeadersWithoutTimestamp() throws Exception {
         restRequest.params().put("h", "timestamp,epoch,bulk*");
-        restRequest.params().put("ts", "0");
+        restRequest.params().put("ts", "false");
         List<RestTable.DisplayHeader> headers = buildDisplayHeaders(table, restRequest);
 
         List<String> headerNames = getHeaderNames(headers);
@@ -252,8 +252,8 @@ public class RestTableTests extends ESTestCase {
         assertEquals(Arrays.asList(1,0,2), rowOrder);
     }
 
-    private RestResponse assertResponseContentType(Map<String, String> headers, String mediaType) throws Exception {
-        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder().withHeaders(headers).build();
+    private RestResponse assertResponseContentType(Map<String, List<String>> headers, String mediaType) throws Exception {
+        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder(xContentRegistry()).withHeaders(headers).build();
         table.startRow();
         table.addCell("foo");
         table.addCell("foo");
@@ -274,7 +274,7 @@ public class RestTableTests extends ESTestCase {
         return response;
     }
 
-    private void assertResponse(Map<String, String> headers, String mediaType, String body) throws Exception {
+    private void assertResponse(Map<String, List<String>> headers, String mediaType, String body) throws Exception {
         RestResponse response = assertResponseContentType(headers, mediaType);
         assertThat(response.content().utf8ToString(), equalTo(body));
     }
diff --git a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
index fdfcc4fcb91..c90ce51e48e 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasResolveRoutingIT.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
@@ -51,7 +52,7 @@ public class AliasResolveRoutingIT extends ESIntegTestCase {
             client().prepareIndex("test-0", "type1", "2").setSource("field1", "quick brown"),
             client().prepareIndex("test-0", "type1", "3").setSource("field1", "quick"));
         refresh("test-*");
-        assertHitCount(client().prepareSearch().setIndices("alias-*").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(matchQuery("_all", "quick")).get(), 3L);
+        assertHitCount(client().prepareSearch().setIndices("alias-*").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(queryStringQuery("quick")).get(), 3L);
     }
 
     public void testResolveIndexRouting() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
index 621ade9e31d..dc8b6b54464 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasA
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -64,7 +65,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         logger.info("--> updating with id [1] and routing through alias");
         client().prepareUpdate("alias0", "type1", "1")
                 .setUpsert(XContentFactory.jsonBuilder().startObject().field("field", 1).endObject())
-                .setDoc("field", "value2")
+                .setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2")
                 .execute().actionGet();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("alias0", "type1", "1").execute().actionGet().isExists(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java
new file mode 100644
index 00000000000..14634116f04
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.routing;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.mockito.internal.util.collections.Sets;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class PartitionedRoutingIT extends ESIntegTestCase {
+
+    public void testVariousPartitionSizes() throws Exception {
+        for (int shards = 1; shards <= 4; shards++) {
+            for (int partitionSize = 1; partitionSize < shards; partitionSize++) {
+                String index = "index_" + shards + "_" + partitionSize;
+
+                client().admin().indices().prepareCreate(index)
+                    .setSettings(Settings.builder()
+                        .put("index.number_of_shards", shards)
+                        .put("index.routing_partition_size", partitionSize))
+                    .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+                    .execute().actionGet();
+                ensureGreen();
+
+                Map<String, Set<String>> routingToDocumentIds = generateRoutedDocumentIds(index);
+
+                verifyGets(index, routingToDocumentIds);
+                verifyBroadSearches(index, routingToDocumentIds, shards);
+                verifyRoutedSearches(index, routingToDocumentIds, Sets.newSet(partitionSize));
+            }
+        }
+    }
+
+    public void testShrinking() throws Exception {
+        // creates random routing groups and repeatedly halves the index until it is down to 1 shard
+        // verifying that the count is correct for each shrunken index
+        final int partitionSize = 3;
+        final int originalShards = 8;
+        int currentShards = originalShards;
+        String index = "index_" + currentShards;
+
+        client().admin().indices().prepareCreate(index)
+            .setSettings(Settings.builder()
+                .put("index.number_of_shards", currentShards)
+                .put("index.routing_partition_size", partitionSize))
+            .addMapping("type", "{\"type\":{\"_routing\":{\"required\":true}}}")
+            .execute().actionGet();
+        ensureGreen();
+
+        Map<String, Set<String>> routingToDocumentIds = generateRoutedDocumentIds(index);
+
+        while (true) {
+            int factor = originalShards / currentShards;
+
+            verifyGets(index, routingToDocumentIds);
+            verifyBroadSearches(index, routingToDocumentIds, currentShards);
+
+            // we need the floor and ceiling of the routing_partition_size / factor since the partition size of the shrunken
+            // index will be one of those, depending on the routing value
+            verifyRoutedSearches(index, routingToDocumentIds,
+                Math.floorDiv(partitionSize, factor) == 0 ?
+                    Sets.newSet(1, 2) :
+                    Sets.newSet(Math.floorDiv(partitionSize, factor), -Math.floorDiv(-partitionSize, factor)));
+
+            client().admin().indices().prepareUpdateSettings(index)
+                .setSettings(Settings.builder()
+                    .put("index.routing.allocation.require._name", client().admin().cluster().prepareState().get().getState().nodes()
+                        .getDataNodes().values().toArray(DiscoveryNode.class)[0].getName())
+                    .put("index.blocks.write", true)).get();
+            ensureGreen();
+
+            currentShards = Math.floorDiv(currentShards, 2);
+
+            if (currentShards == 0) {
+                break;
+            }
+
+            String previousIndex = index;
+            index = "index_" + currentShards;
+
+            logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]");
+            client().admin().indices().prepareShrinkIndex(previousIndex, index)
+                .setSettings(Settings.builder()
+                    .put("index.number_of_shards", currentShards)
+                    .build()).get();
+            ensureGreen();
+        }
+    }
+
+    private void verifyRoutedSearches(String index, Map<String, Set<String>> routingToDocumentIds, Set<Integer> expectedShards) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+            int expectedDocuments = routingEntry.getValue().size();
+
+            SearchResponse response  = client().prepareSearch()
+                .setQuery(QueryBuilders.termQuery("_routing", routing))
+                .setRouting(routing)
+                .setIndices(index)
+                .setSize(100)
+                .execute().actionGet();
+
+            logger.info("--> routed search on index [" + index + "] visited [" + response.getTotalShards()
+                + "] shards for routing [" + routing + "] and got hits [" + response.getHits().totalHits() + "]");
+
+            assertTrue(response.getTotalShards() + " was not in " + expectedShards + " for " + index,
+                    expectedShards.contains(response.getTotalShards()));
+            assertEquals(expectedDocuments, response.getHits().totalHits());
+
+            Set<String> found = new HashSet<>();
+            response.getHits().forEach(h -> found.add(h.getId()));
+
+            assertEquals(routingEntry.getValue(), found);
+        }
+    }
+
+    private void verifyBroadSearches(String index, Map<String, Set<String>> routingToDocumentIds, int expectedShards) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+            int expectedDocuments = routingEntry.getValue().size();
+
+            SearchResponse response  = client().prepareSearch()
+                .setQuery(QueryBuilders.termQuery("_routing", routing))
+                .setIndices(index)
+                .setSize(100)
+                .execute().actionGet();
+
+            assertEquals(expectedShards, response.getTotalShards());
+            assertEquals(expectedDocuments, response.getHits().totalHits());
+
+            Set<String> found = new HashSet<>();
+            response.getHits().forEach(h -> found.add(h.getId()));
+
+            assertEquals(routingEntry.getValue(), found);
+        }
+    }
+
+    private void verifyGets(String index, Map<String, Set<String>> routingToDocumentIds) {
+        for (Map.Entry<String, Set<String>> routingEntry : routingToDocumentIds.entrySet()) {
+            String routing = routingEntry.getKey();
+
+            for (String id : routingEntry.getValue()) {
+                assertTrue(client().prepareGet(index, "type", id).setRouting(routing).execute().actionGet().isExists());
+            }
+        }
+    }
+
+    private Map<String, Set<String>> generateRoutedDocumentIds(String index) {
+        Map<String, Set<String>> routingToDocumentIds = new HashMap<>();
+        int numRoutingValues = randomIntBetween(5, 15);
+
+        for (int i = 0; i < numRoutingValues; i++) {
+            String routingValue = String.valueOf(i);
+            int numDocuments = randomIntBetween(10, 100);
+            routingToDocumentIds.put(String.valueOf(routingValue), new HashSet<>());
+
+            for (int k = 0; k < numDocuments; k++) {
+                String id = routingValue + "_" + String.valueOf(k);
+                routingToDocumentIds.get(routingValue).add(id);
+
+                client().prepareIndex(index, "type", id)
+                    .setRouting(routingValue)
+                    .setSource("foo", "bar")
+                    .get();
+            }
+        }
+
+        client().admin().indices().prepareRefresh(index).get();
+
+        return routingToDocumentIds;
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
index 2490134db4e..b027a13e867 100644
--- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
@@ -207,13 +207,13 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         }
 
         try {
-            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field", "value2").execute().actionGet();
+            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").execute().actionGet();
             fail("update with missing routing when routing is required should fail");
         } catch(ElasticsearchException e) {
             assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class));
         }
 
-        client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc("field", "value2").execute().actionGet();
+        client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0").setDoc(Requests.INDEX_CONTENT_TYPE, "field", "value2").get();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         for (int i = 0; i < 5; i++) {
@@ -254,7 +254,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
 
         {
             BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1")
-                .source("field", "value")).execute().actionGet();
+                .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
             assertThat(bulkResponse.getItems().length, equalTo(1));
             assertThat(bulkResponse.hasFailures(), equalTo(true));
 
@@ -269,12 +269,13 @@ public class SimpleRoutingIT extends ESIntegTestCase {
 
         {
             BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").routing("0")
-                .source("field", "value")).execute().actionGet();
+                .source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
             assertThat(bulkResponse.hasFailures(), equalTo(false));
         }
 
         {
-            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2"))
+            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1")
+                .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2"))
                 .execute().actionGet();
             assertThat(bulkResponse.getItems().length, equalTo(1));
             assertThat(bulkResponse.hasFailures(), equalTo(true));
@@ -289,7 +290,8 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         }
 
         {
-            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc("field", "value2")
+            BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1")
+                .doc(Requests.INDEX_CONTENT_TYPE, "field", "value2")
                 .routing("0")).execute().actionGet();
             assertThat(bulkResponse.hasFailures(), equalTo(false));
         }
@@ -369,12 +371,12 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         }
 
         UpdateResponse updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setRouting("0")
-                .setDoc("field1", "value1").get();
+                .setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get();
         assertThat(updateResponse.getId(), equalTo("1"));
         assertThat(updateResponse.getVersion(), equalTo(2L));
 
         try {
-            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc("field1", "value1").get();
+            client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "field1", "value1").get();
             fail();
         } catch (RoutingMissingException e) {
             assertThat(e.getMessage(), equalTo("routing is required for [test]/[type1]/[1]"));
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index f5ae13cf3a0..92e659ac755 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -55,7 +55,7 @@ public class FileScriptTests extends ESTestCase {
             .put("script.engine." + MockScriptEngine.NAME + ".file.aggs", "false").build();
         ScriptService scriptService = makeScriptService(settings);
         Script script = new Script(ScriptType.FILE, MockScriptEngine.NAME, "script1", Collections.emptyMap());
-        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH);
         assertNotNull(compiledScript);
         MockCompiledScript executable = (MockCompiledScript) compiledScript.compiled();
         assertEquals("script1.mockscript", executable.getName());
@@ -72,7 +72,7 @@ public class FileScriptTests extends ESTestCase {
         Script script = new Script(ScriptType.FILE, MockScriptEngine.NAME, "script1", Collections.emptyMap());
         for (ScriptContext context : ScriptContext.Standard.values()) {
             try {
-                scriptService.compile(script, context, Collections.emptyMap());
+                scriptService.compile(script, context);
                 fail(context.getKey() + " script should have been rejected");
             } catch(Exception e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [file], operation [" + context.getKey() + "] and lang [" + MockScriptEngine.NAME + "] are disabled"));
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 5423103f92d..bf5c7e0daa7 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -80,7 +80,7 @@ public class NativeScriptTests extends ESTestCase {
 
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
             assertThat(scriptService.compile(new Script(ScriptType.INLINE, NativeScriptEngineService.NAME, "my", Collections.emptyMap()),
-                scriptContext, Collections.emptyMap()), notNullValue());
+                scriptContext), notNullValue());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 9578e0c6281..e25335e5d68 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -19,6 +19,10 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
@@ -37,7 +41,7 @@ public class ScriptContextTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
             // no file watching, so we don't need a ResourceWatcherService
-            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "off")
+            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), "false")
             .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false")
             .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false")
             .build();
@@ -50,8 +54,16 @@ public class ScriptContextTests extends ESTestCase {
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        ScriptService scriptService = new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
 
-        return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        ClusterState empty = ClusterState.builder(new ClusterName("_name")).build();
+        ScriptMetaData smd = empty.metaData().custom(ScriptMetaData.TYPE);
+        smd = ScriptMetaData.putStoredScript(smd, "1", new StoredScriptSource(MockScriptEngine.NAME, "1", Collections.emptyMap()));
+        MetaData.Builder mdb = MetaData.builder(empty.getMetaData()).putCustom(ScriptMetaData.TYPE, smd);
+        ClusterState stored = ClusterState.builder(empty).metaData(mdb).build();
+        scriptService.clusterChanged(new ClusterChangedEvent("test", stored, empty));
+
+        return scriptService;
     }
 
     public void testCustomGlobalScriptContextSettings() throws Exception {
@@ -59,7 +71,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptType scriptType : ScriptType.values()) {
             try {
                 Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), Collections.emptyMap());
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"));
                 fail("script compilation should have been rejected");
             } catch (IllegalStateException e) {
                 assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
@@ -71,16 +83,16 @@ public class ScriptContextTests extends ESTestCase {
         ScriptService scriptService = makeScriptService();
         Script script = new Script(ScriptType.INLINE, MockScriptEngine.NAME, "1", Collections.emptyMap());
         try {
-            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), Collections.emptyMap());
+            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"));
             fail("script compilation should have been rejected");
         } catch (IllegalStateException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
         }
 
         // still works for other script contexts
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS, Collections.emptyMap()));
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap()));
-        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), Collections.emptyMap()));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH));
+        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op")));
     }
 
     public void testUnknownPluginScriptContext() throws Exception {
@@ -88,7 +100,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptType scriptType : ScriptType.values()) {
             try {
                 Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"), Collections.emptyMap());
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"));
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported"));
@@ -107,7 +119,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptType scriptType : ScriptType.values()) {
             try {
                 Script script = new Script(scriptType, MockScriptEngine.NAME, "1", Collections.emptyMap());
-                scriptService.compile(script, context, Collections.emptyMap());
+                scriptService.compile(script, context);
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported"));
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
index 47b6d87d7db..5a02ea1e96a 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
@@ -20,124 +20,83 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
-import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.AbstractSerializingTestCase;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.Collections;
+import java.io.UncheckedIOException;
 
-
-public class ScriptMetaDataTests extends ESTestCase {
+public class ScriptMetaDataTests extends AbstractSerializingTestCase<ScriptMetaData> {
 
     public void testGetScript() throws Exception {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
 
         XContentBuilder sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().startObject("template").field("field", "value").endObject().endObject();
-        builder.storeScript("lang", "template", sourceBuilder.bytes());
+        builder.storeScript("template", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("template", "value").endObject();
-        builder.storeScript("lang", "template_field", sourceBuilder.bytes());
+        builder.storeScript("template_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().startObject("script").field("field", "value").endObject().endObject();
-        builder.storeScript("lang", "script", sourceBuilder.bytes());
+        builder.storeScript("script", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("script", "value").endObject();
-        builder.storeScript("lang", "script_field", sourceBuilder.bytes());
+        builder.storeScript("script_field", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         sourceBuilder = XContentFactory.jsonBuilder();
         sourceBuilder.startObject().field("field", "value").endObject();
-        builder.storeScript("lang", "any", sourceBuilder.bytes());
+        builder.storeScript("any", StoredScriptSource.parse("lang", sourceBuilder.bytes(), sourceBuilder.contentType()));
 
         ScriptMetaData scriptMetaData = builder.build();
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "template"));
-        assertEquals("value", scriptMetaData.getScript("lang", "template_field"));
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "script"));
-        assertEquals("value", scriptMetaData.getScript("lang", "script_field"));
-        assertEquals("{\"field\":\"value\"}", scriptMetaData.getScript("lang", "any"));
-    }
-
-    public void testToAndFromXContent() throws IOException {
-        XContentType contentType = randomFrom(XContentType.values());
-        XContentBuilder xContentBuilder = XContentBuilder.builder(contentType.xContent());
-        ScriptMetaData expected = randomScriptMetaData(contentType);
-
-        xContentBuilder.startObject();
-        expected.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.emptyMap()));
-        xContentBuilder.endObject();
-        xContentBuilder = shuffleXContent(xContentBuilder);
-
-        XContentParser parser = createParser(xContentBuilder);
-        parser.nextToken();
-        ScriptMetaData result = ScriptMetaData.PROTO.fromXContent(parser);
-        assertEquals(expected, result);
-        assertEquals(expected.hashCode(), result.hashCode());
-    }
-
-    public void testReadFromWriteTo() throws IOException {
-        ScriptMetaData expected = randomScriptMetaData(randomFrom(XContentType.values()));
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        expected.writeTo(new OutputStreamStreamOutput(out));
-
-        ScriptMetaData result = ScriptMetaData.PROTO.readFrom(new InputStreamStreamInput(new ByteArrayInputStream(out.toByteArray())));
-        assertEquals(expected, result);
-        assertEquals(expected.hashCode(), result.hashCode());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("template", "lang").getCode());
+        assertEquals("value", scriptMetaData.getStoredScript("template_field", "lang").getCode());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("script", "lang").getCode());
+        assertEquals("value", scriptMetaData.getStoredScript("script_field", "lang").getCode());
+        assertEquals("{\"field\":\"value\"}", scriptMetaData.getStoredScript("any", "lang").getCode());
     }
 
     public void testDiff() throws Exception {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
-        builder.storeScript("lang", "1", new BytesArray("{\"foo\":\"abc\"}"));
-        builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"def\"}"));
-        builder.storeScript("lang", "3", new BytesArray("{\"foo\":\"ghi\"}"));
+        builder.storeScript("1", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"abc\"}"), XContentType.JSON));
+        builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"def\"}"), XContentType.JSON));
+        builder.storeScript("3", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"ghi\"}"), XContentType.JSON));
         ScriptMetaData scriptMetaData1 = builder.build();
 
         builder = new ScriptMetaData.Builder(scriptMetaData1);
-        builder.storeScript("lang", "2", new BytesArray("{\"foo\":\"changed\"}"));
-        builder.deleteScript("lang", "3");
-        builder.storeScript("lang", "4", new BytesArray("{\"foo\":\"jkl\"}"));
+        builder.storeScript("2", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"changed\"}"), XContentType.JSON));
+        builder.deleteScript("3", "lang");
+        builder.storeScript("4", StoredScriptSource.parse("lang", new BytesArray("{\"foo\":\"jkl\"}"), XContentType.JSON));
         ScriptMetaData scriptMetaData2 = builder.build();
 
         ScriptMetaData.ScriptMetadataDiff diff = (ScriptMetaData.ScriptMetadataDiff) scriptMetaData2.diff(scriptMetaData1);
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().size());
-        assertEquals("lang#3", ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().get(0));
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getDiffs().size());
-        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getDiffs().get("lang#2"));
-        assertEquals(1, ((DiffableUtils.MapDiff) diff.pipelines).getUpserts().size());
-        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getUpserts().get("lang#4"));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().size());
+        assertEquals("3", ((DiffableUtils.MapDiff) diff.pipelines).getDeletes().get(0));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getDiffs().size());
+        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getDiffs().get("2"));
+        assertEquals(2, ((DiffableUtils.MapDiff) diff.pipelines).getUpserts().size());
+        assertNotNull(((DiffableUtils.MapDiff) diff.pipelines).getUpserts().get("4"));
 
         ScriptMetaData result = (ScriptMetaData) diff.apply(scriptMetaData1);
-        assertEquals(new BytesArray("{\"foo\":\"abc\"}"), result.getScriptAsBytes("lang", "1"));
-        assertEquals(new BytesArray("{\"foo\":\"changed\"}"), result.getScriptAsBytes("lang", "2"));
-        assertEquals(new BytesArray("{\"foo\":\"jkl\"}"), result.getScriptAsBytes("lang", "4"));
+        assertEquals("{\"foo\":\"abc\"}", result.getStoredScript("1", "lang").getCode());
+        assertEquals("{\"foo\":\"changed\"}", result.getStoredScript("2", "lang").getCode());
+        assertEquals("{\"foo\":\"jkl\"}", result.getStoredScript("4", "lang").getCode());
     }
 
     public void testBuilder() {
         ScriptMetaData.Builder builder = new ScriptMetaData.Builder(null);
-        builder.storeScript("_lang", "_id", new BytesArray("{\"script\":\"1 + 1\"}"));
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-            () -> builder.storeScript("_lang#", "_id", new BytesArray("{\"foo\": \"bar\"}")));
-        assertEquals("stored script language can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.storeScript("_lang", "_id#", new BytesArray("{\"foo\": \"bar\"}")));
-        assertEquals("stored script id can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.deleteScript("_lang#", "_id"));
-        assertEquals("stored script language can't contain: '#'", e.getMessage());
-        e = expectThrows(IllegalArgumentException.class, () -> builder.deleteScript("_lang", "_id#"));
-        assertEquals("stored script id can't contain: '#'", e.getMessage());
+        builder.storeScript("_id", StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"1 + 1\"}"), XContentType.JSON));
 
         ScriptMetaData result = builder.build();
-        assertEquals("1 + 1", result.getScript("_lang", "_id"));
+        assertEquals("1 + 1", result.getStoredScript("_id", "_lang").getCode());
     }
 
     private ScriptMetaData randomScriptMetaData(XContentType sourceContentType) throws IOException {
@@ -146,10 +105,33 @@ public class ScriptMetaDataTests extends ESTestCase {
         for (int i = 0; i < numScripts; i++) {
             String lang = randomAsciiOfLength(4);
             XContentBuilder sourceBuilder = XContentBuilder.builder(sourceContentType.xContent());
-            sourceBuilder.startObject().field(randomAsciiOfLength(4), randomAsciiOfLength(4)).endObject();
-            builder.storeScript(lang, randomAsciiOfLength(i + 1), sourceBuilder.bytes());
+            sourceBuilder.startObject().field("script", randomAsciiOfLength(4)).endObject();
+            builder.storeScript(randomAsciiOfLength(i + 1),
+                StoredScriptSource.parse(lang, sourceBuilder.bytes(), sourceBuilder.contentType()));
         }
         return builder.build();
     }
 
+    @Override
+    protected ScriptMetaData createTestInstance() {
+        try {
+            return randomScriptMetaData(randomFrom(XContentType.values()));
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+
+    @Override
+    protected Writeable.Reader<ScriptMetaData> instanceReader() {
+        return ScriptMetaData::new;
+    }
+
+    @Override
+    protected ScriptMetaData doParseInstance(XContentParser parser) {
+        try {
+            return ScriptMetaData.fromXContent(parser);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 9d0c5b1867f..3482dc8bb34 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -19,9 +19,7 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -32,6 +30,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
@@ -121,9 +120,9 @@ public class ScriptServiceTests extends ESTestCase {
         // TODO:
         scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            String getScriptFromClusterState(String scriptLang, String id) {
+            StoredScriptSource getScriptFromClusterState(String id, String lang) {
                 //mock the script that gets retrieved from an index
-                return "100";
+                return new StoredScriptSource(lang, "100", Collections.emptyMap());
             }
         };
     }
@@ -170,7 +169,7 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService.notifyNow();
 
         CompiledScript compiledScript = scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap());
+                ScriptContext.Standard.SEARCH);
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file"));
 
         Files.delete(testFileNoExt);
@@ -178,11 +177,10 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService.notifyNow();
 
         try {
-            scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()), ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap());
+            scriptService.compile(new Script(ScriptType.FILE, "test", "test_script", Collections.emptyMap()), ScriptContext.Standard.SEARCH);
             fail("the script test_script should no longer exist");
         } catch (IllegalArgumentException ex) {
-            assertThat(ex.getMessage(), containsString("Unable to find on disk file script [test_script] using lang [test]"));
+            assertThat(ex.getMessage(), containsString("unable to find file script [test_script] using lang [test]"));
         }
     }
 
@@ -197,7 +195,7 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService.notifyNow();
 
         CompiledScript compiledScript = scriptService.compile(new Script(ScriptType.FILE, "test", "file_script", Collections.emptyMap()),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap());
+                ScriptContext.Standard.SEARCH);
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file_script"));
 
         Files.delete(testHiddenFile);
@@ -208,9 +206,9 @@ public class ScriptServiceTests extends ESTestCase {
     public void testInlineScriptCompiledOnceCache() throws IOException {
         buildScriptService(Settings.EMPTY);
         CompiledScript compiledScript1 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()),
-                randomFrom(scriptContexts), Collections.emptyMap());
+                randomFrom(scriptContexts));
         CompiledScript compiledScript2 = scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()),
-                randomFrom(scriptContexts), Collections.emptyMap());
+                randomFrom(scriptContexts));
         assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
     }
 
@@ -333,7 +331,7 @@ public class ScriptServiceTests extends ESTestCase {
         String type = scriptEngineService.getType();
         try {
             scriptService.compile(new Script(randomFrom(ScriptType.values()), type, "test", Collections.emptyMap()),
-                new ScriptContext.Plugin(pluginName, unknownContext), Collections.emptyMap());
+                new ScriptContext.Plugin(pluginName, unknownContext));
             fail("script compilation should have been rejected");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported"));
@@ -342,8 +340,7 @@ public class ScriptServiceTests extends ESTestCase {
 
     public void testCompileCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
@@ -364,8 +361,7 @@ public class ScriptServiceTests extends ESTestCase {
         int numberOfCompilations = randomIntBetween(1, 1024);
         for (int i = 0; i < numberOfCompilations; i++) {
             scriptService
-                    .compile(new Script(ScriptType.INLINE, "test", i + " + " + i, Collections.emptyMap()), randomFrom(scriptContexts),
-                            Collections.emptyMap());
+                    .compile(new Script(ScriptType.INLINE, "test", i + " + " + i, Collections.emptyMap()), randomFrom(scriptContexts));
         }
         assertEquals(numberOfCompilations, scriptService.stats().getCompilations());
     }
@@ -383,15 +379,13 @@ public class ScriptServiceTests extends ESTestCase {
     public void testFileScriptCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
         createFileScripts("test");
-        scriptService.compile(new Script(ScriptType.FILE, "test", "file_script", Collections.emptyMap()), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.FILE, "test", "file_script", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
     public void testIndexedScriptCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        scriptService.compile(new Script(ScriptType.STORED, "test", "script", Collections.emptyMap()), randomFrom(scriptContexts),
-                Collections.emptyMap());
+        scriptService.compile(new Script(ScriptType.STORED, "test", "script", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
@@ -411,8 +405,7 @@ public class ScriptServiceTests extends ESTestCase {
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
         CompiledScript script = scriptService.compile(
-            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 + 1", Collections.emptyMap()),
-                randomFrom(scriptContexts), Collections.emptyMap());
+            new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "1 + 1", Collections.emptyMap()), randomFrom(scriptContexts));
         assertEquals(script.lang(), Script.DEFAULT_SCRIPT_LANG);
     }
 
@@ -421,68 +414,42 @@ public class ScriptServiceTests extends ESTestCase {
                     .field("script", "abc")
                 .endObject().bytes();
 
-        ClusterState empty = ClusterState.builder(new ClusterName("_name")).build();
-        PutStoredScriptRequest request = new PutStoredScriptRequest("_lang", "_id")
-                .script(script);
-        ClusterState result = ScriptService.innerStoreScript(empty, "_lang", request);
-        ScriptMetaData scriptMetaData = result.getMetaData().custom(ScriptMetaData.TYPE);
+        ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id",
+            StoredScriptSource.parse("_lang", script, XContentType.JSON));
         assertNotNull(scriptMetaData);
-        assertEquals("abc", scriptMetaData.getScript("_lang", "_id"));
+        assertEquals("abc", scriptMetaData.getStoredScript("_id", "_lang").getCode());
     }
 
     public void testDeleteScript() throws Exception {
-        ClusterState cs = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder()
-                        .putCustom(ScriptMetaData.TYPE,
-                                new ScriptMetaData.Builder(null).storeScript("_lang", "_id",
-                                    new BytesArray("{\"script\":\"abc\"}")).build()))
-                .build();
-
-        DeleteStoredScriptRequest request = new DeleteStoredScriptRequest("_lang", "_id");
-        ClusterState result = ScriptService.innerDeleteScript(cs, "_lang", request);
-        ScriptMetaData scriptMetaData = result.getMetaData().custom(ScriptMetaData.TYPE);
+        ScriptMetaData scriptMetaData = ScriptMetaData.putStoredScript(null, "_id",
+            StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON));
+        scriptMetaData = ScriptMetaData.deleteStoredScript(scriptMetaData, "_id", "_lang");
         assertNotNull(scriptMetaData);
-        assertNull(scriptMetaData.getScript("_lang", "_id"));
-        assertNull(scriptMetaData.getScriptAsBytes("_lang", "_id"));
+        assertNull(scriptMetaData.getStoredScript("_id", "_lang"));
 
+        ScriptMetaData errorMetaData = scriptMetaData;
         ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, () -> {
-            ScriptService.innerDeleteScript(cs, "_lang", new DeleteStoredScriptRequest("_lang", "_non_existing_id"));
+            ScriptMetaData.deleteStoredScript(errorMetaData, "_id", "_lang");
         });
-        assertEquals("Stored script with id [_non_existing_id] for language [_lang] does not exist", e.getMessage());
+        assertEquals("stored script [_id] using lang [_lang] does not exist and cannot be deleted", e.getMessage());
     }
 
     public void testGetStoredScript() throws Exception {
         buildScriptService(Settings.EMPTY);
         ClusterState cs = ClusterState.builder(new ClusterName("_name"))
-                .metaData(MetaData.builder()
-                        .putCustom(ScriptMetaData.TYPE,
-                                new ScriptMetaData.Builder(null).storeScript("_lang", "_id",
-                                        new BytesArray("{\"script\":\"abc\"}")).build()))
-                .build();
+            .metaData(MetaData.builder()
+                .putCustom(ScriptMetaData.TYPE,
+                    new ScriptMetaData.Builder(null).storeScript("_id",
+                        StoredScriptSource.parse("_lang", new BytesArray("{\"script\":\"abc\"}"), XContentType.JSON)).build()))
+            .build();
 
-        assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id")));
-        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id2")));
+        assertEquals("abc", scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")).getCode());
+        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id2", "_lang")));
 
         cs = ClusterState.builder(new ClusterName("_name")).build();
-        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_lang", "_id")));
+        assertNull(scriptService.getStoredScript(cs, new GetStoredScriptRequest("_id", "_lang")));
     }
 
-    public void testValidateScriptSize() throws Exception {
-        int maxSize = 0xFFFF;
-        buildScriptService(Settings.EMPTY);
-        // allowed
-        scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 13) + "\"}"));
-
-        // disallowed
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> {
-                    scriptService.validateStoredScript("_id", "test", new BytesArray("{\"script\":\"" + randomAsciiOfLength(maxSize - 12) + "\"}"));
-                });
-        assertThat(e.getMessage(), equalTo(
-                "Limit of script size in bytes [" + maxSize+ "] has been exceeded for script [_id] with size [" + (maxSize + 1) + "]"));
-    }
-
-
     private void createFileScripts(String... langs) throws IOException {
         for (String lang : langs) {
             Path scriptPath = scriptsFilePath.resolve("file_script." + lang);
@@ -493,7 +460,7 @@ public class ScriptServiceTests extends ESTestCase {
 
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
-            scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext, Collections.emptyMap());
+            scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext);
             fail("compile should have been rejected for lang [" + lang + "], script_type [" + scriptType + "], scripted_op [" + scriptContext + "]");
         } catch(IllegalStateException e) {
             //all good
@@ -502,7 +469,7 @@ public class ScriptServiceTests extends ESTestCase {
 
     private void assertCompileAccepted(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         assertThat(
-                scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext, Collections.emptyMap()),
+                scriptService.compile(new Script(scriptType, lang, script, Collections.emptyMap()), scriptContext),
                 notNullValue()
         );
     }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptTests.java b/core/src/test/java/org/elasticsearch/script/ScriptTests.java
index 4ad02460d4c..fc841bd1648 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -45,7 +44,7 @@ public class ScriptTests extends ESTestCase {
         try (XContentBuilder builder = XContentBuilder.builder(xContent)) {
             expectedScript.toXContent(builder, ToXContent.EMPTY_PARAMS);
             try (XContentParser parser = createParser(builder)) {
-                Script actualScript = Script.parse(parser, ParseFieldMatcher.STRICT);
+                Script actualScript = Script.parse(parser);
                 assertThat(actualScript, equalTo(expectedScript));
             }
         }
@@ -79,10 +78,9 @@ public class ScriptTests extends ESTestCase {
         }
         return new Script(
             scriptType,
-            randomFrom("_lang1", "_lang2", "_lang3"),
+            scriptType == ScriptType.STORED ? null : randomFrom("_lang1", "_lang2", "_lang3"),
             script,
-            scriptType == ScriptType.INLINE ?
-                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, xContent.type().mediaType()) : Collections.emptyMap(),
+            scriptType == ScriptType.INLINE ? Collections.singletonMap(Script.CONTENT_TYPE_OPTION, xContent.type().mediaType()) : null,
             params
         );
     }
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java
new file mode 100644
index 00000000000..5f000f28cbc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptTests.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractSerializingTestCase;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class StoredScriptTests extends AbstractSerializingTestCase<StoredScriptSource> {
+
+    public void testBasicAddDelete() {
+        StoredScriptSource source = new StoredScriptSource("lang", "code", emptyMap());
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test", source);
+
+        assertThat(smd.getStoredScript("test", null), equalTo(source));
+        assertThat(smd.getStoredScript("test", "lang"), equalTo(source));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test", null);
+
+        assertThat(smd.getStoredScript("test", null), nullValue());
+        assertThat(smd.getStoredScript("test", "lang"), nullValue());
+    }
+
+    public void testDifferentMultiAddDelete() {
+        StoredScriptSource source0 = new StoredScriptSource("lang0", "code0", emptyMap());
+        StoredScriptSource source1 = new StoredScriptSource("lang0", "code1", emptyMap());
+        StoredScriptSource source2 = new StoredScriptSource("lang1", "code2",
+            singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
+
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test0", source0);
+        smd = ScriptMetaData.putStoredScript(smd, "test1", source1);
+        smd = ScriptMetaData.putStoredScript(smd, "test2", source2);
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test2", "lang1"), equalTo(source2));
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", null);
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test2", "lang1"), equalTo(source2));
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test2", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), equalTo(source1));
+        assertThat(smd.getStoredScript("test1", "lang0"), equalTo(source1));
+        assertThat(smd.getStoredScript("test2", null), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang1"), nullValue());
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test1", "lang0");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test1", null), nullValue());
+        assertThat(smd.getStoredScript("test1", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test2", null), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang1"), nullValue());
+
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test2", "lang0"), nullValue());
+    }
+
+    public void testSameMultiAddDelete() {
+        StoredScriptSource source0 = new StoredScriptSource("lang0", "code0", emptyMap());
+        StoredScriptSource source1 = new StoredScriptSource("lang1", "code1", emptyMap());
+        StoredScriptSource source2 = new StoredScriptSource("lang2", "code1", emptyMap());
+        StoredScriptSource source3 = new StoredScriptSource("lang1", "code2",
+            singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType()));
+
+        ScriptMetaData smd = ScriptMetaData.putStoredScript(null, "test0", source0);
+        smd = ScriptMetaData.putStoredScript(smd, "test0", source1);
+        assertWarnings("stored script [test0] already exists using a different lang [lang0]," +
+            " the new namespace for stored scripts will only use (id) instead of (lang, id)");
+        smd = ScriptMetaData.putStoredScript(smd, "test3", source3);
+        smd = ScriptMetaData.putStoredScript(smd, "test0", source2);
+        assertWarnings("stored script [test0] already exists using a different lang [lang1]," +
+            " the new namespace for stored scripts will only use (id) instead of (lang, id)");
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), equalTo(source1));
+        assertThat(smd.getStoredScript("test0", "lang2"), equalTo(source2));
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), equalTo(source2));
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), equalTo(source2));
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", null);
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), equalTo(source3));
+        assertThat(smd.getStoredScript("test3", "lang1"), equalTo(source3));
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test3", "lang1");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), equalTo(source0));
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test3", "lang1"), nullValue());
+
+        smd = ScriptMetaData.deleteStoredScript(smd, "test0", "lang0");
+
+        assertThat(smd.getStoredScript("test0", null), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang0"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang1"), nullValue());
+        assertThat(smd.getStoredScript("test0", "lang2"), nullValue());
+        assertThat(smd.getStoredScript("test3", null), nullValue());
+        assertThat(smd.getStoredScript("test3", "lang1"), nullValue());
+    }
+
+    public void testInvalidDelete() {
+        ResourceNotFoundException rnfe =
+            expectThrows(ResourceNotFoundException.class, () -> ScriptMetaData.deleteStoredScript(null, "test", "lang"));
+        assertThat(rnfe.getMessage(), equalTo("stored script [test] using lang [lang] does not exist and cannot be deleted"));
+
+        rnfe = expectThrows(ResourceNotFoundException.class, () -> ScriptMetaData.deleteStoredScript(null, "test", null));
+        assertThat(rnfe.getMessage(), equalTo("stored script [test] does not exist and cannot be deleted"));
+    }
+
+    public void testSourceParsing() throws Exception {
+        // simple script value string
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script", "code").endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // simple template value string
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template", "code").endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code",
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template with wrapper template object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template").startObject().field("query", "code").endObject().endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code,
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template with no wrapper object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("query", "code").endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code,
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex template using script as the field name
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("query", "code").endObject().endObject();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse("lang", builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code,
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script with script object
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code").endObject().endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script with script object and empty options
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code")
+                .field("options").startObject().endObject().endObject().endObject();
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", "code", Collections.emptyMap());
+
+            assertThat(parsed, equalTo(source));
+        }
+
+        // complex script with embedded template
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").startObject("code").field("query", "code")
+                .endObject().startObject("options").endObject().endObject().endObject().string();
+            String code;
+
+            try (XContentBuilder cb = XContentFactory.contentBuilder(builder.contentType())) {
+                code = cb.startObject().field("query", "code").endObject().string();
+            }
+
+            StoredScriptSource parsed = StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON);
+            StoredScriptSource source = new StoredScriptSource("lang", code,
+                Collections.singletonMap(Script.CONTENT_TYPE_OPTION, builder.contentType().mediaType()));
+
+            assertThat(parsed, equalTo(source));
+        }
+    }
+
+    public void testSourceParsingErrors() throws Exception {
+        // check for missing lang parameter when parsing a template
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("template", "code").endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("unexpected stored script format"));
+        }
+
+        // check for missing lang parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("code", "code").endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("must specify lang for stored script"));
+        }
+
+        // check for missing code parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("must specify code for stored script"));
+        }
+
+        // check for illegal options parameter when parsing a script
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code")
+                .startObject("options").field("option", "option").endObject().endObject().endObject();
+
+            IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(iae.getMessage(), equalTo("illegal compiler options [{option=option}] specified"));
+        }
+
+        // check for illegal use of content type option
+        try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) {
+            builder.startObject().field("script").startObject().field("lang", "lang").field("code", "code")
+                .startObject("options").field("content_type", "option").endObject().endObject().endObject();
+
+            ParsingException pe = expectThrows(ParsingException.class, () ->
+                StoredScriptSource.parse(null, builder.bytes(), XContentType.JSON));
+            assertThat(pe.getRootCause().getMessage(), equalTo("content_type cannot be user-specified"));
+        }
+    }
+
+    @Override
+    protected StoredScriptSource createTestInstance() {
+        return new StoredScriptSource(
+            randomAsciiOfLength(randomIntBetween(4, 32)),
+            randomAsciiOfLength(randomIntBetween(4, 16383)),
+            Collections.emptyMap());
+    }
+
+    @Override
+    protected Writeable.Reader<StoredScriptSource> instanceReader() {
+        return StoredScriptSource::new;
+    }
+
+    @Override
+    protected StoredScriptSource doParseInstance(XContentParser parser) {
+        try {
+            return StoredScriptSource.fromXContent(parser);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
index 2bc58bb0d22..f5c70edb024 100644
--- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -20,6 +20,7 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -50,37 +51,38 @@ public class StoredScriptsIT extends ESIntegTestCase {
 
     public void testBasics() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(LANG)
+                .setLang(LANG)
                 .setId("foobar")
-                .setSource(new BytesArray("{\"script\":\"1\"}")));
+                .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON));
         String script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
-                .get().getStoredScript();
+                .get().getSource().getCode();
         assertNotNull(script);
         assertEquals("1", script);
 
         assertAcked(client().admin().cluster().prepareDeleteStoredScript()
                 .setId("foobar")
-                .setScriptLang(LANG));
-        script = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
-                .get().getStoredScript();
-        assertNull(script);
+                .setLang(LANG));
+        StoredScriptSource source = client().admin().cluster().prepareGetStoredScript(LANG, "foobar")
+                .get().getSource();
+        assertNull(source);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript()
-                .setScriptLang("lang#")
+                .setLang("lang#")
                 .setId("id#")
-                .setSource(new BytesArray("{}"))
+                .setContent(new BytesArray("{}"), XContentType.JSON)
                 .get());
-        assertEquals("Validation Failed: 1: id can't contain: '#';2: lang can't contain: '#';", e.getMessage());
+        assertEquals("Validation Failed: 1: id cannot contain '#' for stored script;" +
+            "2: lang cannot contain '#' for stored script;", e.getMessage());
     }
 
     public void testMaxScriptSize() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(LANG)
+                .setLang(LANG)
                 .setId("foobar")
-                .setSource(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)))
+                .setContent(new BytesArray(randomAsciiOfLength(SCRIPT_MAX_SIZE_IN_BYTES + 1)), XContentType.JSON)
                 .get()
         );
-        assertEquals("Limit of script size in bytes [64] has been exceeded for script [foobar] with size [65]", e.getMessage());
+        assertEquals("exceeded max allowed stored script size in bytes [64] with size [65] for script [foobar]", e.getMessage());
     }
 
     public static class CustomScriptPlugin extends MockScriptPlugin {
diff --git a/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
index cf290b2b1b6..ba4c794a090 100644
--- a/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java
@@ -20,18 +20,20 @@
 package org.elasticsearch.search;
 
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilderTests;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
 import org.elasticsearch.search.suggest.SuggestBuilderTests;
@@ -52,9 +54,8 @@ import java.util.function.Supplier;
 public abstract class AbstractSearchTestCase extends ESTestCase {
 
     protected NamedWriteableRegistry namedWriteableRegistry;
-    protected SearchRequestParsers searchRequestParsers;
     private TestSearchExtPlugin searchExtPlugin;
-    protected IndicesQueriesRegistry queriesRegistry;
+    private NamedXContentRegistry xContentRegistry;
 
     public void setUp() throws Exception {
         super.setUp();
@@ -65,8 +66,12 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        searchRequestParsers = searchModule.getSearchRequestParsers();
-        queriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 
     protected SearchSourceBuilder createSearchSourceBuilder() {
@@ -86,7 +91,8 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
                 HighlightBuilderTests::randomHighlighterBuilder,
                 SuggestBuilderTests::randomSuggestBuilder,
                 QueryRescoreBuilderTests::randomRescoreBuilder,
-                randomExtBuilders);
+                randomExtBuilders,
+                CollapseBuilderTests::randomCollapseBuilder);
     }
 
     protected SearchRequest createSearchRequest() throws IOException {
@@ -137,7 +143,7 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
         }
     }
 
-    private static class TestSearchExtParser<T extends SearchExtBuilder> implements SearchExtParser<T> {
+    private static class TestSearchExtParser<T extends SearchExtBuilder> implements CheckedFunction<XContentParser, T, IOException> {
         private final Function<String, T> searchExtBuilderFunction;
 
         TestSearchExtParser(Function<String, T> searchExtBuilderFunction) {
@@ -145,7 +151,7 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
         }
 
         @Override
-        public T fromXContent(XContentParser parser) throws IOException {
+        public T apply(XContentParser parser) throws IOException {
             return searchExtBuilderFunction.apply(parseField(parser));
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/ResponseListenerPluginIT.java b/core/src/test/java/org/elasticsearch/search/ResponseListenerPluginIT.java
new file mode 100644
index 00000000000..0fc36a42b18
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/ResponseListenerPluginIT.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class ResponseListenerPluginIT extends ESIntegTestCase {
+
+    static AtomicBoolean hookWasFired = new AtomicBoolean(false);
+
+    public static class FooResponseListenerPlugin extends Plugin implements SearchPlugin {
+        @Override
+        public List<BiConsumer<SearchRequest, SearchResponse>> getSearchResponseListeners() {
+            return singletonList((searchRequest, response) -> {
+                assertThat(response.getTookInMillis(), greaterThan(0L));
+                boolean alreadyFired = hookWasFired.getAndSet(true);
+                assertFalse(alreadyFired);
+            });
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(FooResponseListenerPlugin.class);
+    }
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        createIndex("test");
+        ensureGreen();
+        client().prepareIndex("index", "type", "1").setSource("field", "value").get();
+        refresh();
+    }
+
+    public void testSearchResponseHook() {
+        assertHitCount(client().prepareSearch("index").setQuery(QueryBuilders.matchAllQuery()).get(), 1L);
+        boolean alreadyFired = hookWasFired.get();
+        assertTrue(alreadyFired);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java b/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java
index 840cdc8af6c..7366834391b 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchCancellationIT.java
@@ -268,7 +268,7 @@ public class SearchCancellationIT extends ESIntegTestCase {
 
         private final AtomicBoolean shouldBlock = new AtomicBoolean(true);
 
-        public NativeTestScriptedBlockFactory() {
+        NativeTestScriptedBlockFactory() {
         }
 
         public void reset() {
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 6c3e3f0bc45..0eeab9897fa 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -18,24 +18,23 @@
  */
 package org.elasticsearch.search;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.BaseAggregationBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
@@ -61,24 +60,26 @@ import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PlainHighlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.PostingsHighlighter;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.suggest.CustomSuggester;
-import org.elasticsearch.search.suggest.Suggester;
-import org.elasticsearch.search.suggest.completion.CompletionSuggester;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
-import org.elasticsearch.search.suggest.term.TermSuggester;
+import org.elasticsearch.search.suggest.CustomSuggesterSearchIT.CustomSuggestionBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.hasSize;
 
 public class SearchModuleTests extends ModuleTestCase {
 
@@ -93,13 +94,12 @@ public class SearchModuleTests extends ModuleTestCase {
                 () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeHighlighter)));
 
         SearchPlugin registersDupeSuggester = new SearchPlugin() {
-            @Override
-            public Map<String,org.elasticsearch.search.suggest.Suggester<?>> getSuggesters() {
-                return singletonMap("term", TermSuggester.INSTANCE);
+            public List<SearchPlugin.SuggesterSpec<?>> getSuggesters() {
+                return singletonList(new SuggesterSpec<>("term", TermSuggestionBuilder::new, TermSuggestionBuilder::fromXContent));
             }
         };
-        expectThrows(IllegalArgumentException.class,
-                () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeSuggester)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeSuggester)).getNamedXContents()));
 
         SearchPlugin registersDupeScoreFunction = new SearchPlugin() {
             @Override
@@ -108,8 +108,8 @@ public class SearchModuleTests extends ModuleTestCase {
                         GaussDecayFunctionBuilder.PARSER));
             }
         };
-        expectThrows(IllegalArgumentException.class,
-                () -> new SearchModule(Settings.EMPTY, false, singletonList(registersDupeScoreFunction)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeScoreFunction)).getNamedXContents()));
 
         SearchPlugin registersDupeSignificanceHeuristic = new SearchPlugin() {
             @Override
@@ -143,8 +143,8 @@ public class SearchModuleTests extends ModuleTestCase {
                 return singletonList(new QuerySpec<>(TermQueryBuilder.NAME, TermQueryBuilder::new, TermQueryBuilder::fromXContent));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupeQuery)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(
+                new SearchModule(Settings.EMPTY, false, singletonList(registersDupeQuery)).getNamedXContents()));
 
         SearchPlugin registersDupeAggregation = new SearchPlugin() {
             public List<AggregationSpec> getAggregations() {
@@ -152,8 +152,8 @@ public class SearchModuleTests extends ModuleTestCase {
                         TermsAggregationBuilder::parse));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupeAggregation)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false,
+                singletonList(registersDupeAggregation)).getNamedXContents()));
 
         SearchPlugin registersDupePipelineAggregation = new SearchPlugin() {
             public List<PipelineAggregationSpec> getPipelineAggregations() {
@@ -165,21 +165,34 @@ public class SearchModuleTests extends ModuleTestCase {
                             .addResultReader(InternalDerivative::new));
             }
         };
-        expectThrows(IllegalArgumentException.class, () -> new SearchModule(Settings.EMPTY, false,
-                singletonList(registersDupePipelineAggregation)));
+        expectThrows(IllegalArgumentException.class, () -> new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false,
+                singletonList(registersDupePipelineAggregation)).getNamedXContents()));
     }
 
     public void testRegisterSuggester() {
         SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
             @Override
-            public Map<String, Suggester<?>> getSuggesters() {
-                return singletonMap("custom", CustomSuggester.INSTANCE);
+            public List<SuggesterSpec<?>> getSuggesters() {
+                return singletonList(new SuggesterSpec<>("custom", CustomSuggestionBuilder::new, CustomSuggestionBuilder::fromXContent));
             }
         }));
-        assertSame(TermSuggester.INSTANCE, module.getSuggesters().getSuggester("term"));
-        assertSame(PhraseSuggester.INSTANCE, module.getSuggesters().getSuggester("phrase"));
-        assertSame(CompletionSuggester.INSTANCE, module.getSuggesters().getSuggester("completion"));
-        assertSame(CustomSuggester.INSTANCE, module.getSuggesters().getSuggester("custom"));
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("term")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("phrase")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("completion")).count());
+        assertEquals(1, module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.match("custom")).count());
+
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("term")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("phrase")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("completion")).count());
+        assertEquals(1, module.getNamedWriteables().stream()
+                .filter(e -> e.categoryClass.equals(SuggestionBuilder.class) && e.name.equals("custom")).count());
     }
 
     public void testRegisterHighlighter() {
@@ -199,31 +212,22 @@ public class SearchModuleTests extends ModuleTestCase {
     }
 
     public void testRegisteredQueries() throws IOException {
-        SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
         List<String> allSupportedQueries = new ArrayList<>();
         Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES);
         Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES);
-        String[] supportedQueries = allSupportedQueries.toArray(new String[allSupportedQueries.size()]);
-        assertThat(module.getQueryParserRegistry().getNames(), containsInAnyOrder(supportedQueries));
+        SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
 
-        IndicesQueriesRegistry indicesQueriesRegistry = module.getQueryParserRegistry();
-        XContentParser dummyParser = createParser(JsonXContent.jsonXContent, new BytesArray("{}"));
-        for (String queryName : supportedQueries) {
-            indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.EMPTY, dummyParser.getTokenLocation());
-        }
+        Set<String> registeredNonDeprecated = module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(QueryBuilder.class))
+                .map(e -> e.name.getPreferredName())
+                .collect(toSet());
+        Set<String> registeredAll = module.getNamedXContents().stream()
+                .filter(e -> e.categoryClass.equals(QueryBuilder.class))
+                .flatMap(e -> Arrays.stream(e.name.getAllNamesIncludedDeprecated()))
+                .collect(toSet());
 
-        for (String queryName : NON_DEPRECATED_QUERIES) {
-            QueryParser<?> queryParser = indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
-            assertThat(queryParser, notNullValue());
-        }
-        for (String queryName : DEPRECATED_QUERIES) {
-            try {
-                indicesQueriesRegistry.lookup(queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
-                fail("query is deprecated, getQueryParser should have failed in strict mode");
-            } catch(IllegalArgumentException e) {
-                assertThat(e.getMessage(), containsString("Deprecated field [" + queryName + "] used"));
-            }
-        }
+        assertThat(registeredNonDeprecated, containsInAnyOrder(NON_DEPRECATED_QUERIES));
+        assertThat(registeredAll, containsInAnyOrder(allSupportedQueries.toArray(new String[0])));
     }
 
     public void testRegisterAggregation() {
@@ -233,7 +237,11 @@ public class SearchModuleTests extends ModuleTestCase {
             }
         }));
 
-        assertNotNull(module.getAggregatorParsers().parser("test", ParseFieldMatcher.STRICT));
+        assertThat(
+                module.getNamedXContents().stream()
+                    .filter(entry -> entry.categoryClass.equals(BaseAggregationBuilder.class) && entry.name.match("test"))
+                    .collect(toList()),
+                hasSize(1));
     }
 
     public void testRegisterPipelineAggregation() {
@@ -244,7 +252,23 @@ public class SearchModuleTests extends ModuleTestCase {
             }
         }));
 
-        assertNotNull(module.getAggregatorParsers().pipelineParser("test", ParseFieldMatcher.STRICT));
+        assertThat(
+                module.getNamedXContents().stream()
+                    .filter(entry -> entry.categoryClass.equals(BaseAggregationBuilder.class) && entry.name.match("test"))
+                    .collect(toList()),
+                hasSize(1));
+    }
+
+    public void testRegisterSearchResponseListener() {
+        BiConsumer<SearchRequest, SearchResponse> listener = (s, r) -> {};
+        SearchModule module = new SearchModule(Settings.EMPTY, false, singletonList(new SearchPlugin() {
+            public List<BiConsumer<SearchRequest, SearchResponse>> getSearchResponseListeners() {
+                return singletonList(listener);
+            }
+        }));
+        List<BiConsumer<SearchRequest, SearchResponse>> listeners = module.getSearchResponseListeners();
+        assertEquals(listeners.size(), 1);
+        assertEquals(listeners.get(0), listener);
     }
 
     private static final String[] NON_DEPRECATED_QUERIES = new String[] {
@@ -309,7 +333,7 @@ public class SearchModuleTests extends ModuleTestCase {
         }
 
         @Override
-        public String getWriteableName() {
+        public String getType() {
             return "test";
         }
 
@@ -350,7 +374,7 @@ public class SearchModuleTests extends ModuleTestCase {
         /**
          * Read from a stream.
          */
-        public TestPipelineAggregationBuilder(StreamInput in) throws IOException {
+        TestPipelineAggregationBuilder(StreamInput in) throws IOException {
             super(in, "test");
         }
 
@@ -395,7 +419,7 @@ public class SearchModuleTests extends ModuleTestCase {
         /**
          * Read from a stream.
          */
-        public TestPipelineAggregator(StreamInput in) throws IOException {
+        TestPipelineAggregator(StreamInput in) throws IOException {
             super(in);
         }
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
index 031cab1286a..9e58bf26744 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
@@ -95,7 +95,7 @@ public class SearchRequestTests extends AbstractSearchTestCase {
         mutators.add(() -> mutation.routing(randomValueOtherThan(searchRequest.routing(), () -> randomAsciiOfLengthBetween(3, 10))));
         mutators.add(() -> mutation.requestCache((randomValueOtherThan(searchRequest.requestCache(), () -> randomBoolean()))));
         mutators.add(() -> mutation
-                .scroll(randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomPositiveLong() % 100000)))));
+                .scroll(randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomNonNegativeLong() % 100000)))));
         mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(SearchType.values()))));
         mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder)));
         randomFrom(mutators).run();
diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
index 6542bad5b8a..0ac5fcd9013 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
@@ -54,7 +54,7 @@ public class SearchWithRejectionsIT extends ESIntegTestCase {
 
         int numSearches = 10;
         Future<SearchResponse>[] responses = new Future[numSearches];
-        SearchType searchType = randomFrom(SearchType.DEFAULT, SearchType.QUERY_AND_FETCH, SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_AND_FETCH, SearchType.DFS_QUERY_THEN_FETCH);
+        SearchType searchType = randomFrom(SearchType.DEFAULT, SearchType.QUERY_AND_FETCH, SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH);
         logger.info("search type is {}", searchType);
         for (int i = 0; i < numSearches; i++) {
             responses[i] = client().prepareSearch()
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
index c81ec476b86..4ddab52110b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationCollectorTests.java
@@ -19,13 +19,10 @@
 
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -60,13 +57,11 @@ public class AggregationCollectorTests extends ESSingleNodeTestCase {
     }
 
     private boolean needsScores(IndexService index, String agg) throws IOException {
-        AggregatorParsers parser = getInstanceFromNode(SearchRequestParsers.class).aggParsers;
-        IndicesQueriesRegistry queriesRegistry = getInstanceFromNode(IndicesQueriesRegistry.class);
         XContentParser aggParser = createParser(JsonXContent.jsonXContent, agg);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, aggParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(aggParser);
         aggParser.nextToken();
         SearchContext context = createSearchContext(index);
-        final AggregatorFactories factories = parser.parseAggregators(parseContext).build(context, null);
+        final AggregatorFactories factories = AggregatorFactories.parseAggregators(parseContext).build(context, null);
         final Aggregator[] aggregators = factories.createTopLevelAggregators();
         assertEquals(1, aggregators.length);
         return aggregators[0].needsScores();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
similarity index 65%
rename from core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
index 26b3dd6f8f2..2aba0ed59ee 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java
@@ -16,46 +16,36 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.List;
 import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
-public class AggregatorParsingTests extends ESTestCase {
-
+public class AggregatorFactoriesTests extends ESTestCase {
     private String[] currentTypes;
 
-    protected String[] getCurrentTypes() {
-        return currentTypes;
-    }
-
-    protected AggregatorParsers aggParsers;
-    protected IndicesQueriesRegistry queriesRegistry;
-    protected ParseFieldMatcher parseFieldMatcher;
-
-    /**
-     * Setup for the whole base test class.
-     */
+    private NamedXContentRegistry xContentRegistry;
     @Override
     public void setUp() throws Exception {
         super.setUp();
@@ -64,8 +54,6 @@ public class AggregatorParsingTests extends ESTestCase {
         Settings settings = Settings.builder().put("node.name", AbstractQueryTestCase.class.toString())
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-        SearchModule searchModule = new SearchModule(settings, false, emptyList());
-        aggParsers = searchModule.getSearchRequestParsers().aggParsers;
         // create some random type with some default field, those types will
         // stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
@@ -73,8 +61,23 @@ public class AggregatorParsingTests extends ESTestCase {
             String type = randomAsciiOfLengthBetween(1, 10);
             currentTypes[i] = type;
         }
-        queriesRegistry = searchModule.getQueryParserRegistry();
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
+        xContentRegistry = new NamedXContentRegistry(new SearchModule(settings, false, emptyList()).getNamedXContents());
+    }
+
+    public void testGetAggregatorFactories_returnsUnmodifiableList() {
+        AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo"));
+        List<AggregationBuilder> aggregatorFactories = builder.getAggregatorFactories();
+        assertThat(aggregatorFactories.size(), equalTo(1));
+        expectThrows(UnsupportedOperationException.class, () -> aggregatorFactories.add(AggregationBuilders.avg("bar")));
+    }
+
+    public void testGetPipelineAggregatorFactories_returnsUnmodifiableList() {
+        AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addPipelineAggregator(
+            PipelineAggregatorBuilders.avgBucket("foo", "path1"));
+        List<PipelineAggregationBuilder> pipelineAggregatorFactories = builder.getPipelineAggregatorFactories();
+        assertThat(pipelineAggregatorFactories.size(), equalTo(1));
+        expectThrows(UnsupportedOperationException.class,
+            () -> pipelineAggregatorFactories.add(PipelineAggregatorBuilders.avgBucket("bar", "path2")));
     }
 
     public void testTwoTypes() throws Exception {
@@ -93,15 +96,11 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Found two aggregation type definitions in [in_stock]: [filter] and [terms]"));
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Found two aggregation type definitions in [in_stock]: [filter] and [terms]"));
     }
 
     public void testTwoAggs() throws Exception {
@@ -130,15 +129,11 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Found two sub aggregation definitions under [by_date]"));
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Found two sub aggregation definitions under [by_date]"));
     }
 
     public void testInvalidAggregationName() throws Exception {
@@ -169,15 +164,11 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            assertThat(e.toString(), containsString("Invalid aggregation name [" + name + "]"));
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Invalid aggregation name [" + name + "]"));
     }
 
     public void testSameAggregationName() throws Exception {
@@ -197,15 +188,11 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertThat(e.toString(), containsString("Two sibling aggregations cannot have the same name: [" + name + "]"));
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Two sibling aggregations cannot have the same name: [" + name + "]"));
     }
 
     public void testMissingName() throws Exception {
@@ -226,15 +213,11 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            // All Good
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Expected [START_OBJECT] under [field], but got a [VALUE_STRING] in [cardinality]"));
     }
 
     public void testMissingType() throws Exception {
@@ -255,14 +238,15 @@ public class AggregatorParsingTests extends ESTestCase {
                         .endObject()
                     .endObject()
                 .endObject();
-        try {
-            XContentParser parser = createParser(source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (ParsingException e) {
-            // All Good
-        }
+        XContentParser parser = createParser(source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(ParsingException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Expected [START_OBJECT] under [field], but got a [VALUE_STRING] in [tag_count]"));
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
new file mode 100644
index 00000000000..aab44c32fcf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.apache.lucene.index.CompositeReaderContext;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCache;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.IndexFieldDataService;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.fetch.FetchPhase;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
+import org.elasticsearch.search.fetch.subphase.FetchSourceSubPhase;
+import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Base class for testing {@link Aggregator} implementations.
+ * Provides helpers for constructing and searching an {@link Aggregator} implementation based on a provided
+ * {@link AggregationBuilder} instance.
+ */
+public abstract class AggregatorTestCase extends ESTestCase {
+    protected <A extends Aggregator, B extends AggregationBuilder> A createAggregator(B aggregationBuilder,
+                                                                                      IndexSearcher indexSearcher,
+                                                                                      MappedFieldType... fieldTypes) throws IOException {
+        IndexSettings indexSettings = new IndexSettings(
+            IndexMetaData.builder("_index").settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+                .numberOfShards(1)
+                .numberOfReplicas(0)
+                .creationDate(System.currentTimeMillis())
+                .build(),
+            Settings.EMPTY
+        );
+
+        Engine.Searcher searcher = new Engine.Searcher("aggregator_test", indexSearcher);
+        QueryCache queryCache = new DisabledQueryCache(indexSettings);
+        QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() {
+            @Override
+            public void onUse(Query query) {
+            }
+
+            @Override
+            public boolean shouldCache(Query query) throws IOException {
+                // never cache a query
+                return false;
+            }
+        };
+        ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(searcher, queryCache, queryCachingPolicy);
+
+        CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
+        SearchContext searchContext = mock(SearchContext.class);
+        when(searchContext.numberOfShards()).thenReturn(1);
+        when(searchContext.searcher()).thenReturn(contextIndexSearcher);
+        when(searchContext.bigArrays()).thenReturn(new MockBigArrays(Settings.EMPTY, circuitBreakerService));
+        when(searchContext.fetchPhase())
+            .thenReturn(new FetchPhase(Arrays.asList(new FetchSourceSubPhase(), new DocValueFieldsFetchSubPhase())));
+
+        // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.hasNested()).thenReturn(false);
+        when(searchContext.mapperService()).thenReturn(mapperService);
+
+        SearchLookup searchLookup = new SearchLookup(mapperService, mock(IndexFieldDataService.class), new String[]{"type"});
+        when(searchContext.lookup()).thenReturn(searchLookup);
+
+        QueryShardContext queryShardContext = mock(QueryShardContext.class);
+        for (MappedFieldType fieldType : fieldTypes) {
+            IndexFieldData<?> fieldData = fieldType.fielddataBuilder().build(indexSettings, fieldType,
+                new IndexFieldDataCache.None(), circuitBreakerService, mock(MapperService.class));
+            when(queryShardContext.fieldMapper(fieldType.name())).thenReturn(fieldType);
+            when(queryShardContext.getForField(fieldType)).thenReturn(fieldData);
+            when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
+        }
+
+        @SuppressWarnings("unchecked")
+        A aggregator = (A) aggregationBuilder.build(searchContext, null).create(null, true);
+        return aggregator;
+    }
+
+    protected <A extends InternalAggregation, C extends Aggregator> A search(IndexSearcher searcher,
+                                                                             Query query,
+                                                                             AggregationBuilder builder,
+                                                                             MappedFieldType... fieldTypes) throws IOException {
+        try (C a = createAggregator(builder, searcher, fieldTypes)) {
+            a.preCollection();
+            searcher.search(query, a);
+            a.postCollection();
+            @SuppressWarnings("unchecked")
+            A internalAgg = (A) a.buildAggregation(0L);
+            return internalAgg;
+        }
+    }
+
+    /**
+     * Divides the provided {@link IndexSearcher} in sub-searcher, one for each segment,
+     * builds an aggregator for each sub-searcher filtered by the provided {@link Query} and
+     * returns the reduced {@link InternalAggregation}.
+     */
+    protected <A extends InternalAggregation, C extends Aggregator> A searchAndReduce(IndexSearcher searcher,
+                                                                                      Query query,
+                                                                                      AggregationBuilder builder,
+                                                                                      MappedFieldType... fieldTypes) throws IOException {
+        final IndexReaderContext ctx = searcher.getTopReaderContext();
+
+        final ShardSearcher[] subSearchers;
+        if (ctx instanceof LeafReaderContext) {
+            subSearchers = new ShardSearcher[1];
+            subSearchers[0] = new ShardSearcher((LeafReaderContext) ctx, ctx);
+        } else {
+            final CompositeReaderContext compCTX = (CompositeReaderContext) ctx;
+            final int size = compCTX.leaves().size();
+            subSearchers = new ShardSearcher[size];
+            for(int searcherIDX=0;searcherIDX<subSearchers.length;searcherIDX++) {
+                final LeafReaderContext leave = compCTX.leaves().get(searcherIDX);
+                subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX);
+            }
+        }
+
+        List<InternalAggregation> aggs = new ArrayList<> ();
+        Query rewritten = searcher.rewrite(query);
+        Weight weight = searcher.createWeight(rewritten, true);
+        C root = createAggregator(builder, searcher, fieldTypes);
+        try {
+            for (ShardSearcher subSearcher : subSearchers) {
+                C a = createAggregator(builder, subSearcher, fieldTypes);
+                try {
+                    a.preCollection();
+                    subSearcher.search(weight, a);
+                    a.postCollection();
+                    aggs.add(a.buildAggregation(0L));
+                } finally {
+                    closeAgg(a);
+                }
+            }
+            if (aggs.isEmpty()) {
+                return null;
+            } else {
+                @SuppressWarnings("unchecked")
+                A internalAgg = (A) aggs.get(0).doReduce(aggs, new InternalAggregation.ReduceContext(root.context().bigArrays(), null));
+                return internalAgg;
+            }
+        } finally {
+            closeAgg(root);
+        }
+    }
+
+    private void closeAgg(Aggregator agg) {
+        agg.close();
+        for (Aggregator sub : ((AggregatorBase) agg).subAggregators) {
+            closeAgg(sub);
+        }
+    }
+
+    private static class ShardSearcher extends IndexSearcher {
+        private final List<LeafReaderContext> ctx;
+
+        ShardSearcher(LeafReaderContext ctx, IndexReaderContext parent) {
+            super(parent);
+            this.ctx = Collections.singletonList(ctx);
+        }
+
+        public void search(Weight weight, Collector collector) throws IOException {
+            search(ctx, weight, collector);
+        }
+
+        @Override
+        public String toString() {
+            return "ShardSearcher(" + ctx.get(0) + ")";
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index e8073bac354..162b7bcd1b6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -33,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
@@ -45,6 +44,7 @@ import java.util.List;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.hasSize;
 
 public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuilder<AB>> extends ESTestCase {
 
@@ -62,11 +62,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
     }
 
     private NamedWriteableRegistry namedWriteableRegistry;
-
-    protected AggregatorParsers aggParsers;
-    protected IndicesQueriesRegistry queriesRegistry;
-    protected ParseFieldMatcher parseFieldMatcher;
-
+    private NamedXContentRegistry xContentRegistry;
     protected abstract AB createTestAggregatorBuilder();
 
     /**
@@ -85,15 +81,18 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        queriesRegistry = searchModule.getQueryParserRegistry();
-        aggParsers = searchModule.getSearchRequestParsers().aggParsers;
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
         for (int i = 0; i < currentTypes.length; i++) {
             String type = randomAsciiOfLengthBetween(1, 10);
             currentTypes[i] = type;
         }
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 
     /**
@@ -111,29 +110,27 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentBuilder shuffled = shuffleXContent(builder);
         XContentParser parser = createParser(shuffled);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.name, parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.type.name(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        AggregationBuilder newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
+        AggregationBuilder newAgg = parse(parser);
         assertNotSame(newAgg, testAgg);
         assertEquals(testAgg, newAgg);
         assertEquals(testAgg.hashCode(), newAgg.hashCode());
     }
 
+    protected AggregationBuilder parse(XContentParser parser) throws IOException {
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        AggregatorFactories.Builder parsed = AggregatorFactories.parseAggregators(parseContext);
+        assertThat(parsed.getAggregatorFactories(), hasSize(1));
+        assertThat(parsed.getPipelineAggregatorFactories(), hasSize(0));
+        AggregationBuilder newAgg = parsed.getAggregatorFactories().get(0);
+        assertNull(parser.nextToken());
+        assertNotNull(newAgg);
+        return newAgg;
+    }
+
     /**
      * Test serialization and deserialization of the test AggregatorFactory.
      */
-
     public void testSerialization() throws IOException {
         AB testAgg = createTestAggregatorBuilder();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 4e51ac13df6..41bbdaae660 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -34,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractQueryTestCase;
@@ -47,6 +46,7 @@ import java.util.List;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.hamcrest.Matchers.hasSize;
 
 public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregationBuilder<AF>> extends ESTestCase {
 
@@ -63,10 +63,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
     }
 
     private NamedWriteableRegistry namedWriteableRegistry;
-
-    protected AggregatorParsers aggParsers;
-    protected IndicesQueriesRegistry queriesRegistry;
-    protected ParseFieldMatcher parseFieldMatcher;
+    private NamedXContentRegistry xContentRegistry;
 
     protected abstract AF createTestAggregatorFactory();
 
@@ -86,15 +83,13 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         entries.addAll(indicesModule.getNamedWriteables());
         entries.addAll(searchModule.getNamedWriteables());
         namedWriteableRegistry = new NamedWriteableRegistry(entries);
-        queriesRegistry = searchModule.getQueryParserRegistry();
-        aggParsers = searchModule.getSearchRequestParsers().aggParsers;
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
         for (int i = 0; i < currentTypes.length; i++) {
             String type = randomAsciiOfLengthBetween(1, 10);
             currentTypes[i] = type;
         }
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
     }
 
     /**
@@ -113,32 +108,29 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
         XContentBuilder shuffled = shuffleXContent(builder);
         XContentParser parser = createParser(shuffled);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         String contentString = factoriesBuilder.toString();
         logger.info("Content string: {}", contentString);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.getName(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.type(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(testAgg.getName(), parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
+        PipelineAggregationBuilder newAgg = parse(parser);
         assertNotSame(newAgg, testAgg);
         assertEquals(testAgg, newAgg);
         assertEquals(testAgg.hashCode(), newAgg.hashCode());
     }
 
+    protected PipelineAggregationBuilder parse(XContentParser parser) throws IOException {
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        AggregatorFactories.Builder parsed = AggregatorFactories.parseAggregators(parseContext);
+        assertThat(parsed.getAggregatorFactories(), hasSize(0));
+        assertThat(parsed.getPipelineAggregatorFactories(), hasSize(1));
+        PipelineAggregationBuilder newAgg = parsed.getPipelineAggregatorFactories().get(0);
+        assertNull(parser.nextToken());
+        assertNotNull(newAgg);
+        return newAgg;
+    }
+
     /**
      * Test serialization and deserialization of the test AggregatorFactory.
      */
-
     public void testSerialization() throws IOException {
         AF testAgg = createTestAggregatorFactory();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
@@ -202,4 +194,9 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
                 return INT_FIELD_NAME;
         }
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java
new file mode 100644
index 00000000000..1be00e31634
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/InternalAggregationTestCase.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+
+public abstract class InternalAggregationTestCase<T extends InternalAggregation> extends AbstractWireSerializingTestCase<T> {
+    private final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(
+            new SearchModule(Settings.EMPTY, false, emptyList()).getNamedWriteables());
+
+    protected abstract T createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
+
+    public final void testReduceRandom() {
+        List<T> inputs = new ArrayList<>();
+        List<InternalAggregation> toReduce = new ArrayList<>();
+        int toReduceSize = between(1, 200);
+        for (int i = 0; i < toReduceSize; i++) {
+            T t = createTestInstance();
+            inputs.add(t);
+            toReduce.add(t);
+        }
+        @SuppressWarnings("unchecked")
+        T reduced = (T) inputs.get(0).reduce(toReduce, null);
+        assertReduced(reduced, inputs);
+    }
+
+    protected abstract void assertReduced(T reduced, List<T> inputs);
+
+    @Override
+    protected final T createTestInstance() {
+        String name = randomAsciiOfLength(5);
+        List<PipelineAggregator> pipelineAggregators = new ArrayList<>();
+        // TODO populate pipelineAggregators
+        Map<String, Object> metaData = new HashMap<>();
+        int metaDataCount = randomBoolean() ? 0 : between(1, 10);
+        while (metaData.size() < metaDataCount) {
+            metaData.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+        }
+        return createTestInstance(name, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java
new file mode 100644
index 00000000000..e02ada02b5a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/AdjacencyMatrixIT.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix.Bucket;
+import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
+import org.elasticsearch.search.aggregations.metrics.avg.Avg;
+import org.elasticsearch.search.query.QueryPhaseExecutionException;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.adjacencyMatrix;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.core.IsNull.notNullValue;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class AdjacencyMatrixIT extends ESIntegTestCase {
+
+    static int numDocs, numSingleTag1Docs, numSingleTag2Docs, numTag1Docs, numTag2Docs, numMultiTagDocs;
+    static final int MAX_NUM_FILTERS = 3;
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        createIndex("idx");
+        createIndex("idx2");
+        assertAcked(client().admin().indices().prepareUpdateSettings("idx")
+                .setSettings(
+                        Settings.builder().put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), MAX_NUM_FILTERS))
+                .get());
+        
+        numDocs = randomIntBetween(5, 20);
+        numTag1Docs = randomIntBetween(1, numDocs - 1);
+        numTag2Docs = randomIntBetween(1, numDocs - numTag1Docs);
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < numTag1Docs; i++) {
+            numSingleTag1Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag1").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                // randomly index the document twice so that we have deleted
+                // docs that match the filter
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs; i < (numTag1Docs + numTag2Docs); i++) {
+            numSingleTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).field("tag", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        for (int i = numTag1Docs + numTag2Docs; i < numDocs; i++) {
+            numMultiTagDocs++;
+            numTag1Docs++;
+            numTag2Docs++;
+            XContentBuilder source = jsonBuilder().startObject().field("value", i + 1).array("tag", "tag1", "tag2").endObject();
+            builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            if (randomBoolean()) {
+                builders.add(client().prepareIndex("idx", "type", "" + i).setSource(source));
+            }
+        }
+        prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer").execute().actionGet();
+        for (int i = 0; i < 2; i++) {
+            builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i)
+                    .setSource(jsonBuilder().startObject().field("value", i * 2).endObject()));
+        }
+        indexRandom(true, builders);
+        ensureSearchable();
+    }
+
+    public void testSimple() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expected = numMultiTagDocs > 0 ? 3 : 2;
+        assertThat(matrix.getBuckets().size(), equalTo(expected));
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+
+        bucket = matrix.getBucketByKey("tag2");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag2Docs));
+
+        bucket = matrix.getBucketByKey("tag1&tag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+
+    public void testCustomSeparator() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t",
+                        newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+
+        AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey("tag1\ttag2");
+        if (numMultiTagDocs == 0) {
+            assertThat(bucket, Matchers.nullValue());
+        } else {
+            assertThat(bucket, Matchers.notNullValue());
+            assertThat(bucket.getDocCount(), equalTo((long) numMultiTagDocs));
+        }
+
+    }
+          
+
+    // See NullPointer issue when filters are empty:
+    // https://github.com/elastic/elasticsearch/issues/8438
+    public void testEmptyFilterDeclarations() throws Exception {
+        QueryBuilder emptyFilter = new BoolQueryBuilder();
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags",
+                        newMap("all", emptyFilter).add("tag1", termQuery("tag", "tag1"))))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix filters = response.getAggregations().get("tags");
+        assertThat(filters, notNullValue());
+        AdjacencyMatrix.Bucket allBucket = filters.getBucketByKey("all");
+        assertThat(allBucket.getDocCount(), equalTo((long) numDocs));
+
+        AdjacencyMatrix.Bucket bucket = filters.getBucketByKey("tag1");
+        assertThat(bucket, Matchers.notNullValue());
+        assertThat(bucket.getDocCount(), equalTo((long) numTag1Docs));
+    }
+
+    public void testWithSubAggregation() throws Exception {
+        BoolQueryBuilder boolQ = new BoolQueryBuilder();
+        boolQ.must(termQuery("tag", "tag1"));
+        boolQ.must(termQuery("tag", "tag2"));
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(
+                        adjacencyMatrix("tags",
+                                newMap("tag1", termQuery("tag", "tag1"))
+                                    .add("tag2", termQuery("tag", "tag2")) 
+                                    .add("both", boolQ))
+                               .subAggregation(avg("avg_value").field("value")))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+        AdjacencyMatrix matrix = response.getAggregations().get("tags");
+        assertThat(matrix, notNullValue());
+        assertThat(matrix.getName(), equalTo("tags"));
+
+        int expectedBuckets = 0;
+        if (numTag1Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numTag2Docs > 0) {
+            expectedBuckets ++;
+        }
+        if (numMultiTagDocs > 0) {
+            // both, both&tag1, both&tag2, tag1&tag2 
+            expectedBuckets += 4;
+        }
+
+        assertThat(matrix.getBuckets().size(), equalTo(expectedBuckets));
+        assertThat(matrix.getProperty("_bucket_count"), equalTo(expectedBuckets));
+
+        Object[] propertiesKeys = (Object[]) matrix.getProperty("_key");
+        Object[] propertiesDocCounts = (Object[]) matrix.getProperty("_count");
+        Object[] propertiesCounts = (Object[]) matrix.getProperty("avg_value.value");
+
+        assertEquals(expectedBuckets, propertiesKeys.length);
+        assertEquals(propertiesKeys.length, propertiesDocCounts.length);
+        assertEquals(propertiesKeys.length, propertiesCounts.length);
+
+        for (int i = 0; i < propertiesCounts.length; i++) {
+            AdjacencyMatrix.Bucket bucket = matrix.getBucketByKey(propertiesKeys[i].toString());
+            assertThat(bucket, Matchers.notNullValue());
+            Avg avgValue = bucket.getAggregations().get("avg_value");
+            assertThat(avgValue, notNullValue());
+            assertThat((long) propertiesDocCounts[i], equalTo(bucket.getDocCount()));
+            assertThat((double) propertiesCounts[i], equalTo(avgValue.getValue()));
+        }
+
+        AdjacencyMatrix.Bucket tag1Bucket = matrix.getBucketByKey("tag1");
+        assertThat(tag1Bucket, Matchers.notNullValue());
+        assertThat(tag1Bucket.getDocCount(), equalTo((long) numTag1Docs));
+        long sum = 0;
+        for (int i = 0; i < numSingleTag1Docs; i++) {
+            sum += i + 1;
+        }
+        for (int i = numSingleTag1Docs + numSingleTag2Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag1Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket1Value = tag1Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket1Value, notNullValue());
+        assertThat(avgBucket1Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket1Value.getValue(), equalTo((double) sum / numTag1Docs));
+
+        Bucket tag2Bucket = matrix.getBucketByKey("tag2");
+        assertThat(tag2Bucket, Matchers.notNullValue());
+        assertThat(tag2Bucket.getDocCount(), equalTo((long) numTag2Docs));
+        sum = 0;
+        for (int i = numSingleTag1Docs; i < numDocs; i++) {
+            sum += i + 1;
+        }
+        assertThat(tag2Bucket.getAggregations().asList().isEmpty(), is(false));
+        Avg avgBucket2Value = tag2Bucket.getAggregations().get("avg_value");
+        assertThat(avgBucket2Value, notNullValue());
+        assertThat(avgBucket2Value.getName(), equalTo("avg_value"));
+        assertThat(avgBucket2Value.getValue(), equalTo((double) sum / numTag2Docs));
+
+        // Check intersection buckets are computed correctly by comparing with
+        // ANDed query bucket results
+        Bucket bucketBothQ = matrix.getBucketByKey("both");
+        if (numMultiTagDocs == 0) {
+            // Empty intersections are not returned.
+            assertThat(bucketBothQ, Matchers.nullValue());
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.nullValue());
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.nullValue());
+        } else
+        {
+            assertThat(bucketBothQ, Matchers.notNullValue());
+            assertThat(bucketBothQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueBothQ = bucketBothQ.getAggregations().get("avg_value");
+            
+            Bucket bucketIntersectQ = matrix.getBucketByKey("tag1&tag2");
+            assertThat(bucketIntersectQ, Matchers.notNullValue());
+            assertThat(bucketIntersectQ.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueIntersectQ = bucketBothQ.getAggregations().get("avg_value");
+            assertThat(avgValueIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+
+            Bucket tag1Both = matrix.getBucketByKey("both&tag1");
+            assertThat(tag1Both, Matchers.notNullValue());
+            assertThat(tag1Both.getDocCount(), equalTo((long) numMultiTagDocs));
+            Avg avgValueTag1BothIntersectQ = tag1Both.getAggregations().get("avg_value");
+            assertThat(avgValueTag1BothIntersectQ.getValue(), equalTo(avgValueBothQ.getValue()));
+        }
+
+
+    }
+    
+    public void testTooLargeMatrix() throws Exception{
+    
+        // Create more filters than is permitted by index settings.        
+        MapBuilder filtersMap = new MapBuilder();
+        for (int i = 0; i <= MAX_NUM_FILTERS; i++) {
+            filtersMap.add("tag" + i, termQuery("tag", "tag" + i));
+        }
+        
+        try {
+            client().prepareSearch("idx")
+                .addAggregation(adjacencyMatrix("tags", "\t", filtersMap))
+                .execute().actionGet();
+            fail("SearchPhaseExecutionException should have been thrown");
+        } catch (SearchPhaseExecutionException ex) {
+            assertThat(ex.getCause().getCause().getMessage(), containsString("Number of filters is too large"));
+        }        
+    }
+
+    public void testAsSubAggregation() {
+        SearchResponse response = client().prepareSearch("idx").addAggregation(histogram("histo").field("value").interval(2L)
+                .subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery())))).get();
+
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getBuckets().size(), greaterThanOrEqualTo(1));
+
+        for (Histogram.Bucket bucket : histo.getBuckets()) {
+            AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+            assertThat(matrix, notNullValue());
+            assertThat(matrix.getBuckets().size(), equalTo(1));
+            AdjacencyMatrix.Bucket filterBucket = matrix.getBuckets().get(0);
+            assertEquals(bucket.getDocCount(), filterBucket.getDocCount());
+        }
+    }
+
+    public void testWithContextBasedSubAggregation() throws Exception {
+
+        try {
+            client().prepareSearch("idx")
+                    .addAggregation(adjacencyMatrix("tags",
+                            newMap("tag1", termQuery("tag", "tag1")).add("tag2", termQuery("tag", "tag2")))
+                            .subAggregation(avg("avg_value")))
+                    .execute().actionGet();
+
+            fail("expected execution to fail - an attempt to have a context based numeric sub-aggregation, but there is not value source"
+                    + "context which the sub-aggregation can inherit");
+
+        } catch (ElasticsearchException e) {
+            assertThat(e.getMessage(), is("all shards failed"));
+        }
+    }
+
+    public void testEmptyAggregation() throws Exception {
+        SearchResponse searchResponse = client()
+                .prepareSearch("empty_bucket_idx").setQuery(matchAllQuery()).addAggregation(histogram("histo").field("value").interval(1L)
+                        .minDocCount(0).subAggregation(adjacencyMatrix("matrix", newMap("all", matchAllQuery()))))
+                .execute().actionGet();
+
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
+        Histogram histo = searchResponse.getAggregations().get("histo");
+        assertThat(histo, Matchers.notNullValue());
+        Histogram.Bucket bucket = histo.getBuckets().get(1);
+        assertThat(bucket, Matchers.notNullValue());
+
+        AdjacencyMatrix matrix = bucket.getAggregations().get("matrix");
+        assertThat(matrix, notNullValue());
+        AdjacencyMatrix.Bucket all = matrix.getBucketByKey("all");
+        assertThat(all, Matchers.nullValue());
+    }
+
+    // Helper methods for building maps of QueryBuilders
+    static MapBuilder newMap(String name, QueryBuilder builder) {
+        return new MapBuilder().add(name, builder);
+    }
+
+    static class MapBuilder extends HashMap<String, QueryBuilder> {
+        public MapBuilder add(String name, QueryBuilder builder) {
+            put(name, builder);
+            return this;
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
index d57e2774b55..22688261004 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
@@ -22,8 +22,10 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.children.Children;
@@ -121,8 +123,8 @@ public class ChildrenIT extends ESIntegTestCase {
         requests.add(client().prepareIndex("test", "article", "b").setSource("category", new String[]{"a", "b"}, "randomized", false));
         requests.add(client().prepareIndex("test", "article", "c").setSource("category", new String[]{"a", "b", "c"}, "randomized", false));
         requests.add(client().prepareIndex("test", "article", "d").setSource("category", new String[]{"c"}, "randomized", false));
-        requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}"));
-        requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}"));
+        requests.add(client().prepareIndex("test", "comment", "a").setParent("a").setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("test", "comment", "c").setParent("c").setSource("{}", XContentType.JSON));
 
         indexRandom(true, requests);
         ensureSearchable("test");
@@ -240,7 +242,7 @@ public class ChildrenIT extends ESIntegTestCase {
         );
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}"));
+        requests.add(client().prepareIndex(indexName, "parent", "1").setSource("{}", XContentType.JSON));
         requests.add(client().prepareIndex(indexName, "child", "0").setParent("1").setSource("count", 1));
         requests.add(client().prepareIndex(indexName, "child", "1").setParent("1").setSource("count", 1));
         requests.add(client().prepareIndex(indexName, "child", "2").setParent("1").setSource("count", 1));
@@ -267,7 +269,7 @@ public class ChildrenIT extends ESIntegTestCase {
              */
             UpdateResponse updateResponse = client().prepareUpdate(indexName, "child", idToUpdate)
                     .setParent("1")
-                    .setDoc("count", 1)
+                    .setDoc(Requests.INDEX_CONTENT_TYPE, "count", 1)
                     .setDetectNoop(false)
                     .get();
             assertThat(updateResponse.getVersion(), greaterThan(1L));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersAggregatorTests.java
new file mode 100644
index 00000000000..da416572755
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersAggregatorTests.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
+import org.junit.Before;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class FiltersAggregatorTests extends AggregatorTestCase {
+    private MappedFieldType fieldType;
+
+    @Before
+    public void setUpTest() throws Exception {
+        super.setUp();
+        fieldType = new KeywordFieldMapper.KeywordFieldType();
+        fieldType.setHasDocValues(true);
+        fieldType.setIndexOptions(IndexOptions.DOCS);
+        fieldType.setName("field");
+    }
+
+    public void testEmpty() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        int numFilters = randomIntBetween(1, 10);
+        QueryBuilder[] filters = new QueryBuilder[numFilters];
+        for (int i = 0; i < filters.length; i++) {
+            filters[i] = QueryBuilders.termQuery("field", randomAsciiOfLength(5));
+        }
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters);
+        builder.otherBucketKey("other");
+        InternalFilters response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+        assertEquals(response.getBuckets().size(), numFilters);
+        for (InternalFilters.InternalBucket filter : response.getBuckets()) {
+            assertEquals(filter.getDocCount(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testKeyedFilter() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new Field("field", "foo", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "else", fieldType));
+        indexWriter.addDocument(document);
+        // make sure we have more than one segment to test the merge
+        indexWriter.commit();
+        document.add(new Field("field", "foo", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "bar", fieldType));
+        indexWriter.addDocument(document);
+        document.clear();
+        document.add(new Field("field", "foobar", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.commit();
+        document.clear();
+        document.add(new Field("field", "something", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.commit();
+        document.clear();
+        document.add(new Field("field", "foobar", fieldType));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        FiltersAggregator.KeyedFilter[] keys = new FiltersAggregator.KeyedFilter[6];
+        keys[0] = new FiltersAggregator.KeyedFilter("foobar", QueryBuilders.termQuery("field", "foobar"));
+        keys[1] = new FiltersAggregator.KeyedFilter("bar", QueryBuilders.termQuery("field", "bar"));
+        keys[2] = new FiltersAggregator.KeyedFilter("foo", QueryBuilders.termQuery("field", "foo"));
+        keys[3] = new FiltersAggregator.KeyedFilter("foo2", QueryBuilders.termQuery("field", "foo"));
+        keys[4] = new FiltersAggregator.KeyedFilter("same", QueryBuilders.termQuery("field", "foo"));
+        // filter name already present so it should be merge with the previous one ?
+        keys[5] = new FiltersAggregator.KeyedFilter("same", QueryBuilders.termQuery("field", "bar"));
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", keys);
+        builder.otherBucket(true);
+        builder.otherBucketKey("other");
+        for (boolean doReduce : new boolean[] {true, false}) {
+            final InternalFilters filters;
+            if (doReduce) {
+                filters = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            } else {
+                filters = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            }
+            assertEquals(filters.getBuckets().size(), 7);
+            assertEquals(filters.getBucketByKey("foobar").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("foo").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("foo2").getDocCount(), 2);
+            assertEquals(filters.getBucketByKey("bar").getDocCount(), 1);
+            assertEquals(filters.getBucketByKey("same").getDocCount(), 1);
+            assertEquals(filters.getBucketByKey("other").getDocCount(), 2);
+        }
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testRandom() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        int numDocs = randomIntBetween(100, 200);
+        int maxTerm = randomIntBetween(10, 50);
+        int[] expectedBucketCount = new int[maxTerm];
+        Document document = new Document();
+        for (int i = 0; i < numDocs; i++) {
+            if (frequently()) {
+                // make sure we have more than one segment to test the merge
+                indexWriter.commit();
+            }
+            int value = randomInt(maxTerm-1);
+            expectedBucketCount[value] += 1;
+            document.add(new Field("field", Integer.toString(value), fieldType));
+            indexWriter.addDocument(document);
+            document.clear();
+        }
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+        int numFilters = randomIntBetween(1, 10);
+        QueryBuilder[] filters = new QueryBuilder[numFilters];
+        int[] filterTerms = new int[numFilters];
+        int expectedOtherCount = numDocs;
+        Set<Integer> filterSet = new HashSet<>();
+        for (int i = 0; i < filters.length; i++) {
+            int value = randomInt(maxTerm-1);
+            filters[i] = QueryBuilders.termQuery("field", Integer.toString(value));
+            filterTerms[i] = value;
+            if (filterSet.contains(value) == false) {
+                expectedOtherCount -= expectedBucketCount[value];
+                filterSet.add(value);
+            }
+        }
+        FiltersAggregationBuilder builder = new FiltersAggregationBuilder("test", filters);
+        builder.otherBucket(true);
+        builder.otherBucketKey("other");
+
+        for (boolean doReduce : new boolean[] {true, false}) {
+            final InternalFilters response;
+            if (doReduce) {
+                response = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            } else {
+                response = search(indexSearcher, new MatchAllDocsQuery(), builder, fieldType);
+            }
+            List<InternalFilters.InternalBucket> buckets = response.getBuckets();
+            assertEquals(buckets.size(), filters.length+1);
+
+            for (InternalFilters.InternalBucket bucket : buckets) {
+                if ("other".equals(bucket.getKey())) {
+                    assertEquals(bucket.getDocCount(), expectedOtherCount);
+                } else {
+                    int index = Integer.parseInt(bucket.getKey());
+                    assertEquals(bucket.getDocCount(), (long) expectedBucketCount[filterTerms[index]]);
+                }
+            }
+        }
+        indexReader.close();
+        directory.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalAggregatorTests.java
new file mode 100644
index 00000000000..571a32b87b7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalAggregatorTests.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
+import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
+import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+
+import java.io.IOException;
+import java.util.function.BiConsumer;
+
+import static java.util.Collections.singleton;
+
+public class GlobalAggregatorTests extends AggregatorTestCase {
+    public void testNoDocs() throws IOException {
+        testCase(iw -> {
+            // Intentionally not writing any docs
+        }, (global, min) -> {
+            assertEquals(0, global.getDocCount());
+            assertEquals(Double.POSITIVE_INFINITY, min.getValue(), 0);
+        });
+    }
+
+    public void testSomeDocs() throws IOException {
+        testCase(iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1)));
+        }, (global, min) -> {
+            assertEquals(2, global.getDocCount());
+            assertEquals(1, min.getValue(), 0);
+        });
+    }
+
+    // Note that `global`'s fancy support for ignoring the query comes from special code in AggregationPhase. We don't test that here.
+
+    private void testCase(CheckedConsumer<RandomIndexWriter, IOException> buildIndex, BiConsumer<InternalGlobal, InternalMin> verify)
+            throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        buildIndex.accept(indexWriter);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        GlobalAggregationBuilder aggregationBuilder = new GlobalAggregationBuilder("_name");
+        aggregationBuilder.subAggregation(new MinAggregationBuilder("in_global").field("number"));
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (GlobalAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            try {
+                aggregator.preCollection();
+                indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                aggregator.postCollection();
+                InternalGlobal result = (InternalGlobal) aggregator.buildAggregation(0L);
+                verify.accept(result, (InternalMin) result.getAggregations().asMap().get("in_global"));
+            } finally {
+                IOUtils.close(aggregator.subAggregators());
+            }
+        }
+        indexReader.close();
+        directory.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
index 51d702e3548..522981b4f01 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NestedIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -397,8 +398,8 @@ public class NestedIT extends ESIntegTestCase {
         ensureGreen("idx2");
 
         List<IndexRequestBuilder> indexRequests = new ArrayList<>(2);
-        indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}"));
-        indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}"));
+        indexRequests.add(client().prepareIndex("idx2", "provider", "1").setSource("{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}", XContentType.JSON));
+        indexRequests.add(client().prepareIndex("idx2", "provider", "2").setSource("{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}", XContentType.JSON));
         indexRandom(true, indexRequests);
 
         SearchResponse response = client().prepareSearch("idx2").setTypes("provider")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
index 3de40c16c28..7e460afd4f0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
@@ -128,7 +128,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testStructuredAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
                            .minDocCount(2))
@@ -144,7 +144,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
         long[] excludeTerms = { MUSIC_CATEGORY };
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "paul"))
+                .setQuery(new TermQueryBuilder("description", "paul"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
                            .minDocCount(1).includeExclude(new IncludeExclude(null, excludeTerms)))
@@ -158,7 +158,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
 
     public void testIncludeExclude() throws Exception {
         SearchResponse response = client().prepareSearch("test")
-                .setQuery(new TermQueryBuilder("_all", "weller"))
+                .setQuery(new TermQueryBuilder("description", "weller"))
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
                         .includeExclude(new IncludeExclude(null, "weller")))
                 .get();
@@ -177,7 +177,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
         assertThat(terms.contains("the"), is(true));
 
         response = client().prepareSearch("test")
-                .setQuery(new TermQueryBuilder("_all", "weller"))
+                .setQuery(new TermQueryBuilder("description", "weller"))
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
                         .includeExclude(new IncludeExclude("weller", null)))
                 .get();
@@ -194,7 +194,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testIncludeExcludeExactValues() throws Exception {
         String []incExcTerms={"weller","nosuchterm"};
         SearchResponse response = client().prepareSearch("test")
-                .setQuery(new TermQueryBuilder("_all", "weller"))
+                .setQuery(new TermQueryBuilder("description", "weller"))
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
                         .includeExclude(new IncludeExclude(null, incExcTerms)))
                 .get();
@@ -207,7 +207,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
         assertEquals(new HashSet<String>(Arrays.asList("jam", "council", "style", "paul", "of", "the")), terms);
 
         response = client().prepareSearch("test")
-                .setQuery(new TermQueryBuilder("_all", "weller"))
+                .setQuery(new TermQueryBuilder("description", "weller"))
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
                         .includeExclude(new IncludeExclude(incExcTerms, null)))
                 .get();
@@ -224,7 +224,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testUnmapped() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("fact_category").executionHint(randomExecutionHint())
                         .minDocCount(2))
@@ -238,7 +238,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testTextAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint())
                            .minDocCount(2))
@@ -252,7 +252,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testTextAnalysisGND() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint()).significanceHeuristic(new GND(true))
                         .minDocCount(2))
@@ -266,7 +266,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testTextAnalysisChiSquare() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description").executionHint(randomExecutionHint()).significanceHeuristic(new ChiSquare(false,true))
                         .minDocCount(2))
@@ -281,7 +281,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
         SearchResponse response = client()
                 .prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0)
                 .setSize(60)
                 .setExplain(true)
@@ -300,7 +300,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
         // as the background source of term statistics.
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
                            .minDocCount(2).backgroundFilter(QueryBuilders.termQuery("fact_category", 1)))
@@ -324,7 +324,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testFilteredAnalysis() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "weller"))
+                .setQuery(new TermQueryBuilder("description", "weller"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
                            .minDocCount(1).backgroundFilter(QueryBuilders.termsQuery("description",  "paul")))
@@ -374,7 +374,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testPartiallyUnmapped() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms").field("description")
                             .executionHint(randomExecutionHint())
@@ -389,7 +389,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testPartiallyUnmappedWithFormat() throws Exception {
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(boolQuery().should(termQuery("_all", "the")).should(termQuery("_all", "terje")))
+                .setQuery(boolQuery().should(termQuery("description", "the")).should(termQuery("description", "terje")))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
                         .field("fact_category")
@@ -426,7 +426,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testDefaultSignificanceHeuristic() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
                         .field("description")
@@ -443,7 +443,7 @@ public class SignificantTermsIT extends ESIntegTestCase {
     public void testMutualInformation() throws Exception {
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
-                .setQuery(new TermQueryBuilder("_all", "terje"))
+                .setQuery(new TermQueryBuilder("description", "terje"))
                 .setFrom(0).setSize(60).setExplain(true)
                 .addAggregation(significantTerms("mySignificantTerms")
                         .field("description")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index c8cd235aede..4a8c88b3093 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
@@ -305,7 +306,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
 
     public void testDeletesIssue7951() throws Exception {
         String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
-        assertAcked(prepareCreate(INDEX_NAME).setSettings(settings)
+        assertAcked(prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON)
                 .addMapping("doc", "text", "type=keyword", CLASS_FIELD, "type=keyword"));
         String[] cat1v1 = {"constant", "one"};
         String[] cat1v2 = {"constant", "uno"};
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
index 764ed3ba73b..f7e3d9a61b5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
@@ -99,10 +100,10 @@ public class TermsShardMinDocCountIT extends ESIntegTestCase {
         String sourceClass = "{\"text\": \"" + term + "\", \"class\":" + "true" + "}";
         String sourceNotClass = "{\"text\": \"" + term + "\", \"class\":" + "false" + "}";
         for (int i = 0; i < numInClass; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON));
         }
         for (int i = 0; i < numNotInClass; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceNotClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceNotClass, XContentType.JSON));
         }
     }
 
@@ -152,7 +153,7 @@ public class TermsShardMinDocCountIT extends ESIntegTestCase {
     private void addTermsDocs(String term, int numDocs, List<IndexRequestBuilder> builders) {
         String sourceClass = "{\"text\": \"" + term + "\"}";
         for (int i = 0; i < numDocs; i++) {
-            builders.add(client().prepareIndex(index, type).setSource(sourceClass));
+            builders.add(client().prepareIndex(index, type).setSource(sourceClass, XContentType.JSON));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
index 48b393ef46d..873084b3a8b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java
@@ -18,25 +18,20 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.instanceOf;
 
 public class GeoHashGridParserTests extends ESTestCase {
-    private static final IndicesQueriesRegistry mockRegistry = new IndicesQueriesRegistry();
-
     public void testParseValidFromInts() throws Exception {
         int precision = randomIntBetween(1, 12);
         XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                 "{\"field\":\"my_loc\", \"precision\":" + precision + ", \"size\": 500, \"shard_size\": 550}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry,
-                stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
@@ -47,7 +42,7 @@ public class GeoHashGridParserTests extends ESTestCase {
         int precision = randomIntBetween(1, 12);
         XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                 "{\"field\":\"my_loc\", \"precision\":\"" + precision + "\", \"size\": \"500\", \"shard_size\": \"550\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         // can create a factory
@@ -56,7 +51,7 @@ public class GeoHashGridParserTests extends ESTestCase {
 
     public void testParseErrorOnNonIntPrecision() throws Exception {
         XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"2.0\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
@@ -70,7 +65,7 @@ public class GeoHashGridParserTests extends ESTestCase {
 
     public void testParseErrorOnBooleanPrecision() throws Exception {
         XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":false}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
@@ -83,7 +78,7 @@ public class GeoHashGridParserTests extends ESTestCase {
 
     public void testParseErrorOnPrecisionOutOfRange() throws Exception {
         XContentParser stParser = createParser(JsonXContent.jsonXContent, "{\"field\":\"my_loc\", \"precision\":\"13\"}");
-        QueryParseContext parseContext = new QueryParseContext(mockRegistry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         XContentParser.Token token = stParser.nextToken();
         assertSame(XContentParser.Token.START_OBJECT, token);
         try {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
new file mode 100644
index 00000000000..1ad8f950b9a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -0,0 +1,404 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.histogram;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+
+public class DateHistogramAggregatorTests extends AggregatorTestCase {
+
+    private static final String DATE_FIELD = "date";
+    private static final String INSTANT_FIELD = "instant";
+
+    private static final List<String> dataset = Arrays.asList(
+            "2010-03-12T01:07:45",
+            "2010-04-27T03:43:34",
+            "2012-05-18T04:11:00",
+            "2013-05-29T05:11:31",
+            "2013-10-31T08:24:05",
+            "2015-02-13T13:09:32",
+            "2015-06-24T13:47:43",
+            "2015-11-13T16:14:34",
+            "2016-03-04T17:09:50",
+            "2017-12-12T22:55:46");
+
+    public void testMatchNoDocs() throws IOException {
+        testBothCases(new MatchNoDocsQuery(), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testMatchAllDocs() throws IOException {
+        Query query = new MatchAllDocsQuery();
+
+        testSearchCase(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(6, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> assertEquals(8, histogram.getBuckets().size())
+        );
+        testBothCases(query, dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD).minDocCount(1L),
+                histogram -> assertEquals(6, histogram.getBuckets().size())
+        );
+    }
+
+    public void testNoDocs() throws IOException {
+        Query query = new MatchNoDocsQuery();
+        List<String> dates = Collections.emptyList();
+        Consumer<DateHistogramAggregationBuilder> aggregation = agg ->
+                agg.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD);
+
+        testSearchCase(query, dates, aggregation,
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+        testSearchAndReduceCase(query, dates, aggregation,
+                histogram -> assertNull(histogram)
+        );
+    }
+
+    public void testAggregateWrongField() throws IOException {
+        testBothCases(new MatchAllDocsQuery(), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field("wrong_field"),
+                histogram -> assertEquals(0, histogram.getBuckets().size())
+        );
+    }
+
+    public void testIntervalYear() throws IOException {
+        testBothCases(LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2015-01-01"), asLong("2017-12-31")), dataset,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.YEAR).field(DATE_FIELD),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2015-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2016-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMonth() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList("2017-01-01", "2017-02-02", "2017-02-03", "2017-03-04", "2017-03-05", "2017-03-06"),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MONTH).field(DATE_FIELD),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-01-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-03-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalDay() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01",
+                        "2017-02-02",
+                        "2017-02-02",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-03",
+                        "2017-02-05"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.DAY).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-02T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-03T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-05T00:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalHour() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:00.000Z",
+                        "2017-02-01T09:35:00.000Z",
+                        "2017-02-01T10:15:00.000Z",
+                        "2017-02-01T13:06:00.000Z",
+                        "2017-02-01T14:04:00.000Z",
+                        "2017-02-01T14:05:00.000Z",
+                        "2017-02-01T15:59:00.000Z",
+                        "2017-02-01T16:06:00.000Z",
+                        "2017-02-01T16:48:00.000Z",
+                        "2017-02-01T16:59:00.000Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.HOUR).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(6, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(4);
+                    assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(5);
+                    assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalMinute() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T09:02:35.000Z",
+                        "2017-02-01T09:02:59.000Z",
+                        "2017-02-01T09:15:37.000Z",
+                        "2017-02-01T09:16:04.000Z",
+                        "2017-02-01T09:16:42.000Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.MINUTE).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T09:02:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T09:15:00.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T09:16:00.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testIntervalSecond() throws IOException {
+        testBothCases(new MatchAllDocsQuery(),
+                Arrays.asList(
+                        "2017-02-01T00:00:05.015Z",
+                        "2017-02-01T00:00:11.299Z",
+                        "2017-02-01T00:00:11.074Z",
+                        "2017-02-01T00:00:37.688Z",
+                        "2017-02-01T00:00:37.210Z",
+                        "2017-02-01T00:00:37.380Z"
+                ),
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.SECOND).field(DATE_FIELD).minDocCount(1L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(3, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:11.000Z", bucket.getKeyAsString());
+                    assertEquals(2, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:37.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    public void testMinDocCount() throws IOException {
+        Query query = LongPoint.newRangeQuery(INSTANT_FIELD, asLong("2017-02-01T00:00:00.000Z"), asLong("2017-02-01T00:00:30.000Z"));
+        List<String> timestamps = Arrays.asList(
+                "2017-02-01T00:00:05.015Z",
+                "2017-02-01T00:00:11.299Z",
+                "2017-02-01T00:00:11.074Z",
+                "2017-02-01T00:00:13.688Z",
+                "2017-02-01T00:00:21.380Z"
+        );
+
+        // 5 sec interval with minDocCount = 0
+        testSearchAndReduceCase(query, timestamps,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(0L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(4, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:05.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+
+                    bucket = buckets.get(1);
+                    assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+
+                    bucket = buckets.get(2);
+                    assertEquals("2017-02-01T00:00:15.000Z", bucket.getKeyAsString());
+                    assertEquals(0, bucket.getDocCount());
+
+                    bucket = buckets.get(3);
+                    assertEquals("2017-02-01T00:00:20.000Z", bucket.getKeyAsString());
+                    assertEquals(1, bucket.getDocCount());
+                }
+        );
+
+        // 5 sec interval with minDocCount = 3
+        testSearchAndReduceCase(query, timestamps,
+                aggregation -> aggregation.dateHistogramInterval(DateHistogramInterval.seconds(5)).field(DATE_FIELD).minDocCount(3L),
+                histogram -> {
+                    List<Histogram.Bucket> buckets = histogram.getBuckets();
+                    assertEquals(1, buckets.size());
+
+                    Histogram.Bucket bucket = buckets.get(0);
+                    assertEquals("2017-02-01T00:00:10.000Z", bucket.getKeyAsString());
+                    assertEquals(3, bucket.getDocCount());
+                }
+        );
+    }
+
+    private void testSearchCase(Query query, List<String> dataset,
+                                Consumer<DateHistogramAggregationBuilder> configure,
+                                Consumer<Histogram> verify) throws IOException {
+        executeTestCase(false, query, dataset, configure, verify);
+    }
+
+    private void testSearchAndReduceCase(Query query, List<String> dataset,
+                                         Consumer<DateHistogramAggregationBuilder> configure,
+                                         Consumer<Histogram> verify) throws IOException {
+        executeTestCase(true, query, dataset, configure, verify);
+    }
+
+    private void testBothCases(Query query, List<String> dataset,
+                               Consumer<DateHistogramAggregationBuilder> configure,
+                               Consumer<Histogram> verify) throws IOException {
+        testSearchCase(query, dataset, configure, verify);
+        testSearchAndReduceCase(query, dataset, configure, verify);
+    }
+
+    private void executeTestCase(boolean reduced, Query query, List<String> dataset,
+                                 Consumer<DateHistogramAggregationBuilder> configure,
+                                 Consumer<Histogram> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                for (String date : dataset) {
+                    if (frequently()) {
+                        indexWriter.commit();
+                    }
+
+                    long instant = asLong(date);
+                    document.add(new SortedNumericDocValuesField(DATE_FIELD, instant));
+                    document.add(new LongPoint(INSTANT_FIELD, instant));
+                    indexWriter.addDocument(document);
+                    document.clear();
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name");
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                DateFieldMapper.Builder builder = new DateFieldMapper.Builder("_name");
+                DateFieldMapper.DateFieldType fieldType = builder.fieldType();
+                fieldType.setHasDocValues(true);
+                fieldType.setName(aggregationBuilder.field());
+
+                InternalDateHistogram histogram;
+                if (reduced) {
+                    histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType);
+                } else {
+                    histogram = search(indexSearcher, query, aggregationBuilder, fieldType);
+                }
+                verify.accept(histogram);
+            }
+        }
+    }
+
+    private static long asLong(String dateTime) {
+        return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(dateTime).getMillis();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
index c232068fffd..16ba3ed6958 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket.histogram;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -100,7 +99,7 @@ public class ExtendedBoundsTests extends ESTestCase {
         SearchContext context = mock(SearchContext.class);
         QueryShardContext qsc = new QueryShardContext(0,
                 new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null,
-                null, null, null, null, () -> now);
+                null, xContentRegistry(), null, null, () -> now);
         when(context.getQueryShardContext()).thenReturn(qsc);
         FormatDateTimeFormatter formatter = Joda.forPattern("dateOptionalTime");
         DocValueFormat format = new DocValueFormat.DateTime(formatter, DateTimeZone.UTC);
@@ -174,7 +173,7 @@ public class ExtendedBoundsTests extends ESTestCase {
                 assertThat(token, equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(in.currentName(), equalTo(ExtendedBounds.EXTENDED_BOUNDS_FIELD.getPreferredName()));
 
-                ExtendedBounds read = ExtendedBounds.PARSER.apply(in, () -> ParseFieldMatcher.STRICT);
+                ExtendedBounds read = ExtendedBounds.PARSER.apply(in, null);
                 assertEquals(orig, read);
             } catch (Exception e) {
                 throw new Exception("Error parsing [" + out.bytes().utf8ToString() + "]", e);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 655a03a75c0..e896e1cc1fb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.significant;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -35,7 +34,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchShardTarget;
@@ -49,7 +47,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Perce
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 
@@ -79,7 +76,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 public class SignificanceHeuristicTests extends ESTestCase {
     static class SignificantTermsTestSearchContext extends TestSearchContext {
 
-        public SignificantTermsTestSearchContext() {
+        SignificantTermsTestSearchContext() {
             super(null);
         }
 
@@ -220,58 +217,56 @@ public class SignificanceHeuristicTests extends ESTestCase {
     public void testBuilderAndParser() throws Exception {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         ParseFieldRegistry<SignificanceHeuristicParser> heuristicParserMapper = searchModule.getSignificanceHeuristicParserRegistry();
-        SearchContext searchContext = new SignificantTermsTestSearchContext();
 
         // test jlh with string
-        assertTrue(parseFromString(heuristicParserMapper, searchContext, "\"jlh\":{}") instanceof JLHScore);
+        assertTrue(parseFromString(heuristicParserMapper, "\"jlh\":{}") instanceof JLHScore);
         // test gnd with string
-        assertTrue(parseFromString(heuristicParserMapper, searchContext, "\"gnd\":{}") instanceof GND);
+        assertTrue(parseFromString(heuristicParserMapper, "\"gnd\":{}") instanceof GND);
         // test mutual information with string
         boolean includeNegatives = randomBoolean();
         boolean backgroundIsSuperset = randomBoolean();
         String mutual = "\"mutual_information\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":"
                 + backgroundIsSuperset + "}";
         assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset),
-                parseFromString(heuristicParserMapper, searchContext, mutual));
+                parseFromString(heuristicParserMapper, mutual));
         String chiSquare = "\"chi_square\":{\"include_negatives\": " + includeNegatives + ", \"background_is_superset\":"
                 + backgroundIsSuperset + "}";
         assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset),
-                parseFromString(heuristicParserMapper, searchContext, chiSquare));
+                parseFromString(heuristicParserMapper, chiSquare));
 
         // test with builders
-        assertThat(parseFromBuilder(heuristicParserMapper, searchContext, new JLHScore()), instanceOf(JLHScore.class));
-        assertThat(parseFromBuilder(heuristicParserMapper, searchContext, new GND(backgroundIsSuperset)), instanceOf(GND.class));
+        assertThat(parseFromBuilder(heuristicParserMapper, new JLHScore()), instanceOf(JLHScore.class));
+        assertThat(parseFromBuilder(heuristicParserMapper, new GND(backgroundIsSuperset)), instanceOf(GND.class));
         assertEquals(new MutualInformation(includeNegatives, backgroundIsSuperset),
-                parseFromBuilder(heuristicParserMapper, searchContext, new MutualInformation(includeNegatives, backgroundIsSuperset)));
+                parseFromBuilder(heuristicParserMapper, new MutualInformation(includeNegatives, backgroundIsSuperset)));
         assertEquals(new ChiSquare(includeNegatives, backgroundIsSuperset),
-                parseFromBuilder(heuristicParserMapper, searchContext, new ChiSquare(includeNegatives, backgroundIsSuperset)));
+                parseFromBuilder(heuristicParserMapper, new ChiSquare(includeNegatives, backgroundIsSuperset)));
 
         // test exceptions
         String faultyHeuristicdefinition = "\"mutual_information\":{\"include_negatives\": false, \"some_unknown_field\": false}";
         String expectedError = "unknown field [some_unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"chi_square\":{\"unknown_field\": true}";
         expectedError = "unknown field [unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"jlh\":{\"unknown_field\": true}";
         expectedError = "expected an empty object, but found ";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
 
         faultyHeuristicdefinition = "\"gnd\":{\"unknown_field\": true}";
         expectedError = "unknown field [unknown_field]";
-        checkParseException(heuristicParserMapper, searchContext, faultyHeuristicdefinition, expectedError);
+        checkParseException(heuristicParserMapper, faultyHeuristicdefinition, expectedError);
     }
 
     protected void checkParseException(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, String faultyHeuristicDefinition, String expectedError) throws IOException {
+            String faultyHeuristicDefinition, String expectedError) throws IOException {
 
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         try {
             XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                     "{\"field\":\"text\", " + faultyHeuristicDefinition + ",\"min_doc_count\":200}");
-            QueryParseContext parseContext = new QueryParseContext(registry, stParser, ParseFieldMatcher.STRICT);
+            QueryParseContext parseContext = new QueryParseContext(stParser);
             stParser.nextToken();
             SignificantTermsAggregationBuilder.getParser(significanceHeuristicParserRegistry).parse("testagg", parseContext);
             fail();
@@ -281,20 +276,19 @@ public class SignificanceHeuristicTests extends ESTestCase {
     }
 
     protected SignificanceHeuristic parseFromBuilder(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, SignificanceHeuristic significanceHeuristic) throws IOException {
+            SignificanceHeuristic significanceHeuristic) throws IOException {
         SignificantTermsAggregationBuilder stBuilder = significantTerms("testagg");
         stBuilder.significanceHeuristic(significanceHeuristic).field("text").minDocCount(200);
         XContentBuilder stXContentBuilder = XContentFactory.jsonBuilder();
         stBuilder.internalXContent(stXContentBuilder, null);
         XContentParser stParser = createParser(JsonXContent.jsonXContent, stXContentBuilder.string());
-        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, searchContext, stParser);
+        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, stParser);
     }
 
     private SignificanceHeuristic parseSignificanceHeuristic(
-            ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry, SearchContext searchContext,
+            ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
             XContentParser stParser) throws IOException {
-        IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
-        QueryParseContext parseContext = new QueryParseContext(registry, stParser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(stParser);
         stParser.nextToken();
         SignificantTermsAggregationBuilder aggregatorFactory =
                 (SignificantTermsAggregationBuilder) SignificantTermsAggregationBuilder.getParser(
@@ -307,10 +301,10 @@ public class SignificanceHeuristicTests extends ESTestCase {
     }
 
     protected SignificanceHeuristic parseFromString(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
-            SearchContext searchContext, String heuristicString) throws IOException {
+            String heuristicString) throws IOException {
         XContentParser stParser = createParser(JsonXContent.jsonXContent, 
                 "{\"field\":\"text\", " + heuristicString + ", \"min_doc_count\":200}");
-        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, searchContext, stParser);
+        return parseSignificanceHeuristic(significanceHeuristicParserRegistry, stParser);
     }
 
     void testBackgroundAssertions(SignificanceHeuristic heuristicIsSuperset, SignificanceHeuristic heuristicNotSuperset) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
new file mode 100644
index 00000000000..cb3165f2bed
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.support.ValueType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TermsAggregatorTests extends AggregatorTestCase {
+
+    public void testTermsAggregator() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("c")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("a")));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedSetDocValuesField("string", new BytesRef("b")));
+        document.add(new SortedSetDocValuesField("string", new BytesRef("d")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader
+        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+
+        for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) {
+            TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.STRING)
+                .executionHint(executionMode.toString())
+                .field("string")
+                .order(Terms.Order.term(true));
+            MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+            fieldType.setName("string");
+            fieldType.setHasDocValues(true );
+            try (TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+                aggregator.preCollection();
+                indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+                aggregator.postCollection();
+                Terms result = (Terms) aggregator.buildAggregation(0L);
+                assertEquals(4, result.getBuckets().size());
+                assertEquals("a", result.getBuckets().get(0).getKeyAsString());
+                assertEquals(2L, result.getBuckets().get(0).getDocCount());
+                assertEquals("b", result.getBuckets().get(1).getKeyAsString());
+                assertEquals(2L, result.getBuckets().get(1).getDocCount());
+                assertEquals("c", result.getBuckets().get(2).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(2).getDocCount());
+                assertEquals("d", result.getBuckets().get(3).getKeyAsString());
+                assertEquals(1L, result.getBuckets().get(3).getDocCount());
+            }
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMixLongAndDouble() throws Exception {
+        for (TermsAggregatorFactory.ExecutionMode executionMode : TermsAggregatorFactory.ExecutionMode.values()) {
+            TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name", ValueType.LONG)
+                .executionHint(executionMode.toString())
+                .field("number")
+                .order(Terms.Order.term(true));
+            List<InternalAggregation> aggs = new ArrayList<> ();
+            int numLongs = randomIntBetween(1, 3);
+            for (int i = 0; i < numLongs; i++) {
+                final Directory dir;
+                try (IndexReader reader = createIndexWithLongs()) {
+                    dir = ((DirectoryReader) reader).directory();
+                    IndexSearcher searcher = new IndexSearcher(reader);
+                    MappedFieldType fieldType =
+                        new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+                    fieldType.setName("number");
+                    fieldType.setHasDocValues(true);
+                    aggs.add(buildInternalAggregation(aggregationBuilder, fieldType, searcher));
+                }
+                dir.close();
+            }
+            int numDoubles = randomIntBetween(1, 3);
+            for (int i = 0; i < numDoubles; i++) {
+                final Directory dir;
+                try (IndexReader reader = createIndexWithDoubles()) {
+                    dir = ((DirectoryReader) reader).directory();
+                    IndexSearcher searcher = new IndexSearcher(reader);
+                    MappedFieldType fieldType =
+                        new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+                    fieldType.setName("number");
+                    fieldType.setHasDocValues(true);
+                    aggs.add(buildInternalAggregation(aggregationBuilder, fieldType, searcher));
+                }
+                dir.close();
+            }
+            InternalAggregation.ReduceContext ctx =
+                new InternalAggregation.ReduceContext(new MockBigArrays(Settings.EMPTY,
+                    new NoneCircuitBreakerService()), null);
+            for (InternalAggregation internalAgg : aggs) {
+                InternalAggregation mergedAggs = internalAgg.doReduce(aggs, ctx);
+                assertTrue(mergedAggs instanceof DoubleTerms);
+                long expected = numLongs + numDoubles;
+                List<Terms.Bucket> buckets = ((DoubleTerms) mergedAggs).getBuckets();
+                assertEquals(4, buckets.size());
+                assertEquals("1.0", buckets.get(0).getKeyAsString());
+                assertEquals(expected, buckets.get(0).getDocCount());
+                assertEquals("10.0", buckets.get(1).getKeyAsString());
+                assertEquals(expected * 2, buckets.get(1).getDocCount());
+                assertEquals("100.0", buckets.get(2).getKeyAsString());
+                assertEquals(expected * 2, buckets.get(2).getDocCount());
+                assertEquals("1000.0", buckets.get(3).getKeyAsString());
+                assertEquals(expected, buckets.get(3).getDocCount());
+            }
+        }
+    }
+
+    private IndexReader createIndexWithLongs() throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 10));
+        document.add(new SortedNumericDocValuesField("number", 100));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 1));
+        document.add(new SortedNumericDocValuesField("number", 100));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 10));
+        document.add(new SortedNumericDocValuesField("number", 1000));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+        return DirectoryReader.open(directory);
+    }
+
+    private IndexReader createIndexWithDoubles() throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(10.0d)));
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(100.0d)));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(1.0d)));
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(100.0d)));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(10.0d)));
+        document.add(new SortedNumericDocValuesField("number", NumericUtils.doubleToSortableLong(1000.0d)));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+        return DirectoryReader.open(directory);
+    }
+
+    private InternalAggregation buildInternalAggregation(TermsAggregationBuilder builder, MappedFieldType fieldType,
+                                                         IndexSearcher searcher) throws IOException {
+        try (TermsAggregator aggregator = createAggregator(builder, searcher, fieldType)) {
+            aggregator.preCollection();
+            searcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            return aggregator.buildAggregation(0L);
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 6a4eec89f23..de8416daf0a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -182,9 +182,8 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase {
                 .order(SortOrder.ASC)).setSize(5000).get();
         assertSearchResponse(response);
         long totalHits = response.getHits().totalHits();
-        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
         logger.info("Full high_card_idx Response Content:\n{ {} }", builder.string());
         for (int i = 0; i < totalHits; i++) {
             SearchHit searchHit = response.getHits().getAt(i);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java
new file mode 100644
index 00000000000..05390b1282d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AdjacencyMatrixTests.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AdjacencyMatrixTests extends BaseAggregationTestCase<AdjacencyMatrixAggregationBuilder> {
+
+    @Override
+    protected AdjacencyMatrixAggregationBuilder createTestAggregatorBuilder() {
+
+        int size = randomIntBetween(1, 20);
+        AdjacencyMatrixAggregationBuilder factory;
+        Map<String, QueryBuilder> filters = new HashMap<>(size);
+        for (String key : randomUnique(() -> randomAsciiOfLengthBetween(1, 20), size)) {
+            filters.put(key, QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
+        }
+        factory = new AdjacencyMatrixAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters)
+                .separator(randomFrom("&","+","\t"));       
+        return factory;
+    }
+
+    /**
+     * Test that when passing in keyed filters as a map they are equivalent
+     */
+    public void testFiltersSameMap() {
+        Map<String, QueryBuilder> original = new HashMap<>();
+        original.put("bbb", new MatchNoneQueryBuilder());
+        original.put("aaa", new MatchNoneQueryBuilder());
+        AdjacencyMatrixAggregationBuilder builder;
+        builder = new AdjacencyMatrixAggregationBuilder("my-agg", original);
+        assertEquals(original, builder.filters());
+        assert original != builder.filters();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
index 496e77f364c..226f3aeada1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -28,7 +27,6 @@ import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
@@ -89,8 +87,7 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregationBuil
         builder.endObject();
         XContentParser parser = createParser(shuffleXContent(builder));
         parser.nextToken();
-        QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser,
-            ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         FiltersAggregationBuilder filters = FiltersAggregationBuilder.parse("agg_name", context);
         // The other bucket is disabled by default
         assertFalse(filters.otherBucket());
@@ -102,7 +99,7 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregationBuil
         builder.endObject();
         parser = createParser(shuffleXContent(builder));
         parser.nextToken();
-        context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        context = new QueryParseContext(parser);
         filters = FiltersAggregationBuilder.parse("agg_name", context);
         // but setting a key enables it automatically
         assertTrue(filters.otherBucket());
@@ -115,7 +112,7 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregationBuil
         builder.endObject();
         parser = createParser(shuffleXContent(builder));
         parser.nextToken();
-        context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        context = new QueryParseContext(parser);
         filters = FiltersAggregationBuilder.parse("agg_name", context);
         // unless the other bucket is explicitly disabled
         assertFalse(filters.otherBucket());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java
new file mode 100644
index 00000000000..de045ff533e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalMaxTests.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalMaxTests extends InternalAggregationTestCase<InternalMax> {
+    @Override
+    protected InternalMax createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalMax(name, randomDouble(),
+                randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), pipelineAggregators,
+                metaData);
+    }
+
+    @Override
+    protected Reader<InternalMax> instanceReader() {
+        return InternalMax::new;
+    }
+
+    @Override
+    protected void assertReduced(InternalMax reduced, List<InternalMax> inputs) {
+        assertEquals(inputs.stream().mapToDouble(InternalMax::value).max().getAsDouble(), reduced.value(), 0);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
new file mode 100644
index 00000000000..c53927a55ba
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxAggregatorTests.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.FieldValueQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregator;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singleton;
+
+public class MaxAggregatorTests extends AggregatorTestCase {
+    public void testNoDocs() throws IOException {
+        testCase(new MatchAllDocsQuery(), iw -> {
+            // Intentionally not writing any docs
+        }, max -> {
+            assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0);
+        });
+    }
+
+    public void testNoMatchingField() throws IOException {
+        testCase(new MatchAllDocsQuery(), iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
+        }, max -> {
+            assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesSortedNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery("number"), iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(7, max.getValue(), 0);
+        });
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery("number"), iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField("number", 7)));
+            iw.addDocument(singleton(new NumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(7, max.getValue(), 0);
+        });
+    }
+
+
+    public void testQueryFiltering() throws IOException {
+        testCase(IntPoint.newRangeQuery("number", 0, 5), iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(1, max.getValue(), 0);
+        });
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        testCase(IntPoint.newRangeQuery("number", -1, 0), iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7)));
+            iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1)));
+        }, max -> {
+            assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0);
+        });
+    }
+
+    private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalMax> verify)
+            throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        buildIndex.accept(indexWriter);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MaxAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(query, aggregator);
+            aggregator.postCollection();
+            verify.accept((InternalMax) aggregator.buildAggregation(0L));
+        }
+        indexReader.close();
+        directory.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
index 545c10bcb03..f431cb0a478 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
@@ -228,24 +229,24 @@ public class ScriptedMetricIT extends ESIntegTestCase {
         // the id of the stored script is used in test method while the source of the stored script
         // must match a predefined script from CustomScriptPlugin.pluginScripts() method
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("initScript_stored")
-                .setSource(new BytesArray("{\"script\":\"vars.multiplier = 3\"}")));
+                .setContent(new BytesArray("{\"script\":\"vars.multiplier = 3\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("mapScript_stored")
-                .setSource(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}")));
+                .setContent(new BytesArray("{\"script\":\"_agg.add(vars.multiplier)\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("combineScript_stored")
-                .setSource(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}")));
+                .setContent(new BytesArray("{\"script\":\"sum agg values as a new aggregation\"}"), XContentType.JSON));
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 .setId("reduceScript_stored")
-                .setSource(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}")));
+                .setContent(new BytesArray("{\"script\":\"sum aggs of agg values as a new aggregation\"}"), XContentType.JSON));
 
         indexRandom(true, builders);
         ensureSearchable();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
index 11c6ed7f6a2..8e09447fef0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
@@ -55,8 +55,9 @@ public class ScriptedMetricTests extends BaseAggregationTestCase<ScriptedMetricA
         if (randomBoolean()) {
             return new Script(script);
         } else {
+            ScriptType type = randomFrom(ScriptType.values());
             return new Script(
-                randomFrom(ScriptType.values()), randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), script, Collections.emptyMap());
+                type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), script, Collections.emptyMap());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
index e076472a133..55fe1b44b79 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
@@ -19,13 +19,12 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -33,6 +32,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -186,15 +186,11 @@ public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregationBuil
             "      }\n" +
             "    }\n" +
             "}";
-        try {
-            XContentParser parser = createParser(JsonXContent.jsonXContent, source);
-            QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-            assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-            aggParsers.parseAggregators(parseContext);
-            fail();
-        } catch (AggregationInitializationException e) {
-            assertThat(e.toString(), containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations"));
-        }
+        XContentParser parser = createParser(JsonXContent.jsonXContent, source);
+        QueryParseContext parseContext = new QueryParseContext(parser);
+        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
+        Exception e = expectThrows(AggregationInitializationException.class, () -> AggregatorFactories.parseAggregators(parseContext));
+        assertThat(e.toString(), containsString("Aggregator [top_tags_hits] of type [top_hits] cannot accept sub-aggregations"));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java
new file mode 100644
index 00000000000..f93e7c5c81e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.min;
+
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalMinTests extends InternalAggregationTestCase<InternalMin> {
+    @Override
+    protected InternalMin createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        return new InternalMin(name, randomDouble(),
+                randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), pipelineAggregators,
+                metaData);
+    }
+
+    @Override
+    protected Reader<InternalMin> instanceReader() {
+        return InternalMin::new;
+    }
+
+    @Override
+    protected void assertReduced(InternalMin reduced, List<InternalMin> inputs) {
+        assertEquals(inputs.stream().mapToDouble(InternalMin::value).min().getAsDouble(), reduced.value(), 0);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java
new file mode 100644
index 00000000000..48b0b115e8a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.min;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+
+public class MinAggregatorTests extends AggregatorTestCase {
+
+    public void testMinAggregator_numericDv() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new NumericDocValuesField("number", 9));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 5));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", 1));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new NumericDocValuesField("number", -1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(-1.0, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_sortedNumericDv() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 9));
+        document.add(new SortedNumericDocValuesField("number", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 5));
+        document.add(new SortedNumericDocValuesField("number", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number", 1));
+        document.add(new SortedNumericDocValuesField("number", -1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(-1.0, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_noValue() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        Document document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 7));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 3));
+        indexWriter.addDocument(document);
+        document = new Document();
+        document.add(new SortedNumericDocValuesField("number1", 1));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number2");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number2");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testMinAggregator_noDocs() throws Exception {
+        Directory directory = newDirectory();
+        RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+        MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
+        MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+        fieldType.setName("number");
+        try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+            aggregator.preCollection();
+            indexSearcher.search(new MatchAllDocsQuery(), aggregator);
+            aggregator.postCollection();
+            InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
+            assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
+        }
+        indexReader.close();
+        directory.close();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java
new file mode 100644
index 00000000000..192dc92d4c3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/InternalTopHitsTests.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.tophits;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopFieldDocs;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.internal.InternalSearchHits;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Comparator.comparing;
+
+public class InternalTopHitsTests extends InternalAggregationTestCase<InternalTopHits> {
+    /**
+     * Should the test instances look like they are sorted by some fields (true) or sorted by score (false). Set here because these need
+     * to be the same across the entirety of {@link #testReduceRandom()}.
+     */
+    private final boolean testInstancesLookSortedByField = randomBoolean();
+    /**
+     * Fields shared by all instances created by {@link #createTestInstance(String, List, Map)}.
+     */
+    private final SortField[] testInstancesSortFields = testInstancesLookSortedByField ? randomSortFields() : new SortField[0];
+
+    @Override
+    protected InternalTopHits createTestInstance(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        int from = 0;
+        int requestedSize = between(1, 40);
+        int actualSize = between(0, requestedSize);
+
+        float maxScore = Float.MIN_VALUE;
+        ScoreDoc[] scoreDocs = new ScoreDoc[actualSize];
+        InternalSearchHit[] hits = new InternalSearchHit[actualSize];
+        Set<Integer> usedDocIds = new HashSet<>();
+        for (int i = 0; i < actualSize; i++) {
+            float score = randomFloat();
+            maxScore = max(maxScore, score);
+            int docId = randomValueOtherThanMany(usedDocIds::contains, () -> between(0, IndexWriter.MAX_DOCS));
+            usedDocIds.add(docId);
+
+            Map<String, SearchHitField> searchHitFields = new HashMap<>();
+            if (testInstancesLookSortedByField) {
+                Object[] fields = new Object[testInstancesSortFields.length];
+                for (int f = 0; f < testInstancesSortFields.length; f++) {
+                    fields[f] = randomOfType(testInstancesSortFields[f].getType());
+                }
+                scoreDocs[i] = new FieldDoc(docId, score, fields);
+            } else {
+                scoreDocs[i] = new ScoreDoc(docId, score);
+            }
+            hits[i] = new InternalSearchHit(docId, Integer.toString(i), new Text("test"), searchHitFields);
+            hits[i].score(score);
+        }
+        int totalHits = between(actualSize, 500000);
+        InternalSearchHits internalSearchHits = new InternalSearchHits(hits, totalHits, maxScore);
+        
+        TopDocs topDocs;
+        Arrays.sort(scoreDocs, scoreDocComparator());
+        if (testInstancesLookSortedByField) {
+            topDocs = new TopFieldDocs(totalHits, scoreDocs, testInstancesSortFields, maxScore);
+        } else {
+            topDocs = new TopDocs(totalHits, scoreDocs, maxScore);
+        }
+
+        return new InternalTopHits(name, from, requestedSize, topDocs, internalSearchHits, pipelineAggregators, metaData);
+    }
+
+    private Object randomOfType(SortField.Type type) {
+        switch (type) {
+        case CUSTOM:
+            throw new UnsupportedOperationException();
+        case DOC:
+            return between(0, IndexWriter.MAX_DOCS);
+        case DOUBLE:
+            return randomDouble();
+        case FLOAT:
+            return randomFloat();
+        case INT:
+            return randomInt();
+        case LONG:
+            return randomLong();
+        case REWRITEABLE:
+            throw new UnsupportedOperationException();
+        case SCORE:
+            return randomFloat();
+        case STRING:
+            return new BytesRef(randomAsciiOfLength(5));
+        case STRING_VAL:
+            return new BytesRef(randomAsciiOfLength(5));
+        default:
+            throw new UnsupportedOperationException("Unkown SortField.Type: " + type);
+        }
+    }
+
+    @Override
+    protected void assertReduced(InternalTopHits reduced, List<InternalTopHits> inputs) {
+        InternalSearchHits actualHits = (InternalSearchHits) reduced.getHits();
+        List<Tuple<ScoreDoc, InternalSearchHit>> allHits = new ArrayList<>();
+        float maxScore = Float.MIN_VALUE;
+        long totalHits = 0;
+        for (int input = 0; input < inputs.size(); input++) {
+            InternalSearchHits internalHits = (InternalSearchHits) inputs.get(input).getHits();
+            totalHits += internalHits.totalHits();
+            maxScore = max(maxScore, internalHits.maxScore());
+            for (int i = 0; i < internalHits.internalHits().length; i++) {
+                ScoreDoc doc = inputs.get(input).getTopDocs().scoreDocs[i];
+                if (testInstancesLookSortedByField) {
+                    doc = new FieldDoc(doc.doc, doc.score, ((FieldDoc) doc).fields, input);
+                } else {
+                    doc = new ScoreDoc(doc.doc, doc.score, input);
+                }
+                allHits.add(new Tuple<>(doc, internalHits.internalHits()[i]));
+            }
+        }
+        allHits.sort(comparing(Tuple::v1, scoreDocComparator()));
+        InternalSearchHit[] expectedHitsHits = new InternalSearchHit[min(inputs.get(0).getSize(), allHits.size())];
+        for (int i = 0; i < expectedHitsHits.length; i++) {
+            expectedHitsHits[i] = allHits.get(i).v2();
+        }
+        InternalSearchHits expectedHits = new InternalSearchHits(expectedHitsHits, totalHits, maxScore);
+        assertEqualsWithErrorMessageFromXContent(expectedHits, actualHits);
+    }
+
+    @Override
+    protected Reader<InternalTopHits> instanceReader() {
+        return InternalTopHits::new;
+    }
+
+    private SortField[] randomSortFields() {
+        SortField[] sortFields = new SortField[between(1, 5)];
+        Set<String> usedSortFields = new HashSet<>();
+        for (int i = 0; i < sortFields.length; i++) {
+            String sortField = randomValueOtherThanMany(usedSortFields::contains, () -> randomAsciiOfLength(5));
+            usedSortFields.add(sortField);
+            SortField.Type type = randomValueOtherThanMany(t -> t == SortField.Type.CUSTOM || t == SortField.Type.REWRITEABLE,
+                    () -> randomFrom(SortField.Type.values()));
+            sortFields[i] = new SortField(sortField, type);
+        }
+        return sortFields;
+    }
+
+    private Comparator<ScoreDoc> scoreDocComparator() {
+        return innerScoreDocComparator().thenComparing(s -> s.shardIndex);
+    }
+    
+    private Comparator<ScoreDoc> innerScoreDocComparator() {
+        if (testInstancesLookSortedByField) {
+            // Values passed to getComparator shouldn't matter
+            @SuppressWarnings("rawtypes")
+            FieldComparator[] comparators = new FieldComparator[testInstancesSortFields.length];
+            for (int i = 0; i < testInstancesSortFields.length; i++) {
+                try {
+                    comparators[i] = testInstancesSortFields[i].getComparator(0, 0);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            return (lhs, rhs) -> {
+                FieldDoc l = (FieldDoc) lhs;
+                FieldDoc r = (FieldDoc) rhs;
+                int i = 0;
+                while (i < l.fields.length) {
+                    @SuppressWarnings("unchecked")
+                    int c = comparators[i].compareValues(l.fields[i], r.fields[i]);
+                    if (c != 0) {
+                        return c;
+                    }
+                    i++;
+                }
+                return 0;
+            };
+        } else {
+            Comparator<ScoreDoc> comparator = comparing(d -> d.score);
+            return comparator.reversed();
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java
new file mode 100644
index 00000000000..43686a1465e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorTests.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.metrics.tophits;
+
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.io.IOException;
+
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
+
+public class TopHitsAggregatorTests extends AggregatorTestCase {
+    public void testTopLevel() throws Exception {
+        Aggregation result;
+        if (randomBoolean()) {
+            result = testCase(new MatchAllDocsQuery(), topHits("_name").sort("string", SortOrder.DESC));
+        } else {
+            Query query = new QueryParser("string", new KeywordAnalyzer()).parse("d^1000 c^100 b^10 a^1");
+            result = testCase(query, topHits("_name"));
+        }
+        SearchHits searchHits = ((TopHits) result).getHits();
+        assertEquals(3L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+        assertEquals("type", searchHits.getAt(0).getType());
+        assertEquals("2", searchHits.getAt(1).getId());
+        assertEquals("type", searchHits.getAt(1).getType());
+        assertEquals("1", searchHits.getAt(2).getId());
+        assertEquals("type", searchHits.getAt(2).getType());
+    }
+
+    public void testNoResults() throws Exception {
+        TopHits result = (TopHits) testCase(new MatchNoDocsQuery(), topHits("_name").sort("string", SortOrder.DESC));
+        SearchHits searchHits = ((TopHits) result).getHits();
+        assertEquals(0L, searchHits.getTotalHits());
+    }
+
+    /**
+     * Tests {@code top_hits} inside of {@code terms}. While not strictly a unit test this is a fairly common way to run {@code top_hits}
+     * and serves as a good example of running {@code top_hits} inside of another aggregation.
+     */
+    public void testInsideTerms() throws Exception {
+        Aggregation result;
+        if (randomBoolean()) {
+            result = testCase(new MatchAllDocsQuery(),
+                    terms("term").field("string")
+                        .subAggregation(topHits("top").sort("string", SortOrder.DESC)));
+        } else {
+            Query query = new QueryParser("string", new KeywordAnalyzer()).parse("d^1000 c^100 b^10 a^1");
+            result = testCase(query,
+                    terms("term").field("string")
+                        .subAggregation(topHits("top")));
+        }
+        Terms terms = (Terms) result;
+
+        // The "a" bucket
+        TopHits hits = (TopHits) terms.getBucketByKey("a").getAggregations().get("top");
+        SearchHits searchHits = (hits).getHits();
+        assertEquals(2L, searchHits.getTotalHits());
+        assertEquals("2", searchHits.getAt(0).getId());
+        assertEquals("1", searchHits.getAt(1).getId());
+
+        // The "b" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("b").getAggregations().get("top")).getHits();
+        assertEquals(2L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+        assertEquals("1", searchHits.getAt(1).getId());
+
+        // The "c" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("c").getAggregations().get("top")).getHits();
+        assertEquals(1L, searchHits.getTotalHits());
+        assertEquals("2", searchHits.getAt(0).getId());
+
+        // The "d" bucket
+        searchHits = ((TopHits) terms.getBucketByKey("d").getAggregations().get("top")).getHits();
+        assertEquals(1L, searchHits.getTotalHits());
+        assertEquals("3", searchHits.getAt(0).getId());
+    }
+
+    private static final MappedFieldType STRING_FIELD_TYPE = new KeywordFieldMapper.KeywordFieldType();
+    static {
+        STRING_FIELD_TYPE.setName("string");
+        STRING_FIELD_TYPE.setHasDocValues(true);
+    }
+
+    private Aggregation testCase(Query query, AggregationBuilder builder) throws IOException {
+        Directory directory = newDirectory();
+        RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
+        iw.addDocument(document("1", "a", "b"));
+        iw.addDocument(document("2", "c", "a"));
+        iw.addDocument(document("3", "b", "d"));
+        iw.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        // We do not use LuceneTestCase.newSearcher because we need a DirectoryReader for "testInsideTerms"
+        IndexSearcher indexSearcher = new IndexSearcher(indexReader);
+
+        Aggregation result = searchAndReduce(indexSearcher, query, builder, STRING_FIELD_TYPE);
+        indexReader.close();
+        directory.close();
+        return result;
+    }
+
+    private Document document(String id, String... stringValues) {
+        Document document = new Document();
+        document.add(new Field(UidFieldMapper.NAME, Uid.createUid("type", id), UidFieldMapper.Defaults.FIELD_TYPE));
+        for (String stringValue : stringValues) {
+            document.add(new Field("string", stringValue, STRING_FIELD_TYPE));
+            document.add(new SortedSetDocValuesField("string", new BytesRef(stringValue)));
+        }
+        return document;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java
new file mode 100644
index 00000000000..ef130655368
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCountTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.valuecount;
+
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.aggregations.InternalAggregationTestCase;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.util.List;
+import java.util.Map;
+
+public class InternalValueCountTests extends InternalAggregationTestCase<InternalValueCount> {
+
+    @Override
+    protected InternalValueCount createTestInstance(String name,
+                                                    List<PipelineAggregator> pipelineAggregators,
+                                                    Map<String, Object> metaData) {
+        return new InternalValueCount(name, randomIntBetween(0, 100), pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected void assertReduced(InternalValueCount reduced, List<InternalValueCount> inputs) {
+        assertEquals(inputs.stream().mapToDouble(InternalValueCount::value).sum(), reduced.getValue(), 0);
+    }
+
+    @Override
+    protected Writeable.Reader<InternalValueCount> instanceReader() {
+        return InternalValueCount::new;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java
new file mode 100644
index 00000000000..bc60524b28d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorTests.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.valuecount;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.FieldValueQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.CheckedConsumer;
+import org.elasticsearch.index.mapper.BooleanFieldMapper;
+import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.IpFieldMapper;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.LatLonPointFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.search.aggregations.AggregatorTestCase;
+import org.elasticsearch.search.aggregations.support.ValueType;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.function.Consumer;
+
+import static java.util.Collections.singleton;
+
+public class ValueCountAggregatorTests extends AggregatorTestCase {
+
+    private static final String FIELD_NAME = "field";
+
+    public void testNoDocs() throws IOException {
+        for (ValueType valueType : ValueType.values()) {
+            testCase(new MatchAllDocsQuery(), valueType, iw -> {
+                // Intentionally not writing any docs
+            }, count -> assertEquals(0L, count.getValue()));
+        }
+    }
+
+    public void testNoMatchingField() throws IOException {
+        testCase(new MatchAllDocsQuery(), ValueType.LONG, iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1)));
+        }, count -> assertEquals(0L, count.getValue()));
+    }
+
+    public void testSomeMatchesSortedNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery(FIELD_NAME), ValueType.NUMERIC, iw -> {
+            iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 7)));
+            iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 1)));
+        }, count -> assertEquals(2L, count.getValue()));
+    }
+
+    public void testSomeMatchesNumericDocValues() throws IOException {
+        testCase(new FieldValueQuery(FIELD_NAME), ValueType.NUMBER, iw -> {
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 7)));
+            iw.addDocument(singleton(new NumericDocValuesField(FIELD_NAME, 1)));
+        }, count -> assertEquals(2L, count.getValue()));
+    }
+
+    public void testQueryFiltering() throws IOException {
+        testCase(IntPoint.newRangeQuery("level", 0, 5), ValueType.STRING, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("level", 0), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 1), new SortedDocValuesField(FIELD_NAME, new BytesRef("bar"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 3), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 5), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 7), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+        }, count -> assertEquals(4L, count.getValue()));
+    }
+
+    public void testQueryFiltersAll() throws IOException {
+        testCase(IntPoint.newRangeQuery("level", -1, 0), ValueType.STRING, iw -> {
+            iw.addDocument(Arrays.asList(new IntPoint("level", 3), new SortedDocValuesField(FIELD_NAME, new BytesRef("foo"))));
+            iw.addDocument(Arrays.asList(new IntPoint("level", 5), new SortedDocValuesField(FIELD_NAME, new BytesRef("baz"))));
+        }, count -> assertEquals(0L, count.getValue()));
+    }
+
+    private void testCase(Query query,
+                          ValueType valueType,
+                          CheckedConsumer<RandomIndexWriter, IOException> indexer,
+                          Consumer<ValueCount> verify) throws IOException {
+
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                indexer.accept(indexWriter);
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                MappedFieldType fieldType = createMappedFieldType(valueType);
+                fieldType.setName(FIELD_NAME);
+                fieldType.setHasDocValues(true);
+
+                ValueCountAggregationBuilder aggregationBuilder = new ValueCountAggregationBuilder("_name", valueType);
+                aggregationBuilder.field(FIELD_NAME);
+
+                try (ValueCountAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
+                    aggregator.preCollection();
+                    indexSearcher.search(query, aggregator);
+                    aggregator.postCollection();
+
+                    verify.accept((ValueCount) aggregator.buildAggregation(0L));
+                }
+            }
+        }
+    }
+
+    private static MappedFieldType createMappedFieldType(ValueType valueType) {
+        switch (valueType) {
+            case BOOLEAN:
+                return new BooleanFieldMapper.BooleanFieldType();
+            case STRING:
+                return new KeywordFieldMapper.KeywordFieldType();
+            case DOUBLE:
+                return new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE);
+            case NUMBER:
+            case NUMERIC:
+            case LONG:
+                return new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+            case DATE:
+                return new DateFieldMapper.Builder("_name").fieldType();
+            case IP:
+                return new IpFieldMapper.Builder("_name").fieldType();
+            case GEOPOINT:
+                return new LatLonPointFieldMapper.Builder("_name").fieldType();
+            default:
+                throw new IllegalArgumentException("Test does not support value type [" + valueType + "]");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
index e76b02a8c95..5b8ea2d5c67 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -480,9 +481,9 @@ public class BucketScriptIT extends ESIntegTestCase {
     public void testStoredScript() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setId("my_script")
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin
-                .setSource(new BytesArray("{ \"script\": \"my_script\" }")));
+                .setContent(new BytesArray("{ \"script\": \"my_script\" }"), XContentType.JSON));
 
         SearchResponse response = client()
                 .prepareSearch("idx")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
index d485a0b6d87..d1f3a31677d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
@@ -46,7 +46,9 @@ public class BucketScriptTests extends BasePipelineAggregationTestCase<BucketScr
             if (randomBoolean()) {
                 params.put("foo", "bar");
             }
-            script = new Script(randomFrom(ScriptType.values()), randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), "script", params);
+            ScriptType type = randomFrom(ScriptType.values());
+            script = new Script(type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG),
+                "script", params);
         }
         BucketScriptPipelineAggregationBuilder factory = new BucketScriptPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
index 67261ee02a4..68904c94ca3 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -432,9 +433,10 @@ public class BucketSelectorIT extends ESIntegTestCase {
     public void testStoredScript() {
         assertAcked(client().admin().cluster().preparePutStoredScript()
                 .setId("my_script")
-                .setScriptLang(CustomScriptPlugin.NAME)
+                .setLang(CustomScriptPlugin.NAME)
                 // Source is not interpreted but my_script is defined in CustomScriptPlugin
-                .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }")));
+                .setContent(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }"),
+                    XContentType.JSON));
 
         Script script = new Script(ScriptType.STORED, CustomScriptPlugin.NAME, "my_script", Collections.emptyMap());
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
index 563894906ed..0311781455a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
@@ -46,7 +46,9 @@ public class BucketSelectorTests extends BasePipelineAggregationTestCase<BucketS
             if (randomBoolean()) {
                 params.put("foo", "bar");
             }
-            script = new Script(randomFrom(ScriptType.values()), randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), "script", params);
+            ScriptType type = randomFrom(ScriptType.values());
+            script =
+                new Script(type, type == ScriptType.STORED ? null : randomFrom("my_lang", Script.DEFAULT_SCRIPT_LANG), "script", params);
         }
         BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
index 670ab0dabde..b77c0bdd972 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
@@ -19,10 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -39,20 +37,18 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<Exte
     }
 
     public void testSigmaFromInt() throws Exception {
-        String content = XContentFactory.jsonBuilder()
+        XContentBuilder content = XContentFactory.jsonBuilder()
             .startObject()
-                .field("sigma", 5)
-                .field("buckets_path", "test")
-            .endObject()
-            .string();
+                .startObject("name")
+                    .startObject("extended_stats_bucket")
+                        .field("sigma", 5)
+                        .field("buckets_path", "test")
+                    .endObject()
+                .endObject()
+            .endObject();
 
-        XContentParser parser = createParser(JsonXContent.jsonXContent, content);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        parser.nextToken(); // skip object start
-
-        ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) aggParsers
-            .pipelineParser(ExtendedStatsBucketPipelineAggregationBuilder.NAME, parseFieldMatcher)
-            .parse("test", parseContext);
+        ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) parse(
+                createParser(content));
 
         assertThat(builder.sigma(), equalTo(5.0));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
index 26ee43e7a11..b04b5743c68 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
@@ -19,10 +19,8 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -44,20 +42,17 @@ public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<Percen
     }
 
     public void testPercentsFromMixedArray() throws Exception {
-        String content = XContentFactory.jsonBuilder()
+        XContentBuilder content = XContentFactory.jsonBuilder()
             .startObject()
-                .field("buckets_path", "test")
-                .array("percents", 0, 20.0, 50, 75.99)
-            .endObject()
-            .string();
+                .startObject("name")
+                    .startObject("percentiles_bucket")
+                        .field("buckets_path", "test")
+                        .array("percents", 0, 20.0, 50, 75.99)
+                    .endObject()
+                .endObject()
+            .endObject();
 
-        XContentParser parser = createParser(JsonXContent.jsonXContent, content);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        parser.nextToken(); // skip object start
-
-        PercentilesBucketPipelineAggregationBuilder builder = (PercentilesBucketPipelineAggregationBuilder) aggParsers
-            .pipelineParser(PercentilesBucketPipelineAggregationBuilder.NAME, parseFieldMatcher)
-            .parse("test", parseContext);
+        PercentilesBucketPipelineAggregationBuilder builder = (PercentilesBucketPipelineAggregationBuilder) parse(createParser(content));
 
         assertThat(builder.percents(), equalTo(new double[]{0.0, 20.0, 50.0, 75.99}));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
index 7b29ae213ff..cb9d99e07cd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
@@ -19,10 +19,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline.moving.avg;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
@@ -32,7 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearMo
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel.SeasonalityType;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
-import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;;
+import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 
 public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineAggregationBuilder> {
 
@@ -102,26 +99,11 @@ public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineA
         String json = "{" +
             "    \"commits_moving_avg\": {" +
             "        \"moving_avg\": {" +
-                "            \"buckets_path\": \"commits\"" +
+            "            \"buckets_path\": \"commits\"" +
             "        }" +
             "    }" +
             "}";
-        XContentParser parser = createParser(JsonXContent.jsonXContent, json);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(expected.getName(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(expected.type(), parser.currentName());
-        assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(expected.getName(), parseContext);
-        assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
-        assertNull(parser.nextToken());
-        assertNotNull(newAgg);
+        PipelineAggregationBuilder newAgg = parse(createParser(JsonXContent.jsonXContent, json));
         assertNotSame(newAgg, expected);
         assertEquals(expected, newAgg);
         assertEquals(expected.hashCode(), newAgg.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
index 0f94a4cbc06..34d20344360 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgUnitTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.pipeline.moving.avg;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.collect.EvictingQueue;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
@@ -605,7 +604,7 @@ public class MovAvgUnitTests extends ESTestCase {
                 settings.put("alpha", v);
 
                 try {
-                    parser.parse(settings, "pipeline", 10, ParseFieldMatcher.STRICT);
+                    parser.parse(settings, "pipeline", 10);
                 } catch (ParseException e) {
                     fail(parser + " parser should not have thrown SearchParseException while parsing [" +
                             v.getClass().getSimpleName() +"]");
@@ -619,7 +618,7 @@ public class MovAvgUnitTests extends ESTestCase {
             settings.put("gamma", "abc");
 
             try {
-                parser.parse(settings, "pipeline", 10, ParseFieldMatcher.STRICT);
+                parser.parse(settings, "pipeline", 10);
             } catch (ParseException e) {
                 //all good
                 continue;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
index 20f38297d9e..42f0df4beb9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java
@@ -25,14 +25,12 @@ import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LongBitSet;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude.OrdinalsFilter;
@@ -43,10 +41,6 @@ import java.util.Collections;
 import java.util.TreeSet;
 
 public class IncludeExcludeTests extends ESTestCase {
-    
-    private final ParseFieldMatcher parseFieldMatcher = ParseFieldMatcher.STRICT;
-    private final IndicesQueriesRegistry queriesRegistry = new IndicesQueriesRegistry();    
-
     public void testEmptyTermsWithOrds() throws IOException {
         IncludeExclude inexcl = new IncludeExclude(
                 new TreeSet<>(Collections.singleton(new BytesRef("foo"))),
@@ -237,7 +231,7 @@ public class IncludeExcludeTests extends ESTestCase {
         assertEquals(field.getPreferredName(), parser.currentName());
         token = parser.nextToken();
 
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         if (field.getPreferredName().equalsIgnoreCase("include")) {
             return IncludeExclude.parseInclude(parser, parseContext);
         } else if (field.getPreferredName().equalsIgnoreCase("exclude")) {
@@ -277,7 +271,7 @@ public class IncludeExcludeTests extends ESTestCase {
         builder.endObject();
 
         XContentParser parser = createParser(builder);
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         XContentParser.Token token = parser.nextToken();
         assertEquals(token, XContentParser.Token.START_OBJECT);
 
@@ -285,10 +279,10 @@ public class IncludeExcludeTests extends ESTestCase {
         IncludeExclude exc = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             assertEquals(XContentParser.Token.FIELD_NAME, token);
-            if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.INCLUDE_FIELD)) {
+            if (IncludeExclude.INCLUDE_FIELD.match(parser.currentName())) {
                 token = parser.nextToken();
                 inc = IncludeExclude.parseInclude(parser, parseContext);
-            } else if (parseFieldMatcher.match(parser.currentName(), IncludeExclude.EXCLUDE_FIELD)) {
+            } else if (IncludeExclude.EXCLUDE_FIELD.match(parser.currentName())) {
                 token = parser.nextToken();
                 exc = IncludeExclude.parseExclude(parser, parseContext);
             } else {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
index 3b4e84f8e7c..2c819a671e5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java
@@ -293,4 +293,9 @@ public class MissingValuesTests extends ESTestCase {
             }
         }
     }
+
+    public void testFloatingPointDetection() {
+        assertFalse(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3).isFloatingPoint());
+        assertTrue(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3.5).isFloatingPoint());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
index dcf6cfdb0ba..f338e961312 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
@@ -108,6 +108,27 @@ public class ScriptValuesTests extends ESTestCase {
         }
     }
 
+    public void testBooleans() {
+        final Object[][] values = new Boolean[randomInt(10)][];
+        for (int i = 0; i < values.length; ++i) {
+            Boolean[] booleans = new Boolean[randomInt(8)];
+            for (int j = 0; j < booleans.length; ++j) {
+                booleans[j] = randomBoolean();
+            }
+            Arrays.sort(booleans);
+            values[i] = booleans;
+        }
+        FakeSearchScript script = new FakeSearchScript(values);
+        ScriptLongValues scriptValues = new ScriptLongValues(script);
+        for (int i = 0; i < values.length; ++i) {
+            scriptValues.setDocument(i);
+            assertEquals(values[i].length, scriptValues.count());
+            for (int j = 0; j < values[i].length; ++j) {
+                assertEquals(values[i][j], scriptValues.valueAt(j) == 1L);
+            }
+        }
+    }
+
     public void testDoubles() {
         final Object[][] values = new Double[randomInt(10)][];
         for (int i = 0; i < values.length; ++i) {
diff --git a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
index c201f03a7d8..1d2eec418d2 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/TransportTwoNodesSearchIT.java
@@ -48,7 +48,6 @@ import java.util.HashSet;
 import java.util.Set;
 import java.util.TreeSet;
 
-import static org.elasticsearch.action.search.SearchType.DFS_QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
@@ -319,58 +318,6 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
         assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0));
     }
 
-    public void testDfsQueryAndFetch() throws Exception {
-        prepareData(3);
-
-        SearchSourceBuilder source = searchSource()
-                .query(termQuery("multi", "test"))
-                .from(0).size(20).explain(true);
-
-        Set<String> expectedIds = new HashSet<>();
-        for (int i = 0; i < 100; i++) {
-            expectedIds.add(Integer.toString(i));
-        }
-
-
-        //SearchResponse searchResponse = client().search(searchRequest("test").source(source).searchType(DFS_QUERY_AND_FETCH).scroll(new Scroll(timeValueMinutes(10)))).actionGet();
-        SearchResponse searchResponse = client().prepareSearch("test").setSearchType(DFS_QUERY_AND_FETCH).setScroll("10m").setSource(source).get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(60)); // 20 per shard
-        for (int i = 0; i < 60; i++) {
-            SearchHit hit = searchResponse.getHits().hits()[i];
-//            System.out.println(hit.shard() + ": " +  hit.explanation());
-            assertThat(hit.explanation(), notNullValue());
-            assertThat(hit.explanation().getDetails().length, equalTo(1));
-            assertThat(hit.explanation().getDetails()[0].getDetails().length, equalTo(2));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails().length, equalTo(2));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getDescription(),
-                equalTo("docFreq"));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[0].getValue(),
-                equalTo(100.0f));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getDescription(),
-                equalTo("docCount"));
-            assertThat(hit.explanation().getDetails()[0].getDetails()[0].getDetails()[1].getValue(),
-                equalTo(100.0f));
-//            assertThat("id[" + hit.id() + "]", hit.id(), equalTo(Integer.toString(100 - i - 1)));
-            assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-        }
-
-        do {
-            searchResponse = client().prepareSearchScroll(searchResponse.getScrollId()).setScroll("10m").get();
-
-            assertThat(searchResponse.getHits().totalHits(), equalTo(100L));
-            assertThat(searchResponse.getHits().hits().length, lessThanOrEqualTo(40));
-            for (int i = 0; i < searchResponse.getHits().hits().length; i++) {
-                SearchHit hit = searchResponse.getHits().hits()[i];
-                // we don't do perfect sorting when it comes to scroll with Query+Fetch
-                assertThat("make sure we don't have duplicates", expectedIds.remove(hit.id()), notNullValue());
-            }
-        } while (searchResponse.getHits().hits().length > 0);
-        clearScroll(searchResponse.getScrollId());
-        assertThat("make sure we got all [" + expectedIds + "]", expectedIds.size(), equalTo(0));
-    }
-
     public void testSimpleFacets() throws Exception {
         prepareData();
 
@@ -421,7 +368,7 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
         SearchSourceBuilder source = searchSource()
                 .query(termQuery("multi", "test"))
                 .from(1000).size(20).explain(true);
-        SearchResponse response = client().search(searchRequest("test").searchType(DFS_QUERY_AND_FETCH).source(source)).actionGet();
+        SearchResponse response = client().search(searchRequest("test").searchType(DFS_QUERY_THEN_FETCH).source(source)).actionGet();
         assertThat(response.getHits().hits().length, equalTo(0));
         assertThat(response.getTotalShards(), equalTo(test.numPrimaries));
         assertThat(response.getSuccessfulShards(), equalTo(test.numPrimaries));
@@ -431,7 +378,7 @@ public class TransportTwoNodesSearchIT extends ESIntegTestCase {
         assertNoFailures(response);
         assertThat(response.getHits().hits().length, equalTo(0));
 
-        response = client().search(searchRequest("test").searchType(DFS_QUERY_AND_FETCH).source(source)).actionGet();
+        response = client().search(searchRequest("test").searchType(DFS_QUERY_THEN_FETCH).source(source)).actionGet();
         assertNoFailures(response);
         assertThat(response.getHits().hits().length, equalTo(0));
 
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 8ec873704d6..ef1750a448c 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.builder;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -65,25 +64,19 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
     }
 
     private void assertParseSearchSource(SearchSourceBuilder testBuilder, XContentParser parser) throws IOException {
-        assertParseSearchSource(testBuilder, parser, ParseFieldMatcher.STRICT);
-    }
-
-    private void assertParseSearchSource(SearchSourceBuilder testBuilder, XContentParser parser, ParseFieldMatcher pfm)
-            throws IOException {
-        QueryParseContext parseContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, pfm);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         if (randomBoolean()) {
             parser.nextToken(); // sometimes we move it on the START_OBJECT to
                                 // test the embedded case
         }
-        SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext, searchRequestParsers.aggParsers,
-            searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+        SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext);
         assertNull(parser.nextToken());
         assertEquals(testBuilder, newBuilder);
         assertEquals(testBuilder.hashCode(), newBuilder.hashCode());
     }
 
     private QueryParseContext createParseContext(XContentParser parser) {
-        return new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT);
+        return new QueryParseContext(parser);
     }
 
     public void testSerialization() throws IOException {
@@ -113,8 +106,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
         {
             String restContent = " { \"_source\": { \"includes\": \"include\", \"excludes\": \"*.field2\"}}";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                        searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertArrayEquals(new String[]{"*.field2"}, searchSourceBuilder.fetchSource().excludes());
                 assertArrayEquals(new String[]{"include"}, searchSourceBuilder.fetchSource().includes());
             }
@@ -122,8 +114,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
         {
             String restContent = " { \"_source\": false}";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().excludes());
                 assertArrayEquals(new String[]{}, searchSourceBuilder.fetchSource().includes());
                 assertFalse(searchSourceBuilder.fetchSource().fetchSource());
@@ -145,8 +136,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                 "    }\n" +
                 "  } }";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers));
+            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser)));
             assertEquals("[multi_match] malformed query, expected [END_OBJECT] but found [FIELD_NAME]", e.getMessage());
         }
     }
@@ -155,8 +145,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
         {
             String restContent = " { \"sort\": \"foo\"}";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.sorts().size());
                 assertEquals(new FieldSortBuilder("foo"), searchSourceBuilder.sorts().get(0));
             }
@@ -171,8 +160,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                     "        \"_score\"\n" +
                     "    ]}";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(5, searchSourceBuilder.sorts().size());
                 assertEquals(new FieldSortBuilder("post_date"), searchSourceBuilder.sorts().get(0));
                 assertEquals(new FieldSortBuilder("user"), searchSourceBuilder.sorts().get(1));
@@ -195,8 +183,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                     "    }\n" +
                     "}\n";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.aggregations().count());
             }
         }
@@ -211,8 +198,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                     "    }\n" +
                     "}\n";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.aggregations().count());
             }
         }
@@ -237,8 +223,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                 "    }\n" +
                 "}\n";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.rescores().size());
                 assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
                         searchSourceBuilder.rescores().get(0));
@@ -260,8 +245,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                 "    } ]\n" +
                 "}\n";
             try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(1, searchSourceBuilder.rescores().size());
                 assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
                         searchSourceBuilder.rescores().get(0));
@@ -273,8 +257,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
         final String timeout = randomTimeValue();
         final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
-            final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+            final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser));
             assertThat(builder.timeout(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout")));
         }
     }
@@ -283,11 +266,8 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
         final int timeout = randomIntBetween(1, 1024);
         final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, query)) {
-            final ElasticsearchParseException e =
-                    expectThrows(
-                            ElasticsearchParseException.class,
-                            () -> SearchSourceBuilder.fromXContent(createParseContext(parser),
-                                searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers));
+            final ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> SearchSourceBuilder.fromXContent(
+                    createParseContext(parser)));
             assertThat(e, hasToString(containsString("unit is missing or unrecognized")));
         }
     }
@@ -300,7 +280,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
             XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
             searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
             BytesReference source = builder.bytes();
-            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false).v2();
+            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2();
             assertEquals(0, sourceAsMap.size());
         }
         {
@@ -309,7 +289,7 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
             XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
             searchSourceBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
             BytesReference source = builder.bytes();
-            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false).v2();
+            Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false, xContentType).v2();
             assertEquals(1, sourceAsMap.size());
             assertEquals("query", sourceAsMap.keySet().iterator().next());
         }
@@ -318,12 +298,12 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
     public void testParseIndicesBoost() throws IOException {
         {
             String restContent = " { \"indices_boost\": {\"foo\": 1.0, \"bar\": 2.0}}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(2, searchSourceBuilder.indexBoosts().size());
                 assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
                 assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
+                assertWarnings("Object format in indices_boost is deprecated, please use array format instead");
             }
         }
 
@@ -334,9 +314,8 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
                 "        { \"bar\" : 2.0 },\n" +
                 "        { \"baz\" : 3.0 }\n" +
                 "    ]}";
-            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser));
                 assertEquals(3, searchSourceBuilder.indexBoosts().size());
                 assertEquals(new SearchSourceBuilder.IndexBoost("foo", 1.0f), searchSourceBuilder.indexBoosts().get(0));
                 assertEquals(new SearchSourceBuilder.IndexBoost("bar", 2.0f), searchSourceBuilder.indexBoosts().get(1));
@@ -382,9 +361,8 @@ public class SearchSourceBuilderTests extends AbstractSearchTestCase {
     }
 
     private void assertIndicesBoostParseErrorMessage(String restContent, String expectedErrorMessage) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser),
-                searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers));
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(createParseContext(parser)));
             assertEquals(expectedErrorMessage, e.getMessage());
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index a3ecc66c030..78cb846932f 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.search.function.CombineFunction;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.HasChildQueryBuilder;
@@ -94,7 +95,6 @@ import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -111,14 +111,9 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
     }
 
     public void testSelfReferentialIsForbidden() {
-        try {
-            prepareCreate("test").addMapping("type", "_parent", "type=type").get();
-            fail("self referential should be forbidden");
-        } catch (Exception e) {
-            Throwable cause = e.getCause();
-            assertThat(cause, instanceOf(IllegalArgumentException.class));
-            assertThat(cause.getMessage(), equalTo("The [_parent.type] option can't point to the same type"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            prepareCreate("test").addMapping("type", "_parent", "type=type").get());
+        assertThat(e.getMessage(), equalTo("The [_parent.type] option can't point to the same type"));
     }
 
     public void testMultiLevelChild() throws Exception {
@@ -827,7 +822,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch("test").setQuery(
                 hasChildQuery("child", matchQuery("c_field", "foo"), ScoreMode.None)
                     .innerHit(new InnerHitBuilder().setHighlightBuilder(
-                        new HighlightBuilder().field(new Field("c_field").highlightQuery(QueryBuilders.matchQuery("c_field", "bar"))))))
+                        new HighlightBuilder().field(new Field("c_field")
+                                .highlightQuery(QueryBuilders.matchQuery("c_field", "bar")))), false))
                 .get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -1014,7 +1010,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
                 .get();
         assertHitCount(response, 0L);
 
-        client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get();
+        client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get();
         refresh();
 
         response = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).get();
@@ -1023,7 +1019,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).get();
         assertHitCount(response, 1L);
 
-        client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get();
+        client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get();
         refresh();
         response = client().prepareSearch("test").setQuery(termsQuery("_parent#parent", "p1", "p2")).get();
         assertHitCount(response, 2L);
@@ -1044,13 +1040,13 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
             .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
 
-        client().prepareIndex("test", "child", "c1").setSource("{}").setParent("p1").get();
+        client().prepareIndex("test", "child", "c1").setSource("{}", XContentType.JSON).setParent("p1").get();
         refresh();
 
         SearchResponse response = client().prepareSearch("test").setQuery(parentId("child", "p1")).get();
         assertHitCount(response, 1L);
 
-        client().prepareIndex("test", "child", "c2").setSource("{}").setParent("p2").get();
+        client().prepareIndex("test", "child", "c2").setSource("{}", XContentType.JSON).setParent("p2").get();
         refresh();
 
         response = client().prepareSearch("test")
@@ -1410,8 +1406,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
                 .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
         for (int i = 0; i < 10; i++) {
-            client().prepareIndex("test", "parent", "p" + i).setSource("{}").get();
-            client().prepareIndex("test", "child", "c" + i).setSource("{}").setParent("p" + i).get();
+            client().prepareIndex("test", "parent", "p" + i).setSource("{}", XContentType.JSON).get();
+            client().prepareIndex("test", "child", "c" + i).setSource("{}", XContentType.JSON).setParent("p" + i).get();
         }
 
         refresh();
@@ -1493,8 +1489,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
 
         List<IndexRequestBuilder> indexRequests = new ArrayList<>();
         indexRequests.add(client().prepareIndex("test", "parent", "1").setSource("field1", "a"));
-        indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}"));
-        indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}"));
+        indexRequests.add(client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON));
+        indexRequests.add(client().prepareIndex("test", "child", "2").setParent("1").setSource("{}", XContentType.JSON));
         indexRandom(true, indexRequests);
 
         SearchResponse searchResponse = client().prepareSearch("test")
@@ -1888,8 +1884,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
 
     public void testParentFieldToNonExistingType() {
         assertAcked(prepareCreate("test").addMapping("parent").addMapping("child", "_parent", "type=parent2"));
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         try {
@@ -1903,8 +1899,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
 
     public void testHasParentInnerQueryType() {
         assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type"));
-        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get();
-        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get();
+        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get();
         refresh();
         //make sure that when we explicitly set a type, the inner query is executed in the context of the parent type instead
         SearchResponse searchResponse = client().prepareSearch("test").setTypes("child-type").setQuery(
@@ -1914,8 +1910,8 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
 
     public void testHasChildInnerQueryType() {
         assertAcked(prepareCreate("test").addMapping("parent-type").addMapping("child-type", "_parent", "type=parent-type"));
-        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}").get();
-        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}").get();
+        client().prepareIndex("test", "child-type", "child-id").setParent("parent-id").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "parent-type", "parent-id").setSource("{}", XContentType.JSON).get();
         refresh();
         //make sure that when we explicitly set a type, the inner query is executed in the context of the child type instead
         SearchResponse searchResponse = client().prepareSearch("test").setTypes("parent-type").setQuery(
diff --git a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
index 5aead911caa..44b01b3a4b9 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ParentFieldLoadingIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
@@ -69,8 +70,8 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
                 .addMapping("child", childMapping(false)));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
@@ -84,8 +85,8 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
                 .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         response = client().admin().cluster().prepareClusterStats().get();
@@ -101,9 +102,9 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
 
         // Need to do 2 separate refreshes, otherwise we have 1 segment and then we can't measure if global ordinals
         // is loaded by the size of the field data cache, because global ordinals on 1 segment shards takes no extra memory.
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
         refresh();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         response = client().admin().cluster().prepareClusterStats().get();
@@ -117,8 +118,8 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
                 .addMapping("child", "_parent", "type=parent"));
         ensureGreen();
 
-        client().prepareIndex("test", "parent", "1").setSource("{}").get();
-        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}").get();
+        client().prepareIndex("test", "parent", "1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("test", "child", "1").setParent("1").setSource("{}", XContentType.JSON).get();
         refresh();
 
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
@@ -153,7 +154,7 @@ public class ParentFieldLoadingIT extends ESIntegTestCase {
 
         // Need to add a new doc otherwise the refresh doesn't trigger a new searcher
         // Because it ends up in its own segment, but isn't of type parent or child, this doc doesn't contribute to the size of the fielddata cache
-        client().prepareIndex("test", "dummy", "dummy").setSource("{}").get();
+        client().prepareIndex("test", "dummy", "dummy").setSource("{}", XContentType.JSON).get();
         refresh();
         response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getIndicesStats().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
diff --git a/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
new file mode 100644
index 00000000000..955b85deb82
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.collapse;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.index.mapper.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitBuilderTests;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.SearchContextException;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class CollapseBuilderTests extends AbstractWireSerializingTestCase {
+    private static NamedWriteableRegistry namedWriteableRegistry;
+    private static NamedXContentRegistry xContentRegistry;
+
+    @BeforeClass
+    public static void init() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
+        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        namedWriteableRegistry = null;
+        xContentRegistry = null;
+    }
+
+    public static CollapseBuilder randomCollapseBuilder() {
+        CollapseBuilder builder = new CollapseBuilder(randomAsciiOfLength(10));
+        if (randomBoolean()) {
+            InnerHitBuilder innerHit = InnerHitBuilderTests.randomInnerHits(false, false);
+            builder.setInnerHits(innerHit);
+        }
+        return builder;
+    }
+
+    @Override
+    protected Writeable createTestInstance() {
+        return randomCollapseBuilder();
+    }
+
+    @Override
+    protected Writeable.Reader<CollapseBuilder> instanceReader() {
+        return CollapseBuilder::new;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return namedWriteableRegistry;
+    }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
+    private SearchContext mockSearchContext() {
+        SearchContext context = mock(SearchContext.class);
+        QueryShardContext shardContext = mock(QueryShardContext.class);
+        when(context.getQueryShardContext()).thenReturn(shardContext);
+        when(context.scrollContext()).thenReturn(null);
+        when(context.rescore()).thenReturn(null);
+        when(context.searchAfter()).thenReturn(null);
+        return context;
+    }
+
+    public void testBuild() throws IOException {
+        Directory dir = new RAMDirectory();
+        try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())))) {
+            writer.commit();
+        }
+        SearchContext searchContext = mockSearchContext();
+        try (IndexReader reader = DirectoryReader.open(dir)) {
+            when(searchContext.getQueryShardContext().getIndexReader()).thenReturn(reader);
+            MappedFieldType numberFieldType =
+                new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
+            MappedFieldType keywordFieldType =
+                new KeywordFieldMapper.KeywordFieldType();
+            for (MappedFieldType fieldType : new MappedFieldType[] {numberFieldType, keywordFieldType}) {
+                fieldType.setName("field");
+                fieldType.setHasDocValues(true);
+                when(searchContext.getQueryShardContext().fieldMapper("field")).thenReturn(fieldType);
+                CollapseBuilder builder = new CollapseBuilder("field");
+                CollapseContext collapseContext = builder.build(searchContext);
+                assertEquals(collapseContext.getFieldType(), fieldType);
+
+                fieldType.setIndexOptions(IndexOptions.NONE);
+                collapseContext = builder.build(searchContext);
+                assertEquals(collapseContext.getFieldType(), fieldType);
+
+                fieldType.setHasDocValues(false);
+                SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
+                assertEquals(exc.getMessage(), "cannot collapse on field `field` without `doc_values`");
+
+                fieldType.setHasDocValues(true);
+                builder.setInnerHits(new InnerHitBuilder());
+                exc = expectThrows(SearchContextException.class, () -> builder.build(searchContext));
+                assertEquals(exc.getMessage(),
+                    "cannot expand `inner_hits` for collapse field `field`, " +
+                        "only indexed field can retrieve `inner_hits`");
+            }
+        }
+    }
+
+    public void testBuildWithSearchContextExceptions() throws IOException {
+        SearchContext context = mockSearchContext();
+        {
+            CollapseBuilder builder = new CollapseBuilder("unknown_field");
+            SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(context));
+            assertEquals(exc.getMessage(), "no mapping found for `unknown_field` in order to collapse on");
+        }
+
+        {
+            MappedFieldType fieldType = new MappedFieldType() {
+                @Override
+                public MappedFieldType clone() {
+                    return null;
+                }
+
+                @Override
+                public String typeName() {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, QueryShardContext context) {
+                    return null;
+                }
+            };
+            fieldType.setName("field");
+            fieldType.setHasDocValues(true);
+            when(context.getQueryShardContext().fieldMapper("field")).thenReturn(fieldType);
+            CollapseBuilder builder = new CollapseBuilder("field");
+            SearchContextException exc = expectThrows(SearchContextException.class, () -> builder.build(context));
+            assertEquals(exc.getMessage(), "unknown type for collapse field `field`, only keywords and numbers are accepted");
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 02def44416e..58170fe04a8 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.termvectors.TermVectorsService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.SearchExtBuilder;
-import org.elasticsearch.search.SearchExtParser;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.InternalSearchHitField;
@@ -114,7 +113,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
         @Override
         public List<SearchExtSpec<?>> getSearchExts() {
             return Collections.singletonList(new SearchExtSpec<>(TermVectorsFetchSubPhase.NAME,
-                    TermVectorsFetchBuilder::new, TermVectorsFetchParser.INSTANCE));
+                    TermVectorsFetchBuilder::new, TermVectorsFetchBuilder::fromXContent));
         }
     }
 
@@ -153,15 +152,8 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
         }
     }
 
-    private static final class TermVectorsFetchParser implements SearchExtParser<TermVectorsFetchBuilder> {
-
-        private static final TermVectorsFetchParser INSTANCE = new TermVectorsFetchParser();
-
-        private TermVectorsFetchParser() {
-        }
-
-        @Override
-        public TermVectorsFetchBuilder fromXContent(XContentParser parser) throws IOException {
+    private static final class TermVectorsFetchBuilder extends SearchExtBuilder {
+        public static TermVectorsFetchBuilder fromXContent(XContentParser parser) throws IOException {
             String field;
             XContentParser.Token token = parser.currentToken();
             if (token == XContentParser.Token.VALUE_STRING) {
@@ -174,9 +166,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
             }
             return new TermVectorsFetchBuilder(field);
         }
-    }
 
-    private static final class TermVectorsFetchBuilder extends SearchExtBuilder {
         private final String field;
 
         private TermVectorsFetchBuilder(String field) {
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
index 809591cd688..1d24a1f16d5 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -123,7 +124,7 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         SearchResponse response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder().setName("comment"))
+                        .innerHit(new InnerHitBuilder().setName("comment"), false)
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -141,7 +142,7 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder().setName("comment"))
+                        .innerHit(new InnerHitBuilder().setName("comment"), false)
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -168,8 +169,8 @@ public class InnerHitsIT extends ESIntegTestCase {
                                 .addDocValueField("comments.message")
                                 .addScriptField("script",
                                         new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
-                                .setSize(1)
-                )).get();
+                                .setSize(1),
+                        false)).get();
         assertNoFailures(response);
         innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
         assertThat(innerHits.getTotalHits(), equalTo(2L));
@@ -207,9 +208,9 @@ public class InnerHitsIT extends ESIntegTestCase {
         int size = randomIntBetween(0, numDocs);
         BoolQueryBuilder boolQuery = new BoolQueryBuilder();
         boolQuery.should(nestedQuery("field1", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("a").setSize(size)
-                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC))));
+                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)), false));
         boolQuery.should(nestedQuery("field2", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("b")
-                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size)));
+                .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size), false));
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(boolQuery)
                 .setSize(numDocs)
@@ -260,7 +261,8 @@ public class InnerHitsIT extends ESIntegTestCase {
         indexRandom(true, requests);
 
         SearchResponse response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder()))
+                .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None)
+                        .innerHit(new InnerHitBuilder(), false))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -277,7 +279,8 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertThat(innerHits.getAt(1).type(), equalTo("comment"));
 
         response = client().prepareSearch("articles")
-                .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None).innerHit(new InnerHitBuilder()))
+                .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None)
+                        .innerHit(new InnerHitBuilder(), false))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -302,8 +305,8 @@ public class InnerHitsIT extends ESIntegTestCase {
                                         .setHighlightBuilder(new HighlightBuilder().field("message"))
                                         .setExplain(true).setSize(1)
                                         .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
-                                            Collections.emptyMap()))
-                        )
+                                            Collections.emptyMap())),
+                                false)
                 ).get();
         assertNoFailures(response);
         innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
@@ -329,19 +332,19 @@ public class InnerHitsIT extends ESIntegTestCase {
         int[] child2InnerObjects = new int[numDocs];
         for (int parent = 0; parent < numDocs; parent++) {
             String parentId = String.format(Locale.ENGLISH, "%03d", parent);
-            requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}"));
+            requestBuilders.add(client().prepareIndex("idx", "parent", parentId).setSource("{}", XContentType.JSON));
 
             int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
             int limit = child1 + numChildDocs;
             for (; child1 < limit; child1++) {
                 requestBuilders.add(client().prepareIndex("idx", "child1",
-                        String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}"));
+                        String.format(Locale.ENGLISH, "%04d", child1)).setParent(parentId).setSource("{}", XContentType.JSON));
             }
             numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
             limit = child2 + numChildDocs;
             for (; child2 < limit; child2++) {
                 requestBuilders.add(client().prepareIndex("idx", "child2",
-                        String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}"));
+                        String.format(Locale.ENGLISH, "%04d", child2)).setParent(parentId).setSource("{}", XContentType.JSON));
             }
         }
         indexRandom(true, requestBuilders);
@@ -349,9 +352,11 @@ public class InnerHitsIT extends ESIntegTestCase {
         int size = randomIntBetween(0, numDocs);
         BoolQueryBuilder boolQuery = new BoolQueryBuilder();
         boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setName("a").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
+                .innerHit(new InnerHitBuilder().setName("a")
+                        .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size), false)));
         boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setName("b").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size))));
+                .innerHit(new InnerHitBuilder().setName("b")
+                        .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size), false)));
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setSize(numDocs)
                 .setTypes("parent")
@@ -417,7 +422,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         boolQuery()
                                 .must(matchQuery("body", "fail2ban"))
-                                .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder()))
+                                .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder(), false))
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 2);
@@ -455,8 +460,8 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         SearchResponse response = client().prepareSearch("articles")
                 .setQuery(hasChildQuery("comment",
-                            hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder()),
-                        ScoreMode.None).innerHit(new InnerHitBuilder()))
+                            hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder(), false),
+                        ScoreMode.None).innerHit(new InnerHitBuilder(), false))
                 .get();
 
         assertNoFailures(response);
@@ -476,8 +481,8 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         response = client().prepareSearch("articles")
                 .setQuery(hasChildQuery("comment",
-                        hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder()),
-                        ScoreMode.None).innerHit(new InnerHitBuilder()))
+                        hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder(), false),
+                        ScoreMode.None).innerHit(new InnerHitBuilder(), false))
                 .get();
 
         assertNoFailures(response);
@@ -543,8 +548,8 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         nestedQuery("comments",
                                 nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good"), ScoreMode.Avg)
-                                        .innerHit(new InnerHitBuilder().setName("remark")),
-                                ScoreMode.Avg).innerHit(new InnerHitBuilder())
+                                        .innerHit(new InnerHitBuilder().setName("remark"), false),
+                                ScoreMode.Avg).innerHit(new InnerHitBuilder(), false)
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -568,7 +573,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         // Directly refer to the second level:
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder(), false)).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         assertSearchHit(response, 1, hasId("2"));
@@ -586,8 +591,8 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         nestedQuery("comments",
                                 nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"), ScoreMode.Avg)
-                                        .innerHit(new InnerHitBuilder().setName("remark")),
-                                ScoreMode.Avg).innerHit(new InnerHitBuilder())
+                                        .innerHit(new InnerHitBuilder().setName("remark"), false),
+                                ScoreMode.Avg).innerHit(new InnerHitBuilder(), false)
                 ).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -621,7 +626,8 @@ public class InnerHitsIT extends ESIntegTestCase {
         indexRandom(true, requests);
 
         SearchResponse response = client().prepareSearch("articles")
-                .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder()))
+                .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg)
+                        .innerHit(new InnerHitBuilder(), false))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -663,7 +669,7 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         SearchResponse response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder(), false)).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         SearchHit hit = response.getHits().getAt(0);
@@ -677,7 +683,7 @@ public class InnerHitsIT extends ESIntegTestCase {
 
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "bear"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder(), false)).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         hit = response.getHits().getAt(0);
@@ -698,7 +704,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         indexRandom(true, requests);
         response = client().prepareSearch("articles")
                 .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg)
-                        .innerHit(new InnerHitBuilder())).get();
+                        .innerHit(new InnerHitBuilder(), false)).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
         hit = response.getHits().getAt(0);;
@@ -722,32 +728,42 @@ public class InnerHitsIT extends ESIntegTestCase {
         );
 
         List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("royals", "king", "king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king").setSource("{}"));
-        requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king").setSource("{}"));
+        requests.add(client().prepareIndex("royals", "king", "king").setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "prince", "prince").setParent("king").setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "duke", "duke").setParent("prince").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "earl", "earl1").setParent("duke").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "earl", "earl2").setParent("duke").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "earl", "earl3").setParent("duke").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "earl", "earl4").setParent("duke").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "baron", "baron1").setParent("earl1").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "baron", "baron2").setParent("earl2").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "baron", "baron3").setParent("earl3").setRouting("king")
+            .setSource("{}", XContentType.JSON));
+        requests.add(client().prepareIndex("royals", "baron", "baron4").setParent("earl4").setRouting("king")
+            .setSource("{}", XContentType.JSON));
         indexRandom(true, requests);
 
         SearchResponse response = client().prepareSearch("royals")
                 .setTypes("duke")
                 .setQuery(boolQuery()
                         .filter(hasParentQuery("prince",
-                                hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings")),
-                                false).innerHit(new InnerHitBuilder().setName("princes"))
+                                hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings"), false),
+                                false).innerHit(new InnerHitBuilder().setName("princes"), false)
                         )
                         .filter(hasChildQuery("earl",
-                                hasChildQuery("baron", matchAllQuery(), ScoreMode.None).innerHit(new InnerHitBuilder().setName("barons")),
+                                hasChildQuery("baron", matchAllQuery(), ScoreMode.None)
+                                        .innerHit(new InnerHitBuilder().setName("barons"), false),
                                 ScoreMode.None).innerHit(new InnerHitBuilder()
                                     .addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC))
                                     .setName("earls")
-                                    .setSize(4))
+                                    .setSize(4), false)
                         )
                 )
                 .get();
@@ -860,7 +876,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .should(termQuery("nested1.n_field1", "n_value1_3").queryName("test2"))
                 .should(termQuery("nested1.n_field2", "n_value2_2").queryName("test3"));
         query = nestedQuery("nested1", query, ScoreMode.Avg).innerHit(
-                new InnerHitBuilder().addSort(new FieldSortBuilder("nested1.n_field1").order(SortOrder.ASC)));
+                new InnerHitBuilder().addSort(new FieldSortBuilder("nested1.n_field1").order(SortOrder.ASC)), false);
         SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(query)
                 .setSize(numDocs)
@@ -893,16 +909,16 @@ public class InnerHitsIT extends ESIntegTestCase {
     public void testMatchesQueriesParentChildInnerHits() throws Exception {
         assertAcked(prepareCreate("index").addMapping("child", "_parent", "type=parent"));
         List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("index", "parent", "1").setSource("{}"));
+        requests.add(client().prepareIndex("index", "parent", "1").setSource("{}", XContentType.JSON));
         requests.add(client().prepareIndex("index", "child", "1").setParent("1").setSource("field", "value1"));
         requests.add(client().prepareIndex("index", "child", "2").setParent("1").setSource("field", "value2"));
-        requests.add(client().prepareIndex("index", "parent", "2").setSource("{}"));
+        requests.add(client().prepareIndex("index", "parent", "2").setSource("{}", XContentType.JSON));
         requests.add(client().prepareIndex("index", "child", "3").setParent("2").setSource("field", "value1"));
         indexRandom(true, requests);
 
         SearchResponse response = client().prepareSearch("index")
                 .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None)
-                        .innerHit(new InnerHitBuilder()))
+                        .innerHit(new InnerHitBuilder(), false))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 2);
@@ -917,7 +933,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1"));
 
         QueryBuilder query = hasChildQuery("child", matchQuery("field", "value2").queryName("_name2"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder());
+                .innerHit(new InnerHitBuilder(), false);
         response = client().prepareSearch("index")
                 .setQuery(query)
                 .addSort("_uid", SortOrder.ASC)
@@ -932,12 +948,12 @@ public class InnerHitsIT extends ESIntegTestCase {
     public void testDontExplode() throws Exception {
         assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
         List<IndexRequestBuilder> requests = new ArrayList<>();
-        requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}"));
+        requests.add(client().prepareIndex("index1", "parent", "1").setSource("{}", XContentType.JSON));
         requests.add(client().prepareIndex("index1", "child", "1").setParent("1").setSource("field", "value1"));
         indexRandom(true, requests);
 
         QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
+                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false);
         SearchResponse response = client().prepareSearch("index1")
                 .setQuery(query)
                 .addSort("_uid", SortOrder.ASC)
@@ -957,7 +973,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         .get();
 
         query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
-                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1));
+                .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false);
         response = client().prepareSearch("index2")
                 .setQuery(query)
                 .addSort("_uid", SortOrder.ASC)
@@ -983,7 +999,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch()
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
                 .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true,
-                    new String[]{"comments.message"}, null))))
+                    new String[]{"comments.message"}, null)), false))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -1004,7 +1020,7 @@ public class InnerHitsIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch("test")
             .setQuery(boolQuery().must(matchQuery("key", "value"))
                 .should(hasChildQuery("child_type", nestedQuery("nested_type", matchAllQuery(), ScoreMode.None)
-                    .innerHit(new InnerHitBuilder()), ScoreMode.None).innerHit(new InnerHitBuilder())))
+                    .innerHit(new InnerHitBuilder(), false), ScoreMode.None).innerHit(new InnerHitBuilder(), false)))
             .get();
         assertHitCount(response, 1);
         SearchHit hit = response.getHits().getAt(0);
@@ -1012,4 +1028,40 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertThat(hit.getInnerHits().get("child_type").getAt(0).getInnerHits().get("nested_type").getAt(0).field("_parent"), nullValue());
     }
 
+    public void testInnerHitsWithIgnoreUnmapped() throws Exception {
+        assertAcked(prepareCreate("index1")
+            .addMapping("parent_type", "nested_type", "type=nested")
+            .addMapping("child_type", "_parent", "type=parent_type")
+        );
+        createIndex("index2");
+        client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get();
+        client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}", XContentType.JSON).get();
+        client().prepareIndex("index2", "type", "3").setSource("key", "value").get();
+        refresh();
+
+        SearchResponse response = client().prepareSearch("index1", "index2")
+            .setQuery(boolQuery()
+                .should(nestedQuery("nested_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
+                        .innerHit(new InnerHitBuilder(), true))
+                .should(termQuery("key", "value"))
+            )
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+
+        response = client().prepareSearch("index1", "index2")
+            .setQuery(boolQuery()
+                .should(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
+                        .innerHit(new InnerHitBuilder(), true))
+                .should(termQuery("key", "value"))
+            )
+            .addSort("_uid", SortOrder.ASC)
+            .get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
index fc22b3198a1..12d9f77c603 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesIT.java
@@ -292,7 +292,7 @@ public class MatchedQueriesIT extends ESIntegTestCase {
             SearchResponse searchResponse = client().prepareSearch()
                     .setQuery(
                             boolQuery()
-                                    .minimumNumberShouldMatch(1)
+                                    .minimumShouldMatch(1)
                                     .should(queryStringQuery("dolor").queryName("dolor"))
                                     .should(queryStringQuery("elit").queryName("elit"))
                     )
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java
new file mode 100644
index 00000000000..968424eee73
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/ParentFieldSubFetchPhaseTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.subphase;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.ContentPath;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.ParentFieldMapper;
+import org.elasticsearch.test.ESTestCase;
+
+public class ParentFieldSubFetchPhaseTests extends ESTestCase {
+
+    public void testGetParentId() throws Exception {
+        ParentFieldMapper fieldMapper = createParentFieldMapper();
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig());
+        Document document = new Document();
+        document.add(new SortedDocValuesField(fieldMapper.fieldType().name(), new BytesRef("1")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        String id = ParentFieldSubFetchPhase.getParentId(fieldMapper, indexReader.leaves().get(0).reader(), 0);
+        assertEquals("1", id);
+
+        indexReader.close();
+        directory.close();
+    }
+
+    public void testGetParentIdNoParentField() throws Exception {
+        ParentFieldMapper fieldMapper = createParentFieldMapper();
+        Directory directory = newDirectory();
+        IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig());
+        Document document = new Document();
+        document.add(new SortedDocValuesField("different_field", new BytesRef("1")));
+        indexWriter.addDocument(document);
+        indexWriter.close();
+
+        IndexReader indexReader = DirectoryReader.open(directory);
+        String id = ParentFieldSubFetchPhase.getParentId(fieldMapper, indexReader.leaves().get(0).reader(), 0);
+        assertNull(id);
+
+        indexReader.close();
+        directory.close();
+    }
+
+    private ParentFieldMapper createParentFieldMapper() {
+        Settings settings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .build();
+        return new ParentFieldMapper.Builder("type")
+                .type("parent_type")
+                .build(new Mapper.BuilderContext(settings, new ContentPath(0)));
+    }
+
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
index 07a70c91ef9..944427b7e17 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java
@@ -22,12 +22,12 @@ package org.elasticsearch.search.fetch.subphase.highlight;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -46,7 +46,6 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Field;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder.Order;
@@ -75,7 +74,7 @@ public class HighlightBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -84,13 +83,13 @@ public class HighlightBuilderTests extends ESTestCase {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -129,7 +128,7 @@ public class HighlightBuilderTests extends ESTestCase {
             XContentBuilder shuffled = shuffleXContent(builder);
 
             XContentParser parser = createParser(shuffled);
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+            QueryParseContext context = new QueryParseContext(parser);
             parser.nextToken();
             HighlightBuilder secondHighlightBuilder;
             try {
@@ -170,7 +169,7 @@ public class HighlightBuilderTests extends ESTestCase {
 
     private <T extends Throwable> T expectParseThrows(Class<T> exceptionClass, String highlightElement) throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         return expectThrows(exceptionClass, () -> HighlightBuilder.fromXContent(context));
     }
 
@@ -265,7 +264,7 @@ public class HighlightBuilderTests extends ESTestCase {
         Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
-        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, indicesQueriesRegistry,
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
                 null, null, System::currentTimeMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
@@ -380,7 +379,7 @@ public class HighlightBuilderTests extends ESTestCase {
                 "}\n";
         XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
         HighlightBuilder highlightBuilder = HighlightBuilder.fromXContent(context);
         assertArrayEquals("setting tags_schema 'styled' should alter pre_tags", HighlightBuilder.DEFAULT_STYLED_PRE_TAG,
                 highlightBuilder.preTags());
@@ -392,7 +391,7 @@ public class HighlightBuilderTests extends ESTestCase {
                 "}\n";
         parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertArrayEquals("setting tags_schema 'default' should alter pre_tags", HighlightBuilder.DEFAULT_PRE_TAGS,
                 highlightBuilder.preTags());
@@ -413,21 +412,21 @@ public class HighlightBuilderTests extends ESTestCase {
         String highlightElement = "{ }";
         XContentParser parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
         HighlightBuilder highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("expected plain HighlightBuilder", new HighlightBuilder(), highlightBuilder);
 
         highlightElement = "{ \"fields\" : { } }";
         parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("defining no field should return plain HighlightBuilder", new HighlightBuilder(), highlightBuilder);
 
         highlightElement = "{ \"fields\" : { \"foo\" : { } } }";
         parser = createParser(JsonXContent.jsonXContent, highlightElement);
 
-        context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        context = new QueryParseContext(parser);
         highlightBuilder = HighlightBuilder.fromXContent(context);
         assertEquals("expected HighlightBuilder with field", new HighlightBuilder().field(new Field("foo")), highlightBuilder);
     }
@@ -715,4 +714,9 @@ public class HighlightBuilderTests extends ESTestCase {
     private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException {
         return ESTestCase.copyWriteable(original, namedWriteableRegistry, HighlightBuilder::new);
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java
index 6580be52555..027ae53a23d 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightFieldTests.java
@@ -38,7 +38,7 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashC
 public class HighlightFieldTests extends ESTestCase {
 
     public static HighlightField createTestItem() {
-        String name = frequently() ? randomAsciiOfLengthBetween(1, 20) : randomRealisticUnicodeOfCodepointLengthBetween(1, 20);
+        String name = frequently() ? randomAsciiOfLengthBetween(5, 20) : randomRealisticUnicodeOfCodepointLengthBetween(5, 20);
         Text[] fragments = null;
         if (frequently()) {
             int size = randomIntBetween(0, 5);
@@ -63,6 +63,7 @@ public class HighlightFieldTests extends ESTestCase {
         builder.endObject();
         XContentParser parser = createParser(builder);
         parser.nextToken(); // skip to the opening object token, fromXContent advances from here and starts with the field name
+        parser.nextToken();
         HighlightField parsedField = HighlightField.fromXContent(parser);
         assertEquals(highlightField, parsedField);
         if (highlightField.fragments() != null) {
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
index 9b54363936f..ca7548257ef 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlighterSearchIT.java
@@ -20,12 +20,10 @@ package org.elasticsearch.search.fetch.subphase.highlight;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
@@ -99,6 +97,8 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
 
 public class HighlighterSearchIT extends ESIntegTestCase {
+    private static final String[] ALL_TYPES = new String[] {"plain", "postings", "fvh", "unified"};
+    private static final String[] UNIFIED_AND_NULL = new String[] {null, "unified"};
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
@@ -123,9 +123,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             .setSource(jsonBuilder().startObject().field("text", "foo").endObject())
             .get();
         refresh();
-        SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "foo"))
-            .highlighter(new HighlightBuilder().field(new Field("text"))).get();
-        assertHighlight(search, 0, "text", 0, equalTo("<em>foo</em>"));
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "foo"))
+                .highlighter(new HighlightBuilder().field(new Field("text")).highlighterType(type)).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>foo</em>"));
+        }
     }
 
     public void testHighlightingWithWildcardName() throws IOException {
@@ -149,10 +151,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource(jsonBuilder().startObject().field("text", "text").endObject())
                 .get();
         refresh();
-        String highlighter = randomFrom("plain", "postings", "fvh");
-        SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
-                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get();
-        assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+        for (String type : ALL_TYPES) {
+            SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(type))).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+        }
     }
 
     public void testHighlightingWhenFieldsAreNotStoredThereIsNoSource() throws IOException {
@@ -184,14 +187,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource(jsonBuilder().startObject().field("unstored_text", "text").field("text", "text").endObject())
                 .get();
         refresh();
-        String highlighter = randomFrom("plain", "postings", "fvh");
-        SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
-                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(highlighter))).get();
-        assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
-        search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+        for (String type : ALL_TYPES) {
+            SearchResponse search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
+                .highlighter(new HighlightBuilder().field(new Field("*").highlighterType(type))).get();
+            assertHighlight(search, 0, "text", 0, equalTo("<em>text</em>"));
+            search = client().prepareSearch().setQuery(constantScoreQuery(matchQuery("text", "text")))
                 .highlighter(new HighlightBuilder().field(new Field("unstored_text"))).get();
-        assertNoFailures(search);
-        assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0));
+            assertNoFailures(search);
+            assertThat(search.getHits().getAt(0).getHighlightFields().size(), equalTo(0));
+        }
     }
 
     // see #3486
@@ -368,7 +372,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                             + "and should be highlighted")
                 .get();
         refresh();
-
         SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("long_term", "thisisaverylongwordandmakessurethisfails foo highlighed"))
                 .highlighter(new HighlightBuilder().field("long_term", 18, 1))
@@ -418,23 +421,25 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 0))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
-            assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
+                assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -470,23 +475,25 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2).highlighterType(type))
                 .execute().get();
 
-        for (int i = 0; i < 5; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
-            assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < 5; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> 1"));
+                assertHighlight(search, i, "attachments.body", 1, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -534,24 +541,28 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             assertHighlight(search, i, "title", 1, 2, equalTo("This is the second <em>bug</em> to perform highlighting on."));
         }
 
-        search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
                 //sentences will be generated out of each value
-                .highlighter(new HighlightBuilder().field("title")).get();
+                .highlighter(new HighlightBuilder().field("title").highlighterType(type)).get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch."));
-            assertHighlight(search, i, "title", 1, 2, equalTo("This is the second <em>bug</em> to perform highlighting on."));
-        }
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0,
+                    equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch."));
+                assertHighlight(search, i, "title", 1, 2,
+                    equalTo("This is the second <em>bug</em> to perform highlighting on."));
+            }
 
-        search = client().prepareSearch()
+            search = client().prepareSearch()
                 .setQuery(matchQuery("attachments.body", "attachment"))
-                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2))
+                .highlighter(new HighlightBuilder().field("attachments.body", -1, 2).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> for this test"));
-            assertHighlight(search, i, "attachments.body", 1, 2, equalTo("<em>attachment</em> 2"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "attachments.body", 0, equalTo("<em>attachment</em> for this test"));
+                assertHighlight(search, i, "attachments.body", 1, 2, equalTo("<em>attachment</em> 2"));
+            }
         }
     }
 
@@ -654,52 +665,37 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource("field1", "The quick brown fox jumps over the lazy dog", "field2", "second field content").get();
         refresh();
 
-        //works using stored field
-        SearchResponse searchResponse = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            //works using stored field
+            SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(termQuery("field1", "quick"))
-                .highlighter(new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")))
+                .highlighter(new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type)))
                 .get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
 
-        assertFailures(client().prepareSearch("test")
-                        .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
-                                        .highlighterType("plain").forceSource(true))),
+            assertFailures(client().prepareSearch("test")
+                    .setQuery(termQuery("field1", "quick"))
+                    .highlighter(
+                        new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
+                            .highlighterType(type).forceSource(true))),
                 RestStatus.BAD_REQUEST,
                 containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
 
-        assertFailures(client().prepareSearch("test")
-                        .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType("fvh")
-                                        .forceSource(true))),
+            SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
+                .highlighter(highlight().forceSource(true).field("field1").highlighterType(type));
+            assertFailures(client().prepareSearch("test").setSource(searchSource),
                 RestStatus.BAD_REQUEST,
                 containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
 
-        assertFailures(client().prepareSearch("test")
-                .setQuery(termQuery("field1", "quick"))
-                        .highlighter(
-                                new HighlightBuilder().field(new Field("field1").preTags("<xxx>").postTags("</xxx>")
-                                        .highlighterType("postings").forceSource(true))),
-                RestStatus.BAD_REQUEST,
-                containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
-
-        SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
-                .highlighter(highlight().forceSource(true).field("field1"));
-        assertFailures(client().prepareSearch("test").setSource(searchSource),
-                RestStatus.BAD_REQUEST,
-                containsString("source is forced for fields [field1] but type [type1] has disabled _source"));
-
-        searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
-                .highlighter(highlight().forceSource(true).field("field*"));
-        assertFailures(client().prepareSearch("test").setSource(searchSource),
+            searchSource = SearchSourceBuilder.searchSource().query(termQuery("field1", "quick"))
+                .highlighter(highlight().forceSource(true).field("field*").highlighterType(type));
+            assertFailures(client().prepareSearch("test").setSource(searchSource),
                 RestStatus.BAD_REQUEST,
                 matches("source is forced for fields \\[field\\d, field\\d\\] but type \\[type1\\] has disabled _source"));
+        }
     }
 
     public void testPlainHighlighter() throws Exception {
-        createIndex("test");
         ensureGreen();
 
         client().prepareIndex("test", "type1")
@@ -715,49 +711,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
         assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on _all, highlighting on field1");
-        source = searchSource()
-                .query(termQuery("_all", "test"))
-                .highlighter(highlight().field("field1").order("score").preTags("<xxx>").postTags("</xxx>").requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
-
-        logger.info("--> searching on _all, highlighting on field2");
-        source = searchSource()
-                .query(termQuery("_all", "quick"))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
-
-        logger.info("--> searching on _all, highlighting on field2");
-        source = searchSource()
-                .query(prefixQuery("_all", "qui"))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
-
-        logger.info("--> searching on _all with constant score, highlighting on field2");
-        source = searchSource()
-                .query(constantScoreQuery(prefixQuery("_all", "qui")))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
-
-        logger.info("--> searching on _all with constant score, highlighting on field2");
-        source = searchSource()
-                .query(boolQuery().should(constantScoreQuery(prefixQuery("_all", "qui"))))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
     }
 
     public void testFastVectorHighlighter() throws Exception {
@@ -776,36 +729,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
         assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on _all, highlighting on field1");
-        source = searchSource()
-                .query(termQuery("_all", "test"))
-                .highlighter(highlight().field("field1", 100, 0).order("score").preTags("<xxx>").postTags("</xxx>")
-                        .requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
-
-        logger.info("--> searching on _all, highlighting on field2");
-        source = searchSource()
-                .query(termQuery("_all", "quick"))
-                .highlighter(highlight().field("field2", 100, 0).order("score").preTags("<xxx>").postTags("</xxx>")
-                        .requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
-
-        logger.info("--> searching on _all, highlighting on field2");
-        source = searchSource()
-                .query(prefixQuery("_all", "qui"))
-                .highlighter(highlight().field("field2", 100, 0).order("score").preTags("<xxx>").postTags("</xxx>")
-                        .requireFieldMatch(false));
-
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy dog"));
-
         logger.info("--> searching with boundary characters");
         source = searchSource()
                 .query(matchQuery("field2", "quick"))
@@ -1051,22 +974,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
             assertHighlight(searchResponse, i, "field1", 0, 1, equalTo("<em>test</em> " + hit.id()));
         }
-
-        logger.info("--> searching explicitly _all and highlighting on _all");
-        searchResponse = client().prepareSearch()
-                .setSize(COUNT)
-                .setQuery(termQuery("_all", "test"))
-                .highlighter(new HighlightBuilder().field("_all", 100, 0))
-                .get();
-        for (int i = 0; i < COUNT; i++) {
-            SearchHit hit = searchResponse.getHits().getHits()[i];
-            assertHighlight(searchResponse, i, "_all", 0, 1, equalTo("<em>test</em> " + hit.id()));
-        }
     }
 
     public XContentBuilder type1TermVectorMapping() throws IOException {
         return XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
                 .startObject("properties")
                 .startObject("field1").field("type", "text").field("term_vector", "with_positions_offsets").endObject()
                 .startObject("field2").field("type", "text").field("term_vector", "with_positions_offsets").endObject()
@@ -1085,13 +996,16 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "bug"))
-                .highlighter(new HighlightBuilder().field("title", -1, 0))
+                .highlighter(new HighlightBuilder().field("title", -1, 0).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < 5; i++) {
-            assertHighlight(search, i, "title", 0, 1, equalTo("This is a test on the highlighting <em>bug</em> present in elasticsearch"));
+            for (int i = 0; i < 5; i++) {
+                assertHighlight(search, i, "title", 0, 1, equalTo("This is a test on the highlighting <em>bug</em> " +
+                    "present in elasticsearch"));
+            }
         }
     }
 
@@ -1128,14 +1042,16 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1, 10))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1, 10).highlighterType(type))
                 .get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(search, i, "title", 0, 1,
-                    equalTo("This is a html escaping highlighting <em>test</em> for *&amp;? elasticsearch"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(search, i, "title", 0, 1,
+                    startsWith("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            }
         }
     }
 
@@ -1180,21 +1096,24 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         ensureGreen();
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title
-        search = client().prepareSearch()
+            // search on title.key and highlight on title
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperVectorFromSource() throws Exception {
@@ -1219,21 +1138,23 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        search = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperNoVectorWithStore() throws Exception {
@@ -1259,21 +1180,24 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title
-        search = client().prepareSearch()
+            // search on title.key and highlight on title
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testMultiMapperNoVectorFromSource() throws Exception {
@@ -1298,21 +1222,23 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse search = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse search = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(search, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        search = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            search = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1))
+                .highlighter(new HighlightBuilder().encoder("html").field("title.key", 50, 1).highlighterType(type))
                 .get();
 
-        assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(search, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testFastVectorHighlighterShouldFailIfNoTermVectors() throws Exception {
@@ -1419,14 +1345,16 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f))
-                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog"));
+        }
     }
 
     public void testBoostingQueryTermVector() throws IOException {
@@ -1457,12 +1385,14 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource()
                 .query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+        }
     }
 
     public void testCommonTermsTermVector() throws IOException {
@@ -1491,7 +1421,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .putArray("index.analysis.filter.synonym.synonyms", "quick => fast");
 
         assertAcked(prepareCreate("test").setSettings(builder.build()).addMapping("type1", type1TermVectorMapping())
-                .addMapping("type2", "_all", "store=true,term_vector=with_positions_offsets",
+                .addMapping("type2",
                         "field4", "type=text,term_vector=with_positions_offsets,analyzer=synonym",
                         "field3", "type=text,analyzer=synonym"));
         ensureGreen();
@@ -1502,63 +1432,76 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("field1", "The quick browse button is a fancy thing, right bro?").get();
         refresh();
         logger.info("--> highlighting and searching on field0");
-        SearchSourceBuilder source = searchSource()
+
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource()
                 .query(matchPhrasePrefixQuery("field0", "quick bro"))
-                .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field0").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field0", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
 
-        logger.info("--> highlighting and searching on field1");
-        source = searchSource()
+            logger.info("--> highlighting and searching on field1");
+            source = searchSource()
                 .query(matchPhrasePrefixQuery("field1", "quick bro"))
-                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>"));
+                .highlighter(highlight().field("field1").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
+            assertHighlight(searchResponse, 0, "field1", 0, 1, anyOf(
                 equalTo("The <x>quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("The <x>quick brown</x> fox jumps over the lazy dog")));
-        assertHighlight(searchResponse, 1, "field1", 0, 1, anyOf(
+                equalTo("The <x>quick brown</x> fox jumps over the lazy dog"),
+                equalTo("The <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
+            assertHighlight(searchResponse, 1, "field1", 0, 1, anyOf(
                 equalTo("The <x>quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("The <x>quick brown</x> fox jumps over the lazy dog")));
+                equalTo("The <x>quick brown</x> fox jumps over the lazy dog"),
+                equalTo("The <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
 
-        // with synonyms
-        client().prepareIndex("test", "type2", "0").setSource(
+            // with synonyms
+            client().prepareIndex("test", "type2", "0").setSource(
                 "field4", "The quick brown fox jumps over the lazy dog",
                 "field3", "The quick brown fox jumps over the lazy dog").get();
-        client().prepareIndex("test", "type2", "1").setSource(
+            client().prepareIndex("test", "type2", "1").setSource(
                 "field4", "The quick browse button is a fancy thing, right bro?").get();
-        client().prepareIndex("test", "type2", "2").setSource(
+            client().prepareIndex("test", "type2", "2").setSource(
                 "field4", "a quick fast blue car").get();
-        refresh();
+            refresh();
 
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field3", "fast bro"))
-                .highlighter(highlight().field("field3").order("score").preTags("<x>").postTags("</x>"));
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field3", "fast bro"))
+                .highlighter(highlight().field("field3").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field3", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
+            assertHighlight(searchResponse, 0, "field3", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog"));
 
-        logger.info("--> highlighting and searching on field4");
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "the fast bro"))
-                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>"));
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            logger.info("--> highlighting and searching on field4");
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "the fast bro"))
+                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(
+            assertHighlight(searchResponse, 0, "field4", 0, 1, anyOf(
                 equalTo("<x>The quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("<x>The quick brown</x> fox jumps over the lazy dog")));
-        assertHighlight(searchResponse, 1, "field4", 0, 1, anyOf(
+                equalTo("<x>The quick brown</x> fox jumps over the lazy dog"),
+                equalTo("<x>The</x> <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("<x>The</x> <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
+            assertHighlight(searchResponse, 1, "field4", 0, 1, anyOf(
                 equalTo("<x>The quick browse</x> button is a fancy thing, right bro?"),
-                equalTo("<x>The quick brown</x> fox jumps over the lazy dog")));
+                equalTo("<x>The quick brown</x> fox jumps over the lazy dog"),
+                equalTo("<x>The</x> <x>quick</x> <x>browse</x> button is a fancy thing, right bro?"),
+                equalTo("<x>The</x> <x>quick</x> <x>brown</x> fox jumps over the lazy dog")));
 
-        logger.info("--> highlighting and searching on field4");
-        source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "a fast quick blue ca"))
-                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>"));
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            logger.info("--> highlighting and searching on field4");
+            source = searchSource().postFilter(typeQuery("type2")).query(matchPhrasePrefixQuery("field4", "a fast quick blue ca"))
+                .highlighter(highlight().field("field4").order("score").preTags("<x>").postTags("</x>").highlighterType(type));
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field4", 0, 1, equalTo("<x>a quick fast blue car</x>"));
+            assertHighlight(searchResponse, 0, "field4", 0, 1,
+                anyOf(equalTo("<x>a quick fast blue car</x>"),
+                      equalTo("<x>a</x> <x>quick</x> <x>fast</x> <x>blue</x> <x>car</x>")));
+        }
     }
 
     public void testPlainHighlightDifferentFragmenter() throws Exception {
@@ -1649,13 +1592,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                         .endObject()).get();
         refresh();
 
-        // This query used to fail when the field to highlight was absent
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            // This query used to fail when the field to highlight was absent
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("field", "highlight").type(MatchQuery.Type.BOOLEAN))
                 .highlighter(
-                        new HighlightBuilder().field(new HighlightBuilder.Field("highlight_field").fragmentSize(-1).numOfFragments(1)
-                                .fragmenter("simple"))).get();
-        assertThat(response.getHits().hits()[0].highlightFields().isEmpty(), equalTo(true));
+                    new HighlightBuilder().field(new HighlightBuilder.Field("highlight_field").fragmentSize(-1).numOfFragments(1)
+                        .fragmenter("simple")).highlighterType(type)).get();
+            assertThat(response.getHits().hits()[0].highlightFields().isEmpty(), equalTo(true));
+        }
     }
 
     // Issue #3211
@@ -1700,11 +1645,13 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource("text", "elasticsearch test").get();
         refresh();
 
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("text", "test").type(MatchQuery.Type.BOOLEAN))
-                .highlighter(new HighlightBuilder().field("text")).execute().actionGet();
-        // PatternAnalyzer will throw an exception if it is resetted twice
-        assertHitCount(response, 1L);
+                .highlighter(new HighlightBuilder().field("text").highlighterType(type)).execute().actionGet();
+            // PatternAnalyzer will throw an exception if it is resetted twice
+            assertHitCount(response, 1L);
+        }
     }
 
     public void testHighlightUsesHighlightQuery() throws IOException {
@@ -1715,55 +1662,30 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         index("test", "type1", "1", "text", "Testing the highlight query feature");
         refresh();
 
-        HighlightBuilder.Field field = new HighlightBuilder.Field("text");
-
-        HighlightBuilder highlightBuilder = new HighlightBuilder().field(field);
-        SearchRequestBuilder search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing"))
+        for (String type : ALL_TYPES) {
+            HighlightBuilder.Field field = new HighlightBuilder.Field("text");
+            HighlightBuilder highlightBuilder = new HighlightBuilder().field(field).highlighterType(type);
+            SearchRequestBuilder search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing"))
                 .highlighter(highlightBuilder);
-        Matcher<String> searchQueryMatcher = equalTo("<em>Testing</em> the highlight query feature");
+            Matcher<String> searchQueryMatcher = equalTo("<em>Testing</em> the highlight query feature");
 
-        field.highlighterType("plain");
-        SearchResponse response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
-        field.highlighterType("fvh");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, searchQueryMatcher);
+            SearchResponse response = search.get();
+            assertHighlight(response, 0, "text", 0, searchQueryMatcher);
+            field = new HighlightBuilder.Field("text");
 
-        field = new HighlightBuilder.Field("text");
+            Matcher<String> hlQueryMatcher = equalTo("Testing the highlight <em>query</em> feature");
+            field.highlightQuery(matchQuery("text", "query"));
+            highlightBuilder = new HighlightBuilder().field(field);
+            search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")).highlighter(highlightBuilder);
+            response = search.get();
+            assertHighlight(response, 0, "text", 0, hlQueryMatcher);
 
-        Matcher<String> hlQueryMatcher = equalTo("Testing the highlight <em>query</em> feature");
-        field.highlightQuery(matchQuery("text", "query"));
-        highlightBuilder = new HighlightBuilder().field(field);
-        search = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("text", "testing")).highlighter(highlightBuilder);
-
-        field.highlighterType("fvh");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("plain");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        // Make sure the highlightQuery is taken into account when it is set on the highlight context instead of the field
-        highlightBuilder.highlightQuery(matchQuery("text", "query"));
-        field.highlighterType("fvh").highlightQuery(null);
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("plain");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
-
-        field.highlighterType("postings");
-        response = search.get();
-        assertHighlight(response, 0, "text", 0, hlQueryMatcher);
+            // Make sure the highlightQuery is taken into account when it is set on the highlight context instead of the field
+            highlightBuilder.highlightQuery(matchQuery("text", "query"));
+            field.highlighterType(type).highlightQuery(null);
+            response = search.get();
+            assertHighlight(response, 0, "text", 0, hlQueryMatcher);
+        }
     }
 
     private static String randomStoreField() {
@@ -1798,6 +1720,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // When noMatchSize is set to 0 you also shouldn't get any
         field.highlighterType("plain").noMatchSize(0);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1811,6 +1737,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // When noMatchSize is between 0 and the size of the string
         field.highlighterType("plain").noMatchSize(21);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1826,6 +1756,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
 
+        // Unified hl also works but the fragment is the whole first sentence (size ignored)
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
+
         // We can also ask for a fragment longer than the input string and get the whole string
         field.highlighterType("plain").noMatchSize(text.length() * 2);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1840,6 +1775,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
 
+        //no difference using unified hl as the noMatchSize is ignored (just needs to be greater than 0)
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
+
         // We can also ask for a fragment exactly the size of the input field and get the whole field
         field.highlighterType("plain").noMatchSize(text.length());
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1854,6 +1794,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
 
+        //no difference using unified hl as the noMatchSize is ignored (just needs to be greater than 0)
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
+
         // You can set noMatchSize globally in the highlighter as well
         field.highlighterType("plain").noMatchSize(null);
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
@@ -1867,6 +1812,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field).noMatchSize(21)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
+
         // We don't break if noMatchSize is less than zero though
         field.highlighterType("plain").noMatchSize(randomIntBetween(Integer.MIN_VALUE, -1));
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
@@ -1879,6 +1828,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
     }
 
     public void testHighlightNoMatchSizeWithMultivaluedFields() throws IOException {
@@ -1909,6 +1862,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
 
+        // Unified hl also works but the fragment is the whole first sentence (size ignored)
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("I am pretty long so some of me should get cut off."));
+
         // And noMatchSize returns nothing when the first entry is empty string!
         index("test", "type1", "2", "text", new String[] {"", text2});
         refresh();
@@ -1932,6 +1890,12 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test")
+            .setQuery(idsQueryBuilder)
+            .highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // But if the field was actually empty then you should get no highlighting field
         index("test", "type1", "3", "text", new String[] {});
         refresh();
@@ -1954,6 +1918,12 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
 
+        field.highlighterType("unified");
+        response = client().prepareSearch("test")
+            .setQuery(idsQueryBuilder)
+            .highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
+
         // Same for if the field doesn't even exist on the document
         index("test", "type1", "4");
         refresh();
@@ -1991,6 +1961,10 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         field.highlighterType("postings");
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertNotHighlighted(response, 0, "text");
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertNotHighlighted(response, 0, "text");
     }
 
     public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
@@ -2022,6 +1996,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence."));
 
+        // Unified hl also works but the fragment is the whole first sentence (size ignored)
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 1, equalTo("This is the first sentence."));
+
         //if there's a match we only return the values with matches (whole value as number_of_fragments == 0)
         MatchQueryBuilder queryBuilder = QueryBuilders.matchQuery("text", "third fifth");
         field.highlighterType("plain");
@@ -2038,6 +2017,11 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get();
         assertHighlight(response, 0, "text", 0, 2, equalTo("This is the <em>third</em> sentence. This is the fourth sentence."));
         assertHighlight(response, 0, "text", 1, 2, equalTo("This is the <em>fifth</em> sentence"));
+
+        field.highlighterType("unified");
+        response = client().prepareSearch("test").setQuery(queryBuilder).highlighter(new HighlightBuilder().field(field)).get();
+        assertHighlight(response, 0, "text", 0, 2, equalTo("This is the <em>third</em> sentence. This is the fourth sentence."));
+        assertHighlight(response, 0, "text", 1, 2, equalTo("This is the <em>fifth</em> sentence"));
     }
 
     public void testPostingsHighlighter() throws Exception {
@@ -2048,55 +2032,62 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy quick dog").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "test"))
-                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on field1, highlighting on field1");
-        source = searchSource()
+            logger.info("--> searching on field1, highlighting on field1");
+            source = searchSource()
                 .query(termQuery("field1", "test"))
-                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field1").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("this is a <xxx>test</xxx>"));
 
-        logger.info("--> searching on field2, highlighting on field2");
-        source = searchSource()
+            logger.info("--> searching on field2, highlighting on field2");
+            source = searchSource()
                 .query(termQuery("field2", "quick"))
-                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field2").order("score").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
+            assertHighlight(searchResponse, 0, "field2", 0, 1,
                 equalTo("The <xxx>quick</xxx> brown fox jumps over the lazy <xxx>quick</xxx> dog"));
 
-        logger.info("--> searching on field2, highlighting on field2");
-        source = searchSource()
+            logger.info("--> searching on field2, highlighting on field2");
+            source = searchSource()
                 .query(matchPhraseQuery("field2", "quick brown"))
-                .highlighter(highlight().field("field2").preTags("<xxx>").postTags("</xxx>"));
+                .highlighter(highlight().field("field2").preTags("<xxx>").postTags("</xxx>").highlighterType(type));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        //phrase query results in highlighting all different terms regardless of their positions
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
-                equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy <xxx>quick</xxx> dog"));
+            if (type == null) {
+                //phrase query results in highlighting all different terms regardless of their positions
+                assertHighlight(searchResponse, 0, "field2", 0, 1,
+                    equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy <xxx>quick</xxx> dog"));
+            } else {
+                assertHighlight(searchResponse, 0, "field2", 0, 1,
+                    equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy quick dog"));
+            }
 
-        //lets fall back to the standard highlighter then, what people would do to highlight query matches
-        logger.info("--> searching on field2, highlighting on field2, falling back to the plain highlighter");
-        source = searchSource()
-                .query(matchPhraseQuery("_all", "quick brown"))
-                .highlighter(highlight()
+            //lets fall back to the standard highlighter then, what people would do to highlight query matches
+            logger.info("--> searching on field2, highlighting on field2, falling back to the plain highlighter");
+            source = searchSource()
+                    .query(matchPhraseQuery("field2", "quick brown"))
+                    .highlighter(highlight()
                         .field("field2").preTags("<xxx>").postTags("</xxx>").highlighterType("plain").requireFieldMatch(false));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1,
+            assertHighlight(searchResponse, 0, "field2", 0, 1,
                 equalTo("The <xxx>quick</xxx> <xxx>brown</xxx> fox jumps over the lazy quick dog"));
+        }
     }
 
     public void testPostingsHighlighterMultipleFields() throws Exception {
@@ -2108,13 +2099,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                 "field2", "The <b>slow<b> brown fox. Second sentence.");
         refresh();
 
-        SearchResponse response = client().prepareSearch("test")
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse response = client().prepareSearch("test")
                 .setQuery(QueryBuilders.matchQuery("field1", "fox"))
                 .highlighter(
-                        new HighlightBuilder().field(new HighlightBuilder.Field("field1").preTags("<1>").postTags("</1>")
-                                .requireFieldMatch(true)))
+                    new HighlightBuilder().field(new Field("field1").preTags("<1>").postTags("</1>")
+                        .requireFieldMatch(true).highlighterType(type)))
                 .get();
-        assertHighlight(response, 0, "field1", 0, 1, equalTo("The <b>quick<b> brown <1>fox</1>."));
+            assertHighlight(response, 0, "field1", 0, 1, equalTo("The <b>quick<b> brown <1>fox</1>."));
+        }
     }
 
     public void testPostingsHighlighterNumberOfFragments() throws Exception {
@@ -2128,53 +2121,55 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                     + "The quick brown dog jumps over the lazy fox.").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "fox"))
                 .highlighter(highlight()
-                        .field(new HighlightBuilder.Field("field1").numOfFragments(5).preTags("<field1>").postTags("</field1>")));
+                    .field(new Field("field1").numOfFragments(5).preTags("<field1>").postTags("</field1>").highlighterType(type)));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field1", 0, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog."));
-        assertHighlight(searchResponse, 0, "field1", 1, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
-        assertHighlight(searchResponse, 0, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
+            assertHighlight(searchResponse, 0, "field1", 0, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog."));
+            assertHighlight(searchResponse, 0, "field1", 1, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
+            assertHighlight(searchResponse, 0, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
 
-        client().prepareIndex("test", "type1", "2")
-                .setSource("field1", new String[] {
-                        "The quick brown fox jumps over the lazy dog. Second sentence not finished",
-                        "The lazy red fox jumps over the quick dog.",
-                        "The quick brown dog jumps over the lazy fox."}).get();
-        refresh();
+            client().prepareIndex("test", "type1", "2")
+                .setSource("field1", new String[]{
+                    "The quick brown fox jumps over the lazy dog. Second sentence not finished",
+                    "The lazy red fox jumps over the quick dog.",
+                    "The quick brown dog jumps over the lazy fox."}).get();
+            refresh();
 
-        source = searchSource()
+            source = searchSource()
                 .query(termQuery("field1", "fox"))
                 .highlighter(highlight()
-                        .field(new HighlightBuilder.Field("field1").numOfFragments(0).preTags("<field1>").postTags("</field1>")));
+                    .field(new Field("field1").numOfFragments(0).preTags("<field1>").postTags("</field1>").highlighterType(type)));
 
-        searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-        assertHitCount(searchResponse, 2L);
+            searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHitCount(searchResponse, 2L);
 
-        for (SearchHit searchHit : searchResponse.getHits()) {
-            if ("1".equals(searchHit.id())) {
-                assertHighlight(searchHit, "field1", 0, 1, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. "
+            for (SearchHit searchHit : searchResponse.getHits()) {
+                if ("1".equals(searchHit.id())) {
+                    assertHighlight(searchHit, "field1", 0, 1, equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. "
                         + "The lazy red <field1>fox</field1> jumps over the quick dog. "
                         + "The quick brown dog jumps over the lazy <field1>fox</field1>."));
-            } else if ("2".equals(searchHit.id())) {
-                assertHighlight(searchHit, "field1", 0, 3,
+                } else if ("2".equals(searchHit.id())) {
+                    assertHighlight(searchHit, "field1", 0, 3,
                         equalTo("The quick brown <field1>fox</field1> jumps over the lazy dog. Second sentence not finished"));
-                assertHighlight(searchHit, "field1", 1, 3, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
-                assertHighlight(searchHit, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
-            } else {
-                fail("Only hits with id 1 and 2 are returned");
+                    assertHighlight(searchHit, "field1", 1, 3, equalTo("The lazy red <field1>fox</field1> jumps over the quick dog."));
+                    assertHighlight(searchHit, "field1", 2, 3, equalTo("The quick brown dog jumps over the lazy <field1>fox</field1>."));
+                } else {
+                    fail("Only hits with id 1 and 2 are returned");
+                }
             }
         }
     }
 
     public void testMultiMatchQueryHighlight() throws IOException {
-        String[] highlighterTypes = new String[] {"fvh", "plain", "postings"};
+        String[] highlighterTypes = new String[] {"fvh", "plain", "postings", "unified"};
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_all").field("store", true).field("index_options", "offsets").endObject()
                 .startObject("properties")
                     .startObject("field1")
                         .field("type", "text")
@@ -2241,24 +2236,26 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                             + "This one contains no matches."}).get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(termQuery("field1", "sentence"))
-                .highlighter(highlight().field("field1").order("score"));
+                .highlighter(highlight().field("field1").order("score").highlighterType(type));
 
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        Map<String,HighlightField> highlightFieldMap = searchResponse.getHits().getAt(0).highlightFields();
-        assertThat(highlightFieldMap.size(), equalTo(1));
-        HighlightField field1 = highlightFieldMap.get("field1");
-        assertThat(field1.fragments().length, equalTo(5));
-        assertThat(field1.fragments()[0].string(),
+            Map<String, HighlightField> highlightFieldMap = searchResponse.getHits().getAt(0).highlightFields();
+            assertThat(highlightFieldMap.size(), equalTo(1));
+            HighlightField field1 = highlightFieldMap.get("field1");
+            assertThat(field1.fragments().length, equalTo(5));
+            assertThat(field1.fragments()[0].string(),
                 equalTo("This <em>sentence</em> contains three <em>sentence</em> occurrences (<em>sentence</em>)."));
-        assertThat(field1.fragments()[1].string(), equalTo("This <em>sentence</em> contains two <em>sentence</em> matches."));
-        assertThat(field1.fragments()[2].string(), equalTo("This is the second value's first <em>sentence</em>."));
-        assertThat(field1.fragments()[3].string(), equalTo("This <em>sentence</em> contains one match, not that short."));
-        assertThat(field1.fragments()[4].string(),
+            assertThat(field1.fragments()[1].string(), equalTo("This <em>sentence</em> contains two <em>sentence</em> matches."));
+            assertThat(field1.fragments()[2].string(), equalTo("This is the second value's first <em>sentence</em>."));
+            assertThat(field1.fragments()[3].string(), equalTo("This <em>sentence</em> contains one match, not that short."));
+            assertThat(field1.fragments()[4].string(),
                 equalTo("One <em>sentence</em> match here and scored lower since the text is quite long, not that appealing."));
+        }
     }
 
     public void testPostingsHighlighterEscapeHtml() throws Exception {
@@ -2272,12 +2269,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         }
         indexRandom(true, indexRequestBuilders);
 
-        SearchResponse searchResponse = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title", "test"))
-                .highlighter(new HighlightBuilder().field("title").encoder("html")).get();
+                .highlighter(new HighlightBuilder().field("title").encoder("html").highlighterType(type)).get();
 
-        for (int i = 0; i < indexRequestBuilders.length; i++) {
-            assertHighlight(searchResponse, i, "title", 0, 1, equalTo("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            for (int i = 0; i < indexRequestBuilders.length; i++) {
+                assertHighlight(searchResponse, i, "title", 0, 1,
+                    equalTo("This is a html escaping highlighting <em>test</em> for *&amp;?"));
+            }
         }
     }
 
@@ -2301,25 +2301,29 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         ensureGreen();
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test . Second sentence.").get();
         refresh();
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse searchResponse = client().prepareSearch()
+
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse searchResponse = client().prepareSearch()
                 //lets make sure we analyze the query and we highlight the resulting terms
                 .setQuery(matchQuery("title", "This is a Test"))
-.highlighter(new HighlightBuilder().field("title")).get();
+                .highlighter(new HighlightBuilder().field("title").highlighterType(type)).get();
 
-        assertHitCount(searchResponse, 1L);
-        SearchHit hit = searchResponse.getHits().getAt(0);
-        //stopwords are not highlighted since not indexed
-        assertHighlight(hit, "title", 0, 1, equalTo("this is a <em>test</em> ."));
+            assertHitCount(searchResponse, 1L);
+            SearchHit hit = searchResponse.getHits().getAt(0);
+            //stopwords are not highlighted since not indexed
+            assertHighlight(hit, "title", 0, 1, equalTo("this is a <em>test</em> ."));
 
-        // search on title.key and highlight on title
-        searchResponse = client().prepareSearch()
+            // search on title.key and highlight on title
+            searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
                 .highlighter(new HighlightBuilder().field("title.key")).get();
-        assertHitCount(searchResponse, 1L);
+            assertHitCount(searchResponse, 1L);
 
-        //stopwords are now highlighted since we used only whitespace analyzer here
-        assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> ."));
+            //stopwords are now highlighted since we used only whitespace analyzer here
+            assertHighlight(searchResponse, 0, "title.key", 0, 1,
+                equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em> ."));
+        }
     }
 
     public void testPostingsHighlighterMultiMapperFromSource() throws Exception {
@@ -2344,20 +2348,22 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("title", "this is a test").get();
         refresh();
 
-        // simple search on body with standard analyzer with a simple field query
-        SearchResponse searchResponse = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            // simple search on body with standard analyzer with a simple field query
+            SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title", "this is a test"))
                 .highlighter(new HighlightBuilder().field("title"))
                 .get();
 
-        assertHighlight(searchResponse, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
+            assertHighlight(searchResponse, 0, "title", 0, 1, equalTo("this is a <em>test</em>"));
 
-        // search on title.key and highlight on title.key
-        searchResponse = client().prepareSearch()
+            // search on title.key and highlight on title.key
+            searchResponse = client().prepareSearch()
                 .setQuery(matchQuery("title.key", "this is a test"))
-                .highlighter(new HighlightBuilder().field("title.key")).get();
+                .highlighter(new HighlightBuilder().field("title.key").highlighterType(type)).get();
 
-        assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+            assertHighlight(searchResponse, 0, "title.key", 0, 1, equalTo("<em>this</em> <em>is</em> <em>a</em> <em>test</em>"));
+        }
     }
 
     public void testPostingsHighlighterShouldFailIfNoOffsets() throws Exception {
@@ -2408,13 +2414,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boostingQuery(termQuery("field2", "brown"), termQuery("field2", "foobar")).negativeBoost(0.5f))
-                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog!"));
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The quick <x>brown</x> fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterCommonTermsQuery() throws IOException {
@@ -2424,13 +2432,16 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
-                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>"));
-        SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
-        assertHitCount(searchResponse, 1L);
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(commonTermsQuery("field2", "quick brown").cutoffFrequency(100))
+                .highlighter(highlight().field("field2").preTags("<x>").postTags("</x>").highlighterType(type));
+            SearchResponse searchResponse = client().search(searchRequest("test").source(source)).actionGet();
+            assertHitCount(searchResponse, 1L);
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <x>quick</x> <x>brown</x> fox jumps over the lazy dog!"));
+        }
     }
 
     private static XContentBuilder type1PostingsffsetsMapping() throws IOException {
@@ -2451,11 +2462,12 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         refresh();
         logger.info("--> highlighting and searching on field2");
 
-        SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
-
+        for (String type : UNIFIED_AND_NULL) {
+            SearchSourceBuilder source = searchSource().query(prefixQuery("field2", "qui"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterFuzzyQuery() throws Exception {
@@ -2465,12 +2477,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(fuzzyQuery("field2", "quck"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterRegexpQuery() throws Exception {
@@ -2480,12 +2495,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(regexpQuery("field2", "qu[a-l]+k"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterWildcardQuery() throws Exception {
@@ -2495,19 +2513,22 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(wildcardQuery("field2", "qui*"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        source = searchSource().query(wildcardQuery("field2", "qu*k"))
-                .highlighter(highlight().field("field2"));
-        searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHitCount(searchResponse, 1L);
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+            source = searchSource().query(wildcardQuery("field2", "qu*k"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHitCount(searchResponse, 1L);
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterTermRangeQuery() throws Exception {
@@ -2516,12 +2537,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
         client().prepareIndex("test", "type1").setSource("field1", "this is a test", "field2", "aaab").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
 
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("<em>aaab</em>"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(rangeQuery("field2").gte("aaaa").lt("zzzz"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("<em>aaab</em>"));
+        }
     }
 
     public void testPostingsHighlighterQueryString() throws Exception {
@@ -2531,11 +2555,14 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1")
             .setSource("field1", "this is a test", "field2", "The quick brown fox jumps over the lazy dog! Second sentence.").get();
         refresh();
-        logger.info("--> highlighting and searching on field2");
-        SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2"))
-                .highlighter(highlight().field("field2"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field2");
+            SearchSourceBuilder source = searchSource().query(queryStringQuery("qui*").defaultField("field2"))
+                .highlighter(highlight().field("field2").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field2", 0, 1, equalTo("The <em>quick</em> brown fox jumps over the lazy dog!"));
+        }
     }
 
     public void testPostingsHighlighterRegexpQueryWithinConstantScoreQuery() throws Exception {
@@ -2545,11 +2572,13 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(constantScoreQuery(regexpQuery("field1", "pho[a-z]+")))
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterMultiTermQueryMultipleLevels() throws Exception {
@@ -2559,14 +2588,16 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(boolQuery()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(boolQuery()
                 .should(boolQuery().mustNot(QueryBuilders.existsQuery("field1")))
                 .should(matchQuery("field1", "test"))
                 .should(constantScoreQuery(queryStringQuery("field1:photo*"))))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterPrefixQueryWithinBooleanQuery() throws Exception {
@@ -2576,12 +2607,14 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource()
                 .query(boolQuery().must(prefixQuery("field1", "photo")).should(matchQuery("field1", "test").minimumShouldMatch("0")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterQueryStringWithinFilteredQuery() throws Exception {
@@ -2591,13 +2624,15 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1").setSource("field1", "The photography word will get highlighted").get();
         refresh();
 
-        logger.info("--> highlighting and searching on field1");
-        SearchSourceBuilder source = searchSource().query(boolQuery()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> highlighting and searching on field1");
+            SearchSourceBuilder source = searchSource().query(boolQuery()
                 .must(queryStringQuery("field1:photo*"))
                 .mustNot(existsQuery("field_null")))
-                .highlighter(highlight().field("field1"));
-        SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
-        assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+                .highlighter(highlight().field("field1").highlighterType(type));
+            SearchResponse searchResponse = client().prepareSearch("test").setSource(source).get();
+            assertHighlight(searchResponse, 0, "field1", 0, 1, equalTo("The <em>photography</em> word will get highlighted"));
+        }
     }
 
     public void testPostingsHighlighterManyDocs() throws Exception {
@@ -2619,18 +2654,20 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         logger.info("--> indexing docs");
         indexRandom(true, indexRequestBuilders);
 
-        logger.info("--> searching explicitly on field1 and highlighting on it");
-        SearchRequestBuilder searchRequestBuilder = client().prepareSearch()
+        for (String type : UNIFIED_AND_NULL) {
+            logger.info("--> searching explicitly on field1 and highlighting on it");
+            SearchRequestBuilder searchRequestBuilder = client().prepareSearch()
                 .setSize(COUNT)
                 .setQuery(termQuery("field1", "test"))
-                .highlighter(new HighlightBuilder().field("field1"));
-        SearchResponse searchResponse =
+                .highlighter(new HighlightBuilder().field("field1").highlighterType(type));
+            SearchResponse searchResponse =
                 searchRequestBuilder.get();
-        assertHitCount(searchResponse, COUNT);
-        assertThat(searchResponse.getHits().hits().length, equalTo(COUNT));
-        for (SearchHit hit : searchResponse.getHits()) {
-            String prefix = prefixes.get(hit.id());
-            assertHighlight(hit, "field1", 0, 1, equalTo("Sentence " + prefix + " <em>test</em>."));
+            assertHitCount(searchResponse, COUNT);
+            assertThat(searchResponse.getHits().hits().length, equalTo(COUNT));
+            for (SearchHit hit : searchResponse.getHits()) {
+                String prefix = prefixes.get(hit.id());
+                assertHighlight(hit, "field1", 0, 1, equalTo("Sentence " + prefix + " <em>test</em>."));
+            }
         }
     }
 
@@ -2645,7 +2682,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test typename"));
 
-        for (String highlighter: new String[] {"plain", "fvh", "postings"}) {
+        for (String highlighter : ALL_TYPES) {
             SearchResponse response = client().prepareSearch("test").setTypes("typename").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> typename"));
@@ -2664,7 +2701,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
         indexRandom(true, client().prepareIndex("test", "typename").setSource("foo", "test japanese"));
 
-        for (String highlighter: new String[] {"plain", "fvh", "postings"}) {
+        for (String highlighter : ALL_TYPES) {
             SearchResponse response = client().prepareSearch("filtered_alias").setTypes("typename").setQuery(matchQuery("foo", "test"))
                     .highlighter(new HighlightBuilder().field("foo").highlighterType(highlighter).requireFieldMatch(false)).get();
             assertHighlight(response, 0, "foo", 0, 1, equalTo("<em>test</em> japanese"));
@@ -2772,7 +2809,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             .setSource(jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject())
             .get();
         refresh();
-        String highlighterType = randomFrom("plain", "fvh", "postings");
+        String highlighterType = randomFrom("plain", "fvh", "postings", "unified");
         QueryBuilder query = QueryBuilders.boolQuery().should(QueryBuilders.geoBoundingBoxQuery("geo_point")
             .setCorners(61.10078883158897, -170.15625, -64.92354174306496, 118.47656249999999))
             .should(QueryBuilders.termQuery("text", "failure"));
@@ -2790,9 +2827,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
         mappings.startObject("jobs")
-            .startObject("_all")
-            .field("enabled", false)
-            .endObject()
             .startObject("properties")
             .startObject("loc")
             .field("type", "geo_point")
@@ -2896,15 +2930,17 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .get();
 
-        SearchResponse searchResponse = client().prepareSearch()
-            .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")))
-            .highlighter(new HighlightBuilder()
-                .field(new Field("text")))
-            .get();
-        assertHitCount(searchResponse, 1);
-        HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
-        assertThat(field.getFragments().length, equalTo(1));
-        assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro")))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("text")).highlighterType(type))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
     }
 
     public void testFiltersFunctionScoreQueryHighlight() throws Exception {
@@ -2915,16 +2951,19 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         FunctionScoreQueryBuilder.FilterFunctionBuilder filterBuilder =
             new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.termQuery("enable", "yes"),
                 new RandomScoreFunctionBuilder());
-        SearchResponse searchResponse = client().prepareSearch()
-            .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro"),
-                new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {filterBuilder}))
-            .highlighter(new HighlightBuilder()
-                .field(new Field("text")))
-            .get();
-        assertHitCount(searchResponse, 1);
-        HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
-        assertThat(field.getFragments().length, equalTo(1));
-        assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+
+        for (String type : UNIFIED_AND_NULL) {
+            SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(new FunctionScoreQueryBuilder(QueryBuilders.prefixQuery("text", "bro"),
+                    new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{filterBuilder}))
+                .highlighter(new HighlightBuilder()
+                    .field(new Field("text")).highlighterType(type))
+                .get();
+            assertHitCount(searchResponse, 1);
+            HighlightField field = searchResponse.getHits().getAt(0).highlightFields().get("text");
+            assertThat(field.getFragments().length, equalTo(1));
+            assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
+        }
     }
 
     public void testSynonyms() throws IOException {
@@ -2944,7 +2983,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "0").setSource(
             "field1", "The quick brown fox jumps over the lazy dog").get();
         refresh();
-        for (String highlighterType : new String[] {"plain", "postings", "fvh"}) {
+        for (String highlighterType : ALL_TYPES) {
             logger.info("--> highlighting (type=" + highlighterType + ") and searching on field1");
             SearchSourceBuilder source = searchSource()
                 .query(matchQuery("field1", "quick brown fox").operator(Operator.AND))
diff --git a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
index 236d5ce1ab8..a952ae86396 100644
--- a/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -689,8 +690,8 @@ public class SearchFieldsIT extends ESIntegTestCase {
                 .endArray()
                 .endObject().bytes();
 
-        client().prepareIndex("my-index", "my-type1", "1").setSource(source).get();
-        client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source).get();
+        client().prepareIndex("my-index", "my-type1", "1").setSource(source, XContentType.JSON).get();
+        client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source, XContentType.JSON).get();
 
 
         String field = "field1.field2.field3.field4";
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index 2707e3c0b27..568c723f8d1 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -47,6 +47,7 @@ import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
@@ -230,7 +231,7 @@ public class GeoFilterIT extends ESIntegTestCase {
                                 new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close()));
         BytesReference data = jsonBuilder().startObject().field("area", polygon).endObject().bytes();
 
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // Point in polygon
@@ -292,7 +293,7 @@ public class GeoFilterIT extends ESIntegTestCase {
                             new CoordinatesBuilder().coordinate(-4, -4).coordinate(-4, 4).coordinate(4, 4).coordinate(4, -4).close()));
 
         data = jsonBuilder().startObject().field("area", inverse).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "2").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "2").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // re-check point on polygon hole
@@ -326,7 +327,7 @@ public class GeoFilterIT extends ESIntegTestCase {
                 .coordinate(170, -10).coordinate(190, -10).coordinate(190, 10).coordinate(170, 10).close());
 
         data = jsonBuilder().startObject().field("area", builder).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         // Create a polygon crossing longitude 180 with hole.
@@ -335,7 +336,7 @@ public class GeoFilterIT extends ESIntegTestCase {
                     .hole(new LineStringBuilder(new CoordinatesBuilder().coordinate(175, -5).coordinate(185, -5).coordinate(185, 5).coordinate(175, 5).close()));
 
         data = jsonBuilder().startObject().field("area", builder).endObject().bytes();
-        client().prepareIndex("shapes", "polygon", "1").setSource(data).execute().actionGet();
+        client().prepareIndex("shapes", "polygon", "1").setSource(data, XContentType.JSON).execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         result = client().prepareSearch()
@@ -384,7 +385,7 @@ public class GeoFilterIT extends ESIntegTestCase {
 
         client().admin().indices().prepareCreate("countries").setSettings(settings)
                 .addMapping("country", xContentBuilder.string()).execute().actionGet();
-        BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null).execute().actionGet();
+        BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, null, xContentBuilder.contentType()).get();
 
         for (BulkItemResponse item : bulk.getItems()) {
             assertFalse("unable to index data", item.isFailed());
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index 067c2dcac05..7e8aa1ad9d0 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.geo;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.locationtech.spatial4j.shape.Rectangle;
 import com.vividsolutions.jts.geom.Coordinate;
 
@@ -69,7 +70,8 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet();
         ensureGreen();
 
-        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}", XContentType.JSON)
+            .setRefreshPolicy(IMMEDIATE).get();
         GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet();
         assertThat(result.getField("location"), nullValue());
     }
@@ -252,8 +254,8 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .setSource(
                         String.format(
                                 Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location
-                        )
-                ).setRefreshPolicy(IMMEDIATE).get();
+                        ), XContentType.JSON)
+            .setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("test", "type", "1")
                 .setSource(jsonBuilder().startObject().startObject("location")
                         .field("type", "polygon")
diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
index 2cb425d5274..ea2fb1d0db3 100644
--- a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
+++ b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
@@ -19,48 +19,170 @@
 
 package org.elasticsearch.search.internal;
 
+import org.apache.lucene.search.Explanation;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightFieldTests;
+import org.elasticsearch.search.internal.InternalSearchHit.InternalNestedIdentity;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.RandomObjects;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class InternalSearchHitTests extends ESTestCase {
 
+    private static Set<String> META_FIELDS = Sets.newHashSet("_uid", "_all", "_parent", "_routing", "_size", "_timestamp", "_ttl");
+
+    public static InternalSearchHit createTestItem(boolean withOptionalInnerHits) {
+        int internalId = randomInt();
+        String uid = randomAsciiOfLength(10);
+        Text type = new Text(randomAsciiOfLengthBetween(5, 10));
+        InternalNestedIdentity nestedIdentity = null;
+        if (randomBoolean()) {
+            nestedIdentity = InternalNestedIdentityTests.createTestItem(randomIntBetween(0, 2));
+        }
+        Map<String, SearchHitField> fields = new HashMap<>();
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 10);
+            for (int i = 0; i < size; i++) {
+                Tuple<List<Object>, List<Object>> values = RandomObjects.randomStoredFieldValues(random(),
+                        XContentType.JSON);
+                if (randomBoolean()) {
+                    String metaField = randomFrom(META_FIELDS);
+                    fields.put(metaField, new InternalSearchHitField(metaField, values.v1()));
+                } else {
+                    String fieldName = randomAsciiOfLengthBetween(5, 10);
+                    fields.put(fieldName, new InternalSearchHitField(fieldName, values.v1()));
+                }
+            }
+        }
+        InternalSearchHit hit = new InternalSearchHit(internalId, uid, type, nestedIdentity, fields);
+        if (frequently()) {
+            if (rarely()) {
+                hit.score(Float.NaN);
+            } else {
+                hit.score(randomFloat());
+            }
+        }
+        if (frequently()) {
+            hit.sourceRef(RandomObjects.randomSource(random()));
+        }
+        if (randomBoolean()) {
+            hit.version(randomLong());
+        }
+        if (randomBoolean()) {
+            hit.sortValues(SearchSortValuesTests.createTestItem());
+        }
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 5);
+            Map<String, HighlightField> highlightFields = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                highlightFields.put(randomAsciiOfLength(5), HighlightFieldTests.createTestItem());
+            }
+            hit.highlightFields(highlightFields);
+        }
+        if (randomBoolean()) {
+            int size = randomIntBetween(0, 5);
+            String[] matchedQueries = new String[size];
+            for (int i = 0; i < size; i++) {
+                matchedQueries[i] = randomAsciiOfLength(5);
+            }
+            hit.matchedQueries(matchedQueries);
+        }
+        if (randomBoolean()) {
+            hit.explanation(createExplanation(randomIntBetween(0, 5)));
+        }
+        if (withOptionalInnerHits) {
+            int innerHitsSize = randomIntBetween(0, 3);
+            Map<String, InternalSearchHits> innerHits = new HashMap<>(innerHitsSize);
+            for (int i = 0; i < innerHitsSize; i++) {
+                innerHits.put(randomAsciiOfLength(5), InternalSearchHitsTests.createTestItem());
+            }
+            hit.setInnerHits(innerHits);
+        }
+        if (randomBoolean()) {
+            hit.shard(new SearchShardTarget(randomAsciiOfLengthBetween(5, 10),
+                    new ShardId(new Index(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10)), randomInt())));
+        }
+        return hit;
+    }
+
+    public void testFromXContent() throws IOException {
+        InternalSearchHit searchHit = createTestItem(true);
+        boolean humanReadable = randomBoolean();
+        XContentType xContentType = randomFrom(XContentType.values());
+        BytesReference originalBytes = toXContent(searchHit, xContentType, humanReadable);
+
+        InternalSearchHit parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            parser.nextToken(); // jump to first START_OBJECT
+            parsed = InternalSearchHit.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        InternalSearchHit internalSearchHit = new InternalSearchHit(1, "id1", new Text("type"), Collections.emptyMap());
+        internalSearchHit.score(1.5f);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        internalSearchHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":1.5}", builder.string());
+    }
+
     public void testSerializeShardTarget() throws Exception {
         SearchShardTarget target = new SearchShardTarget("_node_id", new Index("_index", "_na_"), 0);
 
         Map<String, InternalSearchHits> innerHits = new HashMap<>();
         InternalSearchHit innerHit1 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit1.shardTarget(target);
+        innerHit1.shard(target);
         InternalSearchHit innerInnerHit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerInnerHit2.shardTarget(target);
+        innerInnerHit2.shard(target);
         innerHits.put("1", new InternalSearchHits(new InternalSearchHit[]{innerInnerHit2}, 1, 1f));
         innerHit1.setInnerHits(innerHits);
         InternalSearchHit innerHit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit2.shardTarget(target);
+        innerHit2.shard(target);
         InternalSearchHit innerHit3 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        innerHit3.shardTarget(target);
+        innerHit3.shard(target);
 
         innerHits = new HashMap<>();
         InternalSearchHit hit1 = new InternalSearchHit(0, "_id", new Text("_type"), null);
         innerHits.put("1", new InternalSearchHits(new InternalSearchHit[]{innerHit1, innerHit2}, 1, 1f));
         innerHits.put("2", new InternalSearchHits(new InternalSearchHit[]{innerHit3}, 1, 1f));
-        hit1.shardTarget(target);
+        hit1.shard(target);
         hit1.setInnerHits(innerHits);
 
         InternalSearchHit hit2 = new InternalSearchHit(0, "_id", new Text("_type"), null);
-        hit2.shardTarget(target);
+        hit2.shard(target);
 
         InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[]{hit1, hit2}, 2, 1f);
 
@@ -94,4 +216,17 @@ public class InternalSearchHitTests extends ESTestCase {
         searchHit.sourceRef(new BytesArray("{}"));
         assertTrue(searchHit.hasSource());
     }
+
+    private static Explanation createExplanation(int depth) {
+        String description = randomAsciiOfLengthBetween(5, 20);
+        float value = randomFloat();
+        List<Explanation> details = new ArrayList<>();
+        if (depth > 0) {
+            int numberOfDetails = randomIntBetween(1, 3);
+            for (int i = 0; i < numberOfDetails; i++) {
+                details.add(createExplanation(depth - 1));
+            }
+        }
+        return Explanation.match(value, description, details);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitsTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitsTests.java
new file mode 100644
index 00000000000..2474893edf5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitsTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class InternalSearchHitsTests extends ESTestCase {
+
+    public static InternalSearchHits createTestItem() {
+        int searchHits = randomIntBetween(0, 5);
+        InternalSearchHit[] hits = new InternalSearchHit[searchHits];
+        for (int i = 0; i < searchHits; i++) {
+            hits[i] = InternalSearchHitTests.createTestItem(false); // creating random innerHits could create loops
+        }
+        long totalHits = randomLong();
+        float maxScore = frequently() ? randomFloat() : Float.NaN;
+        return new InternalSearchHits(hits, totalHits, maxScore);
+    }
+
+    public void testFromXContent() throws IOException {
+        InternalSearchHits searchHits = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(searchHits, xcontentType, humanReadable);
+        InternalSearchHits parsed;
+        try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
+            parsed = InternalSearchHits.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xcontentType, humanReadable), xcontentType);
+    }
+
+    public void testToXContent() throws IOException {
+        InternalSearchHit[] hits = new InternalSearchHit[] {
+                new InternalSearchHit(1, "id1", new Text("type"), Collections.emptyMap()),
+                new InternalSearchHit(2, "id2", new Text("type"), Collections.emptyMap()) };
+
+        long totalHits = 1000;
+        float maxScore = 1.5f;
+        InternalSearchHits searchHits = new InternalSearchHits(hits, totalHits, maxScore);
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        searchHits.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"hits\":{\"total\":1000,\"max_score\":1.5," +
+                "\"hits\":[{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":\"-Infinity\"},"+
+                          "{\"_type\":\"type\",\"_id\":\"id2\",\"_score\":\"-Infinity\"}]}}", builder.string());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/SearchSortValuesTests.java b/core/src/test/java/org/elasticsearch/search/internal/SearchSortValuesTests.java
new file mode 100644
index 00000000000..3cff30cab1c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/internal/SearchSortValuesTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.internal;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class SearchSortValuesTests extends ESTestCase {
+
+    public static SearchSortValues createTestItem() {
+        List<Supplier<Object>> valueSuppliers = new ArrayList<>();
+        // this should reflect all values that are allowed to go through the transport layer
+        valueSuppliers.add(() -> null);
+        valueSuppliers.add(() -> randomInt());
+        valueSuppliers.add(() -> randomLong());
+        valueSuppliers.add(() -> randomDouble());
+        valueSuppliers.add(() -> randomFloat());
+        valueSuppliers.add(() -> randomByte());
+        valueSuppliers.add(() -> randomShort());
+        valueSuppliers.add(() -> randomBoolean());
+        valueSuppliers.add(() -> frequently() ? randomAsciiOfLengthBetween(1, 30) : randomRealisticUnicodeOfCodepointLength(30));
+
+        int size = randomIntBetween(1, 20);
+        Object[] values = new Object[size];
+        for (int i = 0; i < size; i++) {
+            Supplier<Object> supplier = randomFrom(valueSuppliers);
+            values[i] = supplier.get();
+        }
+        return new SearchSortValues(values);
+    }
+
+    public void testFromXContent() throws IOException {
+        SearchSortValues sortValues = createTestItem();
+        XContentType xcontentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(sortValues, xcontentType, humanReadable);
+
+        SearchSortValues parsed;
+        try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
+            parser.nextToken(); // skip to the elements start array token, fromXContent advances from there if called
+            parser.nextToken();
+            parser.nextToken();
+            parsed = SearchSortValues.fromXContent(parser);
+            parser.nextToken();
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xcontentType, humanReadable), xcontentType);
+    }
+
+    public void testToXContent() throws IOException {
+        SearchSortValues sortValues = new SearchSortValues(new Object[]{ 1, "foo", 3.0});
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        sortValues.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"sort\":[1,\"foo\",3.0]}", builder.string());
+    }
+
+    /**
+     * Test equality and hashCode properties
+     */
+    public void testEqualsAndHashcode() {
+        checkEqualsAndHashCode(createTestItem(), SearchSortValuesTests::copy, SearchSortValuesTests::mutate);
+    }
+
+    public void testSerialization() throws IOException {
+        SearchSortValues sortValues = createTestItem();
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            sortValues.writeTo(output);
+            try (StreamInput in = output.bytes().streamInput()) {
+                SearchSortValues deserializedCopy = new SearchSortValues(in);
+                assertEquals(sortValues, deserializedCopy);
+                assertEquals(sortValues.hashCode(), deserializedCopy.hashCode());
+                assertNotSame(sortValues, deserializedCopy);
+            }
+        }
+    }
+
+    private static SearchSortValues mutate(SearchSortValues original) {
+        Object[] sortValues = original.sortValues();
+        if (sortValues.length == 0) {
+            return new SearchSortValues(new Object[] { 1 });
+        }
+        return new SearchSortValues(Arrays.copyOf(sortValues, sortValues.length + 1));
+    }
+
+    private static SearchSortValues copy(SearchSortValues original) {
+        return new SearchSortValues(Arrays.copyOf(original.sortValues(), original.sortValues().length));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
index be7e69ee76d..d0132cca7ad 100644
--- a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java
@@ -23,8 +23,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -48,7 +48,6 @@ import org.elasticsearch.search.AbstractSearchTestCase;
 
 import java.io.IOException;
 import java.util.Base64;
-import java.util.function.Function;
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.Matchers.containsString;
@@ -163,9 +162,12 @@ public class ShardSearchTransportRequestTests extends AbstractSearchTestCase {
     }
 
     public QueryBuilder aliasFilter(IndexMetaData indexMetaData, String... aliasNames) {
-        Function<XContentParser, QueryParseContext> contextFactory = (p) -> new QueryParseContext(queriesRegistry,
-            p, new ParseFieldMatcher(Settings.EMPTY));
-        return ShardSearchRequest.parseAliasFilter(contextFactory, indexMetaData, aliasNames);
+        CheckedFunction<byte[], QueryBuilder, IOException> filterParser = bytes -> {
+            try (XContentParser parser = XContentFactory.xContent(bytes).createParser(xContentRegistry(), bytes)) {
+                return new QueryParseContext(parser).parseInnerQueryBuilder();
+            }
+        };
+        return ShardSearchRequest.parseAliasFilter(filterParser, indexMetaData, aliasNames);
     }
 
     // BWC test for changes from #20916
@@ -196,9 +198,9 @@ public class ShardSearchTransportRequestTests extends AbstractSearchTestCase {
                 .putAlias(AliasMetaData.newAliasMetaDataBuilder("UjLlLkjwWh").filter("{\"term\" : {\"foo\" : \"bar1\"}}"))
                 .putAlias(AliasMetaData.newAliasMetaDataBuilder("uBpgtwuqDG").filter("{\"term\" : {\"foo\" : \"bar2\"}}"));
             IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY);
-            final long nowInMillis = randomPositiveLong();
+            final long nowInMillis = randomNonNegativeLong();
             QueryShardContext context = new QueryShardContext(
-                0, indexSettings, null, null, null, null, null, queriesRegistry, null, null, () -> nowInMillis);
+                0, indexSettings, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis);
             readRequest.rewrite(context);
             QueryBuilder queryBuilder = readRequest.filteringAliases();
             assertEquals(queryBuilder, QueryBuilders.boolQuery()
diff --git a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index 1fd51966c2b..ab42a09a6fd 100644
--- a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -263,9 +263,9 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         assertThrows(client().prepareSearch().setQuery(
                 new MoreLikeThisQueryBuilder(new String[] {"string_value", "int_value"}, null, new Item[] {new Item("test", "type", "1")}).minTermFreq(1).minDocFreq(1)), SearchPhaseExecutionException.class);
 
-        // mlt query with no field -> OK
+        // mlt query with no field -> No results (because _all is not enabled)
         searchResponse = client().prepareSearch().setQuery(moreLikeThisQuery(new String[] {"index"}).minTermFreq(1).minDocFreq(1)).execute().actionGet();
-        assertHitCount(searchResponse, 2L);
+        assertHitCount(searchResponse, 0L);
 
         // mlt query with string fields
         searchResponse = client().prepareSearch().setQuery(moreLikeThisQuery(new String[]{"string_value"}, new String[] {"index"}, null).minTermFreq(1).minDocFreq(1)).execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
index ccc3102f3a8..c7e78f2cc28 100644
--- a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.msearch;
 
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -59,7 +60,7 @@ public class MultiSearchIT extends ESIntegTestCase {
         createIndex("test");
         int numDocs = randomIntBetween(0, 16);
         for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get();
+            client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}", XContentType.JSON).get();
         }
         refresh();
 
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 73b512ddf90..a7a59db4d4d 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -56,11 +56,13 @@ import static org.hamcrest.Matchers.startsWith;
 
 public class SimpleNestedIT extends ESIntegTestCase {
     public void testSimpleNested() throws Exception {
-        assertAcked(prepareCreate("test").addMapping("type1", "nested1", "type=nested").addMapping("type2", "nested1", "type=nested"));
+        assertAcked(prepareCreate("test")
+                .addMapping("type1", "nested1", "type=nested")
+                .addMapping("type2", "nested1", "type=nested"));
         ensureGreen();
 
         // check on no data, see it works
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(termQuery("_all", "n_value1_1")).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch("test").execute().actionGet();
         assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
         searchResponse = client().prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")).execute().actionGet();
         assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
@@ -89,9 +91,6 @@ public class SimpleNestedIT extends ESIntegTestCase {
         // check the numDocs
         assertDocumentCount("test", 3);
 
-        // check that _all is working on nested docs
-        searchResponse = client().prepareSearch("test").setQuery(termQuery("_all", "n_value1_1")).execute().actionGet();
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         searchResponse = client().prepareSearch("test").setQuery(termQuery("n_field1", "n_value1_1")).execute().actionGet();
         assertThat(searchResponse.getHits().totalHits(), equalTo(0L));
 
diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
index 3758715cbac..4b3b890a20d 100644
--- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
@@ -27,6 +27,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -90,15 +91,15 @@ public class SearchPreferenceIT extends ESIntegTestCase {
 
         final Client client = internalCluster().smartClient();
         SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).execute().actionGet();
-        String firstNodeId = searchResponse.getHits().getAt(0).shard().nodeId();
+        String firstNodeId = searchResponse.getHits().getAt(0).shard().getNodeId();
         searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).execute().actionGet();
-        String secondNodeId = searchResponse.getHits().getAt(0).shard().nodeId();
+        String secondNodeId = searchResponse.getHits().getAt(0).shard().getNodeId();
 
         assertThat(firstNodeId, not(equalTo(secondNodeId)));
     }
 
     public void testSimplePreference() throws Exception {
-        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}").get();
+        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get();
         ensureGreen();
 
         client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
@@ -131,7 +132,7 @@ public class SearchPreferenceIT extends ESIntegTestCase {
     }
 
     public void testReplicaPreference() throws Exception {
-        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}").get();
+        client().admin().indices().prepareCreate("test").setSettings("{\"number_of_replicas\": 0}", XContentType.JSON).get();
         ensureGreen();
 
         client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
@@ -147,7 +148,7 @@ public class SearchPreferenceIT extends ESIntegTestCase {
         SearchResponse resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica_first").execute().actionGet();
         assertThat(resp.getHits().totalHits(), equalTo(1L));
 
-        client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}").get();
+        client().admin().indices().prepareUpdateSettings("test").setSettings("{\"number_of_replicas\": 1}", XContentType.JSON).get();
         ensureGreen("test");
 
         resp = client().prepareSearch().setQuery(matchAllQuery()).setPreference("_replica").execute().actionGet();
@@ -220,7 +221,7 @@ public class SearchPreferenceIT extends ESIntegTestCase {
         for (int i = 0; i < 2; i++) {
             SearchResponse searchResponse = request.execute().actionGet();
             assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
-            hitNodes.add(searchResponse.getHits().getAt(0).shard().nodeId());
+            hitNodes.add(searchResponse.getHits().getAt(0).shard().getNodeId());
         }
         assertThat(hitNodes.size(), greaterThan(1));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
new file mode 100644
index 00000000000..fddbd6eaaef
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/ProfileResultTests.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class ProfileResultTests extends ESTestCase {
+
+    public static ProfileResult createTestItem(int depth) {
+        String type = randomAsciiOfLengthBetween(5, 10);
+        String description = randomAsciiOfLengthBetween(5, 10);
+        int timingsSize = randomIntBetween(0, 5);
+        Map<String, Long> timings = new HashMap<>(timingsSize);
+        for (int i = 0; i < timingsSize; i++) {
+            long time = randomNonNegativeLong() / timingsSize;
+            if (randomBoolean()) {
+                // also often use "small" values in tests
+                time = randomNonNegativeLong() % 10000;
+            }
+            timings.put(randomAsciiOfLengthBetween(5, 10), time); // don't overflow Long.MAX_VALUE;
+        }
+        int childrenSize = depth > 0 ? randomIntBetween(0, 1) : 0;
+        List<ProfileResult> children = new ArrayList<>(childrenSize);
+        for (int i = 0; i < childrenSize; i++) {
+            children.add(createTestItem(depth - 1));
+        }
+        return new ProfileResult(type, description, timings, children);
+    }
+
+    public void testFromXContent() throws IOException {
+        ProfileResult profileResult = createTestItem(2);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(profileResult, xContentType, humanReadable);
+        ProfileResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = ProfileResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertEquals(profileResult.getTime(), parsed.getTime());
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        List<ProfileResult> children = new ArrayList<>();
+        children.add(new ProfileResult("child1", "desc1", Collections.singletonMap("key1", 100L), Collections.emptyList()));
+        children.add(new ProfileResult("child2", "desc2", Collections.singletonMap("key1", 123356L), Collections.emptyList()));
+        Map<String, Long> timings3 = new HashMap<>();
+        timings3.put("key1", 123456L);
+        timings3.put("key2", 100000L);
+        ProfileResult result = new ProfileResult("someType", "some description", timings3, children);
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"someType\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time_in_nanos\" : 223456,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 123456,\n" +
+                "    \"key2\" : 100000\n" +
+                "  },\n" +
+                "  \"children\" : [\n" +
+                "    {\n" +
+                "      \"type\" : \"child1\",\n" +
+                "      \"description\" : \"desc1\",\n" +
+                "      \"time_in_nanos\" : 100,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 100\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"type\" : \"child2\",\n" +
+                "      \"description\" : \"desc2\",\n" +
+                "      \"time_in_nanos\" : 123356,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 123356\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+          "}", builder.string());
+
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"someType\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"223.4micros\",\n" +
+                "  \"time_in_nanos\" : 223456,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 123456,\n" +
+                "    \"key2\" : 100000\n" +
+                "  },\n" +
+                "  \"children\" : [\n" +
+                "    {\n" +
+                "      \"type\" : \"child1\",\n" +
+                "      \"description\" : \"desc1\",\n" +
+                "      \"time\" : \"100nanos\",\n" +
+                "      \"time_in_nanos\" : 100,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 100\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"type\" : \"child2\",\n" +
+                "      \"description\" : \"desc2\",\n" +
+                "      \"time\" : \"123.3micros\",\n" +
+                "      \"time_in_nanos\" : 123356,\n" +
+                "      \"breakdown\" : {\n" +
+                "        \"key1\" : 123356\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+          "}", builder.string());
+
+        result = new ProfileResult("profileName", "some description", Collections.singletonMap("key1", 12345678L), Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"profileName\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"12.3ms\",\n" +
+                "  \"time_in_nanos\" : 12345678,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 12345678\n" +
+                "  }\n" +
+              "}", builder.string());
+
+        result = new ProfileResult("profileName", "some description", Collections.singletonMap("key1", 1234567890L),
+                Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"type\" : \"profileName\",\n" +
+                "  \"description\" : \"some description\",\n" +
+                "  \"time\" : \"1.2s\",\n" +
+                "  \"time_in_nanos\" : 1234567890,\n" +
+                "  \"breakdown\" : {\n" +
+                "    \"key1\" : 1234567890\n" +
+                "  }\n" +
+              "}", builder.string());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java b/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java
new file mode 100644
index 00000000000..2d9cf847861
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResultTests;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;;
+
+public class SearchProfileShardResultsTests  extends ESTestCase {
+
+    public static SearchProfileShardResults createTestItem() {
+        int size = rarely() ? 0 : randomIntBetween(1, 2);
+        Map<String, ProfileShardResult> searchProfileResults = new HashMap<>(size);
+        for (int i = 0; i < size; i++) {
+            List<QueryProfileShardResult> queryProfileResults = new ArrayList<>();
+            int queryItems = rarely() ? 0 : randomIntBetween(1, 2);
+            for (int q = 0; q < queryItems; q++) {
+                queryProfileResults.add(QueryProfileShardResultTests.createTestItem());
+            }
+            AggregationProfileShardResult aggProfileShardResult = AggregationProfileShardResultTests.createTestItem(1);
+            searchProfileResults.put(randomAsciiOfLengthBetween(5, 10), new ProfileShardResult(queryProfileResults, aggProfileShardResult));
+        }
+        return new SearchProfileShardResults(searchProfileResults);
+    }
+
+    public void testFromXContent() throws IOException {
+        SearchProfileShardResults shardResult = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(shardResult, xContentType, humanReadable);
+        SearchProfileShardResults parsed = null;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            ensureFieldName(parser, parser.nextToken(), SearchProfileShardResults.PROFILE_FIELD);
+            ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            parsed = SearchProfileShardResults.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
new file mode 100644
index 00000000000..ca5f792e729
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResultTests.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the \"License\"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class AggregationProfileShardResultTests extends ESTestCase {
+
+    public static AggregationProfileShardResult createTestItem(int depth) {
+        int size = randomIntBetween(0, 5);
+        List<ProfileResult> aggProfileResults = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            aggProfileResults.add(ProfileResultTests.createTestItem(1));
+        }
+        return new AggregationProfileShardResult(aggProfileResults);
+    }
+
+    public void testFromXContent() throws IOException {
+        AggregationProfileShardResult profileResult = createTestItem(2);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(profileResult, xContentType, humanReadable);
+
+        AggregationProfileShardResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            XContentParserUtils.ensureFieldName(parser, parser.nextToken(), AggregationProfileShardResult.AGGREGATIONS);
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_ARRAY, parser::getTokenLocation);
+            parsed = AggregationProfileShardResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        List<ProfileResult> profileResults = new ArrayList<>();
+        Map<String, Long> timings = new HashMap<>();
+        timings.put("timing1", 2000L);
+        timings.put("timing2", 4000L);
+        ProfileResult profileResult = new ProfileResult("someType", "someDescription", timings, Collections.emptyList());
+        profileResults.add(profileResult);
+        AggregationProfileShardResult aggProfileResults = new AggregationProfileShardResult(profileResults);
+        BytesReference xContent = toXContent(aggProfileResults, XContentType.JSON, false);
+        assertEquals("{\"aggregations\":["
+                        + "{\"type\":\"someType\","
+                            + "\"description\":\"someDescription\","
+                            + "\"time_in_nanos\":6000,"
+                            + "\"breakdown\":{\"timing1\":2000,\"timing2\":4000}"
+                        + "}"
+                   + "]}", xContent.utf8ToString());
+
+        xContent = toXContent(aggProfileResults, XContentType.JSON, true);
+        assertEquals("{\"aggregations\":["
+                        + "{\"type\":\"someType\","
+                            + "\"description\":\"someDescription\","
+                            + "\"time\":\"6micros\","
+                            + "\"time_in_nanos\":6000,"
+                            + "\"breakdown\":{\"timing1\":2000,\"timing2\":4000}"
+                        + "}"
+                   + "]}", xContent.utf8ToString());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
new file mode 100644
index 00000000000..43e1e32db36
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/CollectorResultTests.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class CollectorResultTests extends ESTestCase {
+
+    public static CollectorResult createTestItem(int depth) {
+        String name = randomAsciiOfLengthBetween(5, 10);
+        String reason = randomAsciiOfLengthBetween(5, 10);
+        long time = randomNonNegativeLong();
+        if (randomBoolean()) {
+            // also often use relatively "small" values, otherwise we will mostly test huge longs
+            time = time % 100000;
+        }
+        int size = randomIntBetween(0, 5);
+        List<CollectorResult> children = new ArrayList<>(size);
+        if (depth > 0) {
+            for (int i = 0; i < size; i++) {
+                children.add(createTestItem(depth - 1));
+            }
+        }
+        return new CollectorResult(name, reason, time, children);
+    }
+
+    public void testFromXContent() throws IOException {
+        CollectorResult collectorResult = createTestItem(1);
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(collectorResult, xContentType, humanReadable);
+
+        CollectorResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
+            parsed = CollectorResult.fromXContent(parser);
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+    public void testToXContent() throws IOException {
+        List<CollectorResult> children = new ArrayList<>();
+        children.add(new CollectorResult("child1", "reason1", 100L, Collections.emptyList()));
+        children.add(new CollectorResult("child2", "reason1", 123356L, Collections.emptyList()));
+        CollectorResult result = new CollectorResult("collectorName", "some reason", 123456L, children);
+        XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+            "  \"name\" : \"collectorName\",\n" +
+            "  \"reason\" : \"some reason\",\n" +
+            "  \"time_in_nanos\" : 123456,\n" +
+            "  \"children\" : [\n" +
+            "    {\n" +
+            "      \"name\" : \"child1\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time_in_nanos\" : 100\n" +
+            "    },\n" +
+            "    {\n" +
+            "      \"name\" : \"child2\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time_in_nanos\" : 123356\n" +
+            "    }\n" +
+            "  ]\n" +
+          "}", builder.string());
+
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+            "  \"name\" : \"collectorName\",\n" +
+            "  \"reason\" : \"some reason\",\n" +
+            "  \"time\" : \"123.4micros\",\n" +
+            "  \"time_in_nanos\" : 123456,\n" +
+            "  \"children\" : [\n" +
+            "    {\n" +
+            "      \"name\" : \"child1\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time\" : \"100nanos\",\n" +
+            "      \"time_in_nanos\" : 100\n" +
+            "    },\n" +
+            "    {\n" +
+            "      \"name\" : \"child2\",\n" +
+            "      \"reason\" : \"reason1\",\n" +
+            "      \"time\" : \"123.3micros\",\n" +
+            "      \"time_in_nanos\" : 123356\n" +
+            "    }\n" +
+            "  ]\n" +
+          "}", builder.string());
+
+        result = new CollectorResult("collectorName", "some reason", 12345678L, Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"name\" : \"collectorName\",\n" +
+                "  \"reason\" : \"some reason\",\n" +
+                "  \"time\" : \"12.3ms\",\n" +
+                "  \"time_in_nanos\" : 12345678\n" +
+              "}", builder.string());
+
+        result = new CollectorResult("collectorName", "some reason", 1234567890L, Collections.emptyList());
+        builder = XContentFactory.jsonBuilder().prettyPrint().humanReadable(true);
+        result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertEquals("{\n" +
+                "  \"name\" : \"collectorName\",\n" +
+                "  \"reason\" : \"some reason\",\n" +
+                "  \"time\" : \"1.2s\",\n" +
+                "  \"time_in_nanos\" : 1234567890\n" +
+              "}", builder.string());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
new file mode 100644
index 00000000000..1542ce2eca0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfileShardResultTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileResultTests;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+
+public class QueryProfileShardResultTests extends ESTestCase {
+
+    public static QueryProfileShardResult createTestItem() {
+        int size = randomIntBetween(0, 5);
+        List<ProfileResult> queryProfileResults = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            queryProfileResults.add(ProfileResultTests.createTestItem(1));
+        }
+        CollectorResult profileCollector = CollectorResultTests.createTestItem(2);
+        long rewriteTime = randomNonNegativeLong();
+        if (randomBoolean()) {
+            rewriteTime = rewriteTime % 1000; // make sure to often test this with small values too
+        }
+        return new QueryProfileShardResult(queryProfileResults, rewriteTime, profileCollector);
+    }
+
+    public void testFromXContent() throws IOException {
+        QueryProfileShardResult profileResult = createTestItem();
+        XContentType xContentType = randomFrom(XContentType.values());
+        boolean humanReadable = randomBoolean();
+        BytesReference originalBytes = toXContent(profileResult, xContentType, humanReadable);
+
+        QueryProfileShardResult parsed;
+        try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
+            XContentParserUtils.ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
+            parsed = QueryProfileShardResult.fromXContent(parser);
+            assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+            assertNull(parser.nextToken());
+        }
+        assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
index c003038c7c2..1dbeb24d4ed 100644
--- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java
@@ -19,50 +19,53 @@
 
 package org.elasticsearch.search.query;
 
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
 public class QueryStringIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created
+    }
 
     @Before
     public void setup() throws Exception {
         String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json");
-        prepareCreate("test").setSource(indexBody).get();
+        prepareCreate("test").setSource(indexBody, XContentType.JSON).get();
         ensureGreen("test");
     }
 
@@ -149,7 +152,7 @@ public class QueryStringIT extends ESIntegTestCase {
     public void testDocWithAllTypes() throws Exception {
         List<IndexRequestBuilder> reqs = new ArrayList<>();
         String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody));
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
@@ -212,7 +215,7 @@ public class QueryStringIT extends ESIntegTestCase {
 
     public void testExplicitAllFieldsRequested() throws Exception {
         String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json");
-        prepareCreate("test2").setSource(indexBody).get();
+        prepareCreate("test2").setSource(indexBody, XContentType.JSON).get();
         ensureGreen("test2");
 
         List<IndexRequestBuilder> reqs = new ArrayList<>();
@@ -263,6 +266,92 @@ public class QueryStringIT extends ESIntegTestCase {
                 containsString("unit [D] not supported for date math [-2D]"));
     }
 
+    private void setupIndexWithGraph(String index) throws Exception {
+        CreateIndexRequestBuilder builder = prepareCreate(index).setSettings(
+            Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.filter.graphsyns.type", "synonym_graph")
+                .putArray("index.analysis.filter.graphsyns.synonyms", "wtf, what the fudge", "foo, bar baz")
+                .put("index.analysis.analyzer.lower_graphsyns.type", "custom")
+                .put("index.analysis.analyzer.lower_graphsyns.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.lower_graphsyns.filter", "lowercase", "graphsyns")
+        );
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject(index).startObject("properties")
+            .startObject("field").field("type", "text").endObject().endObject().endObject().endObject();
+
+        assertAcked(builder.addMapping(index, mapping));
+        ensureGreen();
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        builders.add(client().prepareIndex(index, index, "1").setSource("field", "say wtf happened foo"));
+        builders.add(client().prepareIndex(index, index, "2").setSource("field", "bar baz what the fudge man"));
+        builders.add(client().prepareIndex(index, index, "3").setSource("field", "wtf"));
+        builders.add(client().prepareIndex(index, index, "4").setSource("field", "what is the name for fudge"));
+        builders.add(client().prepareIndex(index, index, "5").setSource("field", "bar two three"));
+        builders.add(client().prepareIndex(index, index, "6").setSource("field", "bar baz two three"));
+
+        indexRandom(true, false, builders);
+    }
+
+    public void testGraphQueries() throws Exception {
+        String index = "graph_test_index";
+        setupIndexWithGraph(index);
+
+        // phrase
+        SearchResponse searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("\"foo two three\"")
+                .defaultField("field")
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "6");
+
+        // and
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("say what the fudge")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.AND)
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 1L);
+        assertSearchHits(searchResponse, "1");
+
+        // and, split on whitespace means we should not recognize the multi-word synonym
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("say what the fudge")
+                .defaultField("field")
+                .splitOnWhitespace(true)
+                .defaultOperator(Operator.AND)
+                .analyzer("lower_graphsyns")).get();
+
+        assertNoSearchHits(searchResponse);
+
+        // or
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("three what the fudge foo")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.OR)
+                .analyzer("lower_graphsyns")).get();
+
+        assertHitCount(searchResponse, 6L);
+        assertSearchHits(searchResponse, "1", "2", "3", "4", "5", "6");
+
+        // min should match
+        searchResponse = client().prepareSearch(index).setQuery(
+            QueryBuilders.queryStringQuery("three what the fudge foo")
+                .defaultField("field")
+                .splitOnWhitespace(false)
+                .defaultOperator(Operator.OR)
+                .analyzer("lower_graphsyns")
+                .minimumShouldMatch("80%")).get();
+
+        assertHitCount(searchResponse, 3L);
+        assertSearchHits(searchResponse, "1", "2", "6");
+    }
+
     private void assertHits(SearchHits hits, String... ids) {
         assertThat(hits.totalHits(), equalTo((long) ids.length));
         Set<String> hitIds = new HashSet<>();
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 4528ac3446c..7cb9627cf18 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
@@ -112,37 +113,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         return Math.min(2, cluster().numDataNodes() - 1);
     }
 
-    public void testOmitNormsOnAll() throws ExecutionException, InterruptedException, IOException {
-        assertAcked(prepareCreate("test")
-                .addMapping("type1", jsonBuilder().startObject().startObject("type1")
-                        .startObject("_all").field("norms", false).endObject()
-                        .endObject().endObject())
-                .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)); // only one shard otherwise IDF might be different for comparing scores
-
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox jumps"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "quick brown"),
-                client().prepareIndex("test", "type1", "3").setSource("field1", "quick"));
-
-        assertHitCount(client().prepareSearch().setQuery(matchQuery("_all", "quick")).get(), 3L);
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("_all", "quick")).setExplain(true).get();
-        SearchHit[] hits = searchResponse.getHits().hits();
-        assertThat(hits.length, equalTo(3));
-        assertThat(hits[0].score(), allOf(equalTo(hits[1].getScore()), equalTo(hits[2].getScore())));
-        cluster().wipeIndices("test");
-
-        createIndex("test");
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox jumps"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "quick brown"),
-                client().prepareIndex("test", "type1", "3").setSource("field1", "quick"));
-
-        assertHitCount(client().prepareSearch().setQuery(matchQuery("_all", "quick")).get(), 3L);
-        searchResponse = client().prepareSearch().setQuery(matchQuery("_all", "quick")).get();
-        hits = searchResponse.getHits().hits();
-        assertThat(hits.length, equalTo(3));
-        assertThat(hits[0].score(), allOf(greaterThan(hits[1].getScore()), greaterThan(hits[2].getScore())));
-
-    }
-
     // see #3952
     public void testEmptyQueryString() throws ExecutionException, InterruptedException, IOException {
         createIndex("test");
@@ -284,20 +254,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         }
     }
 
-    public void testCommonTermsQueryOnAllField() throws Exception {
-        client().admin().indices().prepareCreate("test")
-                .addMapping("type1", "message", "type=text", "comment", "type=text,boost=5.0")
-                .setSettings(SETTING_NUMBER_OF_SHARDS, 1).get();
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("message", "test message", "comment", "whatever"),
-                client().prepareIndex("test", "type1", "2").setSource("message", "hello world", "comment", "test comment"));
-
-        SearchResponse searchResponse = client().prepareSearch().setQuery(commonTermsQuery("_all", "test")).get();
-        assertHitCount(searchResponse, 2L);
-        assertFirstHit(searchResponse, hasId("2"));
-        assertSecondHit(searchResponse, hasId("1"));
-        assertThat(searchResponse.getHits().getHits()[0].getScore(), greaterThan(searchResponse.getHits().getHits()[1].getScore()));
-    }
-
     public void testCommonTermsQuery() throws Exception {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field1", "type=text,analyzer=whitespace")
@@ -528,7 +484,7 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 1L);
 
         SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch()
-                .setQuery(queryStringQuery("future:[now/D TO now+2M/d]")).get());
+                .setQuery(queryStringQuery("future:[now/D TO now+2M/d]").lenient(false)).get());
         assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST));
         assertThat(e.toString(), containsString("unit [D] not supported for date math"));
     }
@@ -941,7 +897,7 @@ public class SearchQueryIT extends ESIntegTestCase {
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3));
+                .minimumShouldMatch(3));
         SearchResponse searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("1"));
@@ -951,9 +907,9 @@ public class SearchQueryIT extends ESIntegTestCase {
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(1))
+                .minimumShouldMatch(1))
             // Only one should clause is defined, returns no docs.
-            .minimumNumberShouldMatch(2);
+            .minimumShouldMatch(2);
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 0L);
 
@@ -962,8 +918,8 @@ public class SearchQueryIT extends ESIntegTestCase {
             .should(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3))
-            .minimumNumberShouldMatch(1);
+                .minimumShouldMatch(3))
+            .minimumShouldMatch(1);
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("1"));
@@ -973,7 +929,7 @@ public class SearchQueryIT extends ESIntegTestCase {
             .must(boolQuery()
                 .should(termQuery("field1", "value1"))
                 .should(termQuery("field1", "value2"))
-                .minimumNumberShouldMatch(3));
+                .minimumShouldMatch(3));
         searchResponse = client().prepareSearch().setQuery(boolQuery).get();
         assertHitCount(searchResponse, 0L);
     }
@@ -1698,23 +1654,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 1L);
     }
 
-    public void testAllFieldEmptyMapping() throws Exception {
-        client().prepareIndex("myindex", "mytype").setId("1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
-        SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
-        assertNoFailures(response);
-    }
-
-    public void testAllDisabledButQueried() throws Exception {
-        createIndex("myindex");
-        assertAcked(client().admin().indices().preparePutMapping("myindex").setType("mytype").setSource(
-                jsonBuilder().startObject().startObject("mytype").startObject("_all").field("enabled", false)
-                .endObject().endObject().endObject()));
-        client().prepareIndex("myindex", "mytype").setId("1").setSource("bar", "foo").setRefreshPolicy(IMMEDIATE).get();
-        SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
-        assertNoFailures(response);
-        assertHitCount(response, 0);
-    }
-
     public void testMinScore() throws ExecutionException, InterruptedException {
         createIndex("test");
 
@@ -1858,8 +1797,8 @@ public class SearchQueryIT extends ESIntegTestCase {
     }
 
     public void testSearchEmptyDoc() {
-        assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}"));
-        client().prepareIndex("test", "type1", "1").setSource("{}").get();
+        assertAcked(prepareCreate("test").setSettings("{\"index.analysis.analyzer.default.type\":\"keyword\"}", XContentType.JSON));
+        client().prepareIndex("test", "type1", "1").setSource("{}", XContentType.JSON).get();
 
         refresh();
         assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index 41085a9901e..1bec4bb20bf 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -21,21 +21,28 @@ package org.elasticsearch.search.query;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.SimpleQueryStringFlag;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -61,6 +68,11 @@ import static org.hamcrest.Matchers.equalTo;
  * Tests for the {@code simple_query_string} query
  */
 public class SimpleQueryStringIT extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created
+    }
+
     public void testSimpleQueryString() throws ExecutionException, InterruptedException {
         createIndex("test");
         indexRandom(true, false,
@@ -252,8 +264,8 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         searchResponse = client()
                 .prepareSearch()
                 .setQuery(
-                        simpleQueryStringQuery("baz | egg*").defaultOperator(Operator.AND).flags(SimpleQueryStringFlag.WHITESPACE,
-                                SimpleQueryStringFlag.PREFIX)).get();
+                        simpleQueryStringQuery("quuz~1 + egg*").flags(SimpleQueryStringFlag.WHITESPACE, SimpleQueryStringFlag.AND,
+                                SimpleQueryStringFlag.FUZZY, SimpleQueryStringFlag.PREFIX)).get();
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("4"));
     }
@@ -460,7 +472,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
 
         List<IndexRequestBuilder> reqs = new ArrayList<>();
         String docBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-example-document.json");
-        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody));
+        reqs.add(client().prepareIndex("test", "doc", "1").setSource(docBody, XContentType.JSON));
         indexRandom(true, false, reqs);
 
         SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
@@ -525,7 +537,10 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
 
     public void testExplicitAllFieldsRequested() throws Exception {
         String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index-with-all.json");
-        prepareCreate("test").setSource(indexBody).get();
+        prepareCreate("test")
+                .setSource(indexBody)
+                // .setSettings(Settings.builder().put("index.version.created", Version.V_5_0_0.id)).get();
+                .get();
         ensureGreen("test");
 
         List<IndexRequestBuilder> reqs = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index 22f2dd53808..53277bd5b3c 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -23,10 +23,10 @@ import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -42,7 +42,6 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext;
 import org.elasticsearch.test.ESTestCase;
@@ -59,7 +58,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -68,13 +67,13 @@ public class QueryRescoreBuilderTests extends ESTestCase {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -119,7 +118,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
 
 
             XContentParser parser = createParser(shuffled);
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(parser);
             parser.nextToken();
             RescoreBuilder<?> secondRescoreBuilder = RescoreBuilder.parseFromXContent(context);
             assertNotSame(rescoreBuilder, secondRescoreBuilder);
@@ -133,12 +132,12 @@ public class QueryRescoreBuilderTests extends ESTestCase {
      * than the test builder
      */
     public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IOException {
-        final long nowInMillis = randomPositiveLong();
+        final long nowInMillis = randomNonNegativeLong();
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
-        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, indicesQueriesRegistry,
+        QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(),
                 null, null, () -> nowInMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
@@ -252,12 +251,17 @@ public class QueryRescoreBuilderTests extends ESTestCase {
      */
     private QueryParseContext createContext(String rescoreElement) throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, rescoreElement);
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         // move to first token, this is where the internal fromXContent
         assertTrue(parser.nextToken() == XContentParser.Token.START_OBJECT);
         return context;
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     private static RescoreBuilder<?> mutate(RescoreBuilder<?> original) throws IOException {
         RescoreBuilder<?> mutation = ESTestCase.copyWriteable(original, namedWriteableRegistry, QueryRescorerBuilder::new);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
index f05f9206836..70d043403e9 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/DuelScrollIT.java
@@ -97,38 +97,6 @@ public class DuelScrollIT extends ESIntegTestCase {
         clearScroll(scrollId);
     }
 
-    public void testDuelQueryAndFetch() throws Exception {
-        // *_QUERY_AND_FETCH search types are tricky: the ordering can be incorrect, since it returns num_shards * (from + size)
-        // a subsequent scroll call can return hits that should have been in the hits of the first scroll call.
-
-        TestContext context = create(SearchType.DFS_QUERY_AND_FETCH, SearchType.QUERY_AND_FETCH);
-        SearchResponse searchScrollResponse = client().prepareSearch("index")
-                .setSearchType(context.searchType)
-                .addSort(context.sort)
-                .setSize(context.scrollRequestSize)
-                .setScroll("10m").get();
-
-        assertNoFailures(searchScrollResponse);
-        assertThat(searchScrollResponse.getHits().getTotalHits(), equalTo((long) context.numDocs));
-
-        int counter = searchScrollResponse.getHits().hits().length;
-        String scrollId = searchScrollResponse.getScrollId();
-        while (true) {
-            searchScrollResponse = client().prepareSearchScroll(scrollId).setScroll("10m").get();
-            assertNoFailures(searchScrollResponse);
-            assertThat(searchScrollResponse.getHits().getTotalHits(), equalTo((long) context.numDocs));
-            if (searchScrollResponse.getHits().hits().length == 0) {
-                break;
-            }
-
-            counter += searchScrollResponse.getHits().hits().length;
-            scrollId = searchScrollResponse.getScrollId();
-        }
-
-        assertThat(counter, equalTo(context.numDocs));
-        clearScroll(scrollId);
-    }
-
 
     private TestContext create(SearchType... searchTypes) throws Exception {
         assertAcked(prepareCreate("index").addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
@@ -289,7 +257,7 @@ public class DuelScrollIT extends ESIntegTestCase {
     }
 
     public void testDuelIndexOrderQueryAndFetch() throws Exception {
-        final SearchType searchType = RandomPicks.randomFrom(random(), Arrays.asList(SearchType.QUERY_AND_FETCH, SearchType.DFS_QUERY_AND_FETCH));
+        final SearchType searchType = SearchType.QUERY_AND_FETCH;
         // QUERY_AND_FETCH only works with a single shard
         final int numDocs = createIndex(true);
         testDuelIndexOrder(searchType, false, numDocs);
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java
index d386f73adf5..ae8ad66ac8d 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/RestClearScrollActionTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestClearScrollAction;
@@ -48,7 +49,8 @@ public class RestClearScrollActionTests extends ESTestCase {
 
     public void testParseClearScrollRequestWithInvalidJsonThrowsException() throws Exception {
         RestClearScrollAction action = new RestClearScrollAction(Settings.EMPTY, mock(RestController.class));
-        RestRequest request = new FakeRestRequest.Builder().withContent(new BytesArray("{invalid_json}")).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
         Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null));
         assertThat(e.getMessage(), equalTo("Failed to parse request body"));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
index f4bb364516a..662bc07f90d 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/RestSearchScrollActionTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
@@ -52,7 +53,8 @@ public class RestSearchScrollActionTests extends ESTestCase {
 
     public void testParseSearchScrollRequestWithInvalidJsonThrowsException() throws Exception {
         RestSearchScrollAction action = new RestSearchScrollAction(Settings.EMPTY, mock(RestController.class));
-        RestRequest request = new FakeRestRequest.Builder().withContent(new BytesArray("{invalid_json}")).build();
+        RestRequest request = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray("{invalid_json}"), XContentType.JSON).build();
         Exception e = expectThrows(IllegalArgumentException.class, () -> action.prepareRequest(request, null));
         assertThat(e.getMessage(), equalTo("Failed to parse request body"));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index c8adf234514..1f368cb35da 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -520,12 +520,8 @@ public class SearchScrollIT extends ESIntegTestCase {
 
     private void assertToXContentResponse(ClearScrollResponse response, boolean succeed, int numFreed) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder();
-        builder.startObject();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder.endObject();
-
-        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false).v2();
-
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map.get("succeeded"), is(succeed));
         assertThat(map.get("num_freed"), equalTo(numFreed));
     }
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
index beb69a0184b..cb45208a39f 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.searchafter;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.text.Text;
@@ -28,14 +27,8 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -44,22 +37,6 @@ import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashC
 
 public class SearchAfterBuilderTests extends ESTestCase {
     private static final int NUMBER_OF_TESTBUILDERS = 20;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-
-    /**
-     * setup for the whole base test class
-     */
-    @BeforeClass
-    public static void init() {
-        indicesQueriesRegistry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(parser, MatchAllQueryBuilder.NAME);
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        indicesQueriesRegistry = null;
-    }
 
     private static SearchAfterBuilder randomSearchAfterBuilder() throws IOException {
         int numSearchFrom = randomIntBetween(1, 10);
@@ -154,7 +131,7 @@ public class SearchAfterBuilderTests extends ESTestCase {
         parser.nextToken();
         parser.nextToken();
         parser.nextToken();
-        return SearchAfterBuilder.fromXContent(parser, null);
+        return SearchAfterBuilder.fromXContent(parser);
     }
 
     private static SearchAfterBuilder serializedCopy(SearchAfterBuilder original) throws IOException {
@@ -189,11 +166,10 @@ public class SearchAfterBuilderTests extends ESTestCase {
             searchAfterBuilder.innerToXContent(builder);
             builder.endObject();
             XContentParser parser = createParser(shuffleXContent(builder));
-            new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
             parser.nextToken();
             parser.nextToken();
             parser.nextToken();
-            SearchAfterBuilder secondSearchAfterBuilder = SearchAfterBuilder.fromXContent(parser, null);
+            SearchAfterBuilder secondSearchAfterBuilder = SearchAfterBuilder.fromXContent(parser);
             assertNotSame(searchAfterBuilder, secondSearchAfterBuilder);
             assertEquals(searchAfterBuilder, secondSearchAfterBuilder);
             assertEquals(searchAfterBuilder.hashCode(), secondSearchAfterBuilder.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 95978991046..f63f13b6dd2 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
@@ -52,6 +53,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 import static org.hamcrest.Matchers.containsString;
 
 public class SimpleSearchIT extends ESIntegTestCase {
+
     public void testSearchNullIndex() {
         expectThrows(NullPointerException.class,
                 () -> client().prepareSearch((String) null).setQuery(QueryBuilders.termQuery("_id", "XXX1")).get());
@@ -288,7 +290,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
     public void testInsaneFromAndSize() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertWindowFails(client().prepareSearch("idx").setFrom(Integer.MAX_VALUE));
         assertWindowFails(client().prepareSearch("idx").setSize(Integer.MAX_VALUE));
@@ -296,7 +298,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
     public void testTooLargeFromAndSize() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)));
         assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1));
@@ -306,7 +308,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
     public void testLargeFromAndSizeSucceeds() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
@@ -317,7 +319,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
     public void testTooLargeFromAndSizeOkBySetting() throws Exception {
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
                 IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
@@ -332,7 +334,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
                         Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
                                 IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
@@ -342,7 +344,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
     public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception {
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
         assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
@@ -352,7 +354,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
     public void testTooLargeRescoreWindow() throws Exception {
         createIndex("idx");
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertRescoreWindowFails(Integer.MAX_VALUE);
         assertRescoreWindowFails(IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY) + 1);
@@ -362,7 +364,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
         int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY);
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(),
                 defaultMaxWindow * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -373,7 +375,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
         int defaultMaxWindow = IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY);
         prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), // Note that this is the RESULT window.
                 defaultMaxWindow * 2).get();
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -387,7 +389,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
                 .setSettings(
                         Settings.builder().put(IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey(), defaultMaxWindow * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -402,7 +404,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
                         // Note that this is the RESULT window
                         Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), defaultMaxWindow * 2))
                 .get());
-        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
+        indexRandom(true, client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
 
         assertHitCount(
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
@@ -417,7 +419,8 @@ public class SimpleSearchIT extends ESIntegTestCase {
             client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get();
             fail("SearchPhaseExecutionException should have been thrown");
         } catch (SearchPhaseExecutionException ex) {
-            assertThat(ex.getCause().getCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields"));
+            assertThat(ex.getRootCause().getMessage(),
+                containsString("Can only use regexp queries on keyword and text fields"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 52bc3bd4633..9de5ee2a583 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -30,7 +30,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -39,14 +38,9 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.UidFieldMapper;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -66,22 +60,6 @@ import static org.mockito.Mockito.when;
 
 public class SliceBuilderTests extends ESTestCase {
     private static final int MAX_SLICE = 20;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-
-    /**
-     * setup for the whole base test class
-     */
-    @BeforeClass
-    public static void init() {
-        indicesQueriesRegistry = new IndicesQueriesRegistry();
-        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(parser, MatchAllQueryBuilder.NAME);
-    }
-
-    @AfterClass
-    public static void afterClass() throws Exception {
-        indicesQueriesRegistry = null;
-    }
 
     private static SliceBuilder randomSliceBuilder() throws IOException {
         int max = randomIntBetween(2, MAX_SLICE);
@@ -125,8 +103,7 @@ public class SliceBuilderTests extends ESTestCase {
         sliceBuilder.innerToXContent(builder);
         builder.endObject();
         XContentParser parser = createParser(shuffleXContent(builder));
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser,
-            ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         SliceBuilder secondSliceBuilder = SliceBuilder.fromXContent(context);
         assertNotSame(sliceBuilder, secondSliceBuilder);
         assertEquals(sliceBuilder, secondSliceBuilder);
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index d545a082b55..17c60152aae 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -23,10 +23,9 @@ import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -51,7 +50,6 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
@@ -78,11 +76,11 @@ import static java.util.Collections.emptyList;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
 public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends ESTestCase {
+    private static final int NUMBER_OF_TESTBUILDERS = 20;
 
     protected static NamedWriteableRegistry namedWriteableRegistry;
 
-    private static final int NUMBER_OF_TESTBUILDERS = 20;
-    static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
     private static ScriptService scriptService;
 
     @BeforeClass
@@ -99,20 +97,20 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         scriptService = new ScriptService(baseSettings, environment,
                 new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
+            public CompiledScript compile(Script script, ScriptContext scriptContext) {
                 return new CompiledScript(ScriptType.INLINE, "mockName", "test", script);
             }
         };
 
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /** Returns random sort that is put under test */
@@ -147,7 +145,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
             String elementName = itemParser.currentName();
             itemParser.nextToken();
 
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+            QueryParseContext context = new QueryParseContext(itemParser);
             T parsedItem = fromXContent(context, elementName);
             assertNotSame(testItem, parsedItem);
             assertEquals(testItem, parsedItem);
@@ -209,9 +207,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
             public void onCache(ShardId shardId, Accountable accountable) {
             }
         });
-        long nowInMillis = randomPositiveLong();
+        long nowInMillis = randomNonNegativeLong();
         return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService,
-                indicesQueriesRegistry, null, null, () -> nowInMillis) {
+                xContentRegistry(), null, null, () -> nowInMillis) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 return provideMappedFieldType(name);
@@ -236,6 +234,11 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         return doubleFieldType;
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     protected static QueryBuilder randomNestedFilter() {
         int id = randomIntBetween(0, 2);
         switch(id) {
@@ -250,7 +253,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
 
     @SuppressWarnings("unchecked")
     private T copy(T original) throws IOException {
-        return copyWriteable(original, namedWriteableRegistry,
-                (Writeable.Reader<T>) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()));
+        /* The cast below is required to make Java 9 happy. Java 8 infers the T in copyWriterable to be the same as AbstractSortTestCase's
+         * T but Java 9 infers it to be SortBuilder. */
+        return (T) copyWriteable(original, namedWriteableRegistry,
+                namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index 60b9f14c368..111d4256eb3 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -20,7 +20,6 @@ x * Licensed to Elasticsearch under one or more contributor
 package org.elasticsearch.search.sort;
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -134,7 +133,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
 
         try {
           FieldSortBuilder.fromXContent(context, "");
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 84dd3dabf6b..76595765558 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
@@ -91,7 +92,7 @@ public class FieldSortIT extends ESIntegTestCase {
                 assertAcked(prepareCreate("test_" + i).addAlias(new Alias("test")));
             }
             if (i > 0) {
-                client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}").get();
+                client().prepareIndex("test_" + i, "foo", "" + i).setSource("{\"entry\": " + i + "}", XContentType.JSON).get();
             }
         }
         refresh();
@@ -422,9 +423,9 @@ public class FieldSortIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .addMapping("type", "field1", "type=keyword").get());
 
-        client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}").execute().actionGet();
-        client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}").execute().actionGet();
-        client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}").execute().actionGet();
+        client().prepareIndex("test", "post", "1").setSource("{\"field1\":\"value1\"}", XContentType.JSON).execute().actionGet();
+        client().prepareIndex("test", "post", "2").setSource("{\"field1\":\"value2\"}", XContentType.JSON).execute().actionGet();
+        client().prepareIndex("test", "post", "3").setSource("{\"field1\":\"value3\"}", XContentType.JSON).execute().actionGet();
         refresh();
         SearchResponse result = client().prepareSearch("test").setQuery(matchAllQuery()).setTrackScores(true)
                 .addSort("field1", SortOrder.ASC).execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index 707f646282b..6529b990255 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -99,32 +99,32 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.DESC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 5, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 3, 2, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.DESC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 1, 6, 2, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 4, 1, DistanceUnit.METERS), 10d));
     }
 
     public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedException, IOException {
@@ -157,16 +157,16 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.AVG).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MEDIAN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
-        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 4, DistanceUnit.METERS), 10d));
+        assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(0, 0, 0, 5, DistanceUnit.METERS), 10d));
     }
 
     protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException {
@@ -238,16 +238,16 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
                 .addSort(geoDistanceSortBuilder.sortMode(SortMode.MIN).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(4.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(4.5, 1, 2, 1, DistanceUnit.METERS), 1.e-1));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(geoDistanceSortBuilder.sortMode(SortMode.MAX).order(SortOrder.ASC))
                 .execute().actionGet();
         assertOrderedSearchHits(searchResponse, "d1", "d2");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(3.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(5.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(3.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(5.25, 4, 2, 1, DistanceUnit.METERS), 1.e-1));
 
     }
 
@@ -314,8 +314,8 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
 
     private static void checkCorrectSortOrderForGeoSort(SearchResponse searchResponse) {
         assertOrderedSearchHits(searchResponse, "d2", "d1");
-        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 1, 2, DistanceUnit.METERS), 1.e-1));
-        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.SLOPPY_ARC.calculate(2, 2, 1, 1, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 1, 2, DistanceUnit.METERS), 1.e-1));
+        assertThat((Double) searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.ARC.calculate(2, 2, 1, 1, DistanceUnit.METERS), 1.e-1));
     }
 
     protected void createQPoints(List<String> qHashes, List<GeoPoint> qPoints) {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 131d19f600d..93442091458 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -23,7 +23,6 @@ package org.elasticsearch.search.sort;
 import org.apache.lucene.document.LatLonDocValuesField;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -37,7 +36,6 @@ import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
@@ -208,7 +206,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder.fromXContent(context, "");
@@ -229,7 +227,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder.fromXContent(context, "");
@@ -246,7 +244,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         try {
           GeoDistanceSortBuilder item = GeoDistanceSortBuilder.fromXContent(context, "");
@@ -258,7 +256,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             assertEquals("illegal latitude value [269.384765625] for [GeoDistanceSort] for field [reverse].", e.getMessage());
         }
     }
-    
+
     public void testCoerceIsDeprecated() throws IOException {
         String json = "{\n" +
                 "  \"testname\" : [ {\n" +
@@ -266,18 +264,16 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
-                "  \"mode\" : \"SUM\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
+                "  \"mode\" : \"MAX\",\n" +
                 "  \"coerce\" : true\n" +
                 "}";
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-        
+        QueryParseContext context = new QueryParseContext(itemParser);
+        GeoDistanceSortBuilder.fromXContent(context, "");
+        assertWarnings("Deprecated field [coerce] used, replaced by [validation_method]");
     }
 
     public void testIgnoreMalformedIsDeprecated() throws IOException {
@@ -287,18 +283,16 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
-                "  \"mode\" : \"SUM\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
+                "  \"mode\" : \"MAX\",\n" +
                 "  \"ignore_malformed\" : true\n" +
                 "}";
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
-        assertTrue(e.getMessage().startsWith("Deprecated field "));
-        
+        QueryParseContext context = new QueryParseContext(itemParser);
+        GeoDistanceSortBuilder.fromXContent(context, "");
+        assertWarnings("Deprecated field [ignore_malformed] used, replaced by [validation_method]");
     }
 
     public void testSortModeSumIsRejectedInJSON() throws IOException {
@@ -308,13 +302,13 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "    \"lon\" : -51.94128329747579\n" +
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
-                "  \"distance_type\" : \"sloppy_arc\",\n" +
+                "  \"distance_type\" : \"arc\",\n" +
                 "  \"mode\" : \"SUM\"\n" +
                 "}";
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
         assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage());
@@ -325,7 +319,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "    \"VDcvDuFjE\" : [ \"7umzzv8eychg\", \"dmdgmt5z13uw\", " +
                 "    \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n" +
                 "    \"unit\" : \"m\",\n" +
-                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"distance_type\" : \"arc\",\n" +
                 "    \"mode\" : \"MAX\",\n" +
                 "    \"nested_filter\" : {\n" +
                 "      \"ids\" : {\n" +
@@ -339,7 +333,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, json);
         itemParser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json);
         assertEquals("[-19.700583312660456, -2.8225036337971687, "
@@ -455,13 +449,13 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         sortBuilder.field("unit", "km");
         sortBuilder.field("sort_mode", "max");
         sortBuilder.endObject();
-        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> parse(sortBuilder));
-        assertEquals("Deprecated field [sort_mode] used, expected [mode] instead", ex.getMessage());
+        parse(sortBuilder);
+        assertWarnings("Deprecated field [sort_mode] used, expected [mode] instead");
     }
 
     private GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception {
         XContentParser parser = createParser(sortBuilder);
-        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        QueryParseContext parseContext = new QueryParseContext(parser);
         parser.nextToken();
         return GeoDistanceSortBuilder.fromXContent(parseContext, null);
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index eff8685b9da..1d4389c4e33 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -21,8 +21,6 @@ package org.elasticsearch.search.sort;
 
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -69,7 +67,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScoreSortBuilder scoreSort = ScoreSortBuilder.fromXContent(context, "_score");
         assertEquals(order, scoreSort.order());
     }
@@ -82,7 +80,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
+        QueryParseContext context = new QueryParseContext(parser);
 
         try {
           ScoreSortBuilder.fromXContent(context, "_score");
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 68bc4cfa00a..1ad5de4071f 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.sort;
 
 
 import org.apache.lucene.search.SortField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -172,7 +171,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
         assertEquals("doc['field_name'].value * factor", builder.script().getIdOrCode());
         assertEquals(Script.DEFAULT_SCRIPT_LANG, builder.script().getLang());
@@ -198,7 +197,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         ScriptSortBuilder builder = ScriptSortBuilder.fromXContent(context, null);
         assertEquals("doc['field_name'].value", builder.script().getIdOrCode());
         assertEquals(Script.DEFAULT_SCRIPT_LANG, builder.script().getLang());
@@ -218,7 +217,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
         assertEquals("[_script] unknown field [bad_field], parser not found", e.getMessage());
     }
@@ -231,7 +230,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
         assertEquals("[_script] unknown field [bad_field], parser not found", e.getMessage());
     }
@@ -243,7 +242,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         parser.nextToken();
         parser.nextToken();
 
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(parser);
         Exception e = expectThrows(IllegalArgumentException.class, () -> ScriptSortBuilder.fromXContent(context, null));
         assertEquals("[_script] script doesn't support values of type: START_ARRAY", e.getMessage());
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
index d31dee0b3d2..cc62e546a3d 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -19,17 +19,15 @@ x * Licensed to Elasticsearch under one or more contributor
 
 package org.elasticsearch.search.sort;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -43,24 +41,19 @@ import java.util.List;
 import static java.util.Collections.emptyList;
 
 public class SortBuilderTests extends ESTestCase {
-
     private static final int NUMBER_OF_RUNS = 20;
 
-    protected static NamedWriteableRegistry namedWriteableRegistry;
-
-    static IndicesQueriesRegistry indicesQueriesRegistry;
+    private static NamedXContentRegistry xContentRegistry;
 
     @BeforeClass
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
-        namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        indicesQueriesRegistry = searchModule.getQueryParserRegistry();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
-        namedWriteableRegistry = null;
-        indicesQueriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -237,9 +230,14 @@ public class SortBuilderTests extends ESTestCase {
         assertEquals(new ScoreSortBuilder(), result.get(5));
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
+
     private List<SortBuilder<?>> parseSort(String jsonString) throws IOException {
         XContentParser itemParser = createParser(JsonXContent.jsonXContent, jsonString);
-        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+        QueryParseContext context = new QueryParseContext(itemParser);
 
         assertEquals(XContentParser.Token.START_OBJECT, itemParser.nextToken());
         assertEquals(XContentParser.Token.FIELD_NAME, itemParser.nextToken());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index 0f93e6a36e5..530a505a695 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -19,17 +19,15 @@
 
 package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -44,9 +42,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
 
     private static final int NUMBER_OF_TESTBUILDERS = 20;
     protected static NamedWriteableRegistry namedWriteableRegistry;
-    protected static IndicesQueriesRegistry queriesRegistry;
-    protected static ParseFieldMatcher parseFieldMatcher;
-    protected static Suggesters suggesters;
+    protected static NamedXContentRegistry xContentRegistry;
 
     /**
      * setup for the whole base test class
@@ -55,16 +51,13 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     public static void init() throws IOException {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        queriesRegistry = searchModule.getQueryParserRegistry();
-        suggesters = searchModule.getSuggesters();
-        parseFieldMatcher = ParseFieldMatcher.STRICT;
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
         namedWriteableRegistry = null;
-        suggesters = null;
-        queriesRegistry = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -128,11 +121,10 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
 
             XContentBuilder shuffled = shuffleXContent(xContentBuilder, shuffleProtectedFields());
             XContentParser parser = createParser(shuffled);
-            QueryParseContext context = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
             // we need to skip the start object and the name, those will be parsed by outer SuggestBuilder
             parser.nextToken();
 
-            SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(context, suggesters);
+            SuggestionBuilder<?> secondSuggestionBuilder = SuggestionBuilder.fromXContent(parser);
             assertNotSame(suggestionBuilder, secondSuggestionBuilder);
             assertEquals(suggestionBuilder, secondSuggestionBuilder);
             assertEquals(suggestionBuilder.hashCode(), secondSuggestionBuilder.hashCode());
@@ -190,8 +182,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
                 (Writeable.Reader<SB>) namedWriteableRegistry.getReader(SuggestionBuilder.class, original.getWriteableName()));
     }
 
-    protected static QueryParseContext newParseContext(XContentParser parser) throws IOException {
-        final QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-        return parseContext;
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
index 48b570e880a..63c8aa1e668 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java
@@ -20,11 +20,8 @@ package org.elasticsearch.search.suggest;
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.suggest.CustomSuggesterSearchIT.CustomSuggestionBuilder;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -63,14 +60,4 @@ public class CustomSuggester extends Suggester<CustomSuggester.CustomSuggestions
             this.options = options;
         }
     }
-
-    @Override
-    public SuggestionBuilder<?> innerFromXContent(QueryParseContext context) throws IOException {
-        return CustomSuggestionBuilder.innerFromXContent(context);
-    }
-
-    @Override
-    public SuggestionBuilder<?> read(StreamInput in) throws IOException {
-        return new CustomSuggestionBuilder(in);
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
index 38d2a3976dc..dde5a730457 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.suggest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,9 +28,9 @@ import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -46,6 +45,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.hasSize;
@@ -66,6 +66,14 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
         return Arrays.asList(CustomSuggesterPlugin.class);
     }
 
+    public static class CustomSuggesterPlugin extends Plugin implements SearchPlugin {
+        @Override
+        public List<SuggesterSpec<?>> getSuggesters() {
+            return singletonList(new SuggesterSpec<CustomSuggestionBuilder>("custom", CustomSuggestionBuilder::new,
+                    CustomSuggestionBuilder::fromXContent));
+        }
+    }
+
     public void testThatCustomSuggestersCanBeRegisteredAndWork() throws Exception {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource(jsonBuilder()
@@ -140,9 +148,7 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
             return Objects.hash(randomSuffix);
         }
 
-        static CustomSuggestionBuilder innerFromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser parser = parseContext.parser();
-            ParseFieldMatcher parseFieldMatcher = parseContext.getParseFieldMatcher();
+        public static CustomSuggestionBuilder fromXContent(XContentParser parser) throws IOException {
             XContentParser.Token token;
             String currentFieldName = null;
             String fieldname = null;
@@ -154,15 +160,15 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     currentFieldName = parser.currentName();
                 } else if (token.isValue()) {
-                    if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.ANALYZER_FIELD)) {
+                    if (SuggestionBuilder.ANALYZER_FIELD.match(currentFieldName)) {
                         analyzer = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.FIELDNAME_FIELD)) {
+                    } else if (SuggestionBuilder.FIELDNAME_FIELD.match(currentFieldName)) {
                         fieldname = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SIZE_FIELD)) {
+                    } else if (SuggestionBuilder.SIZE_FIELD.match(currentFieldName)) {
                         sizeField = parser.intValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, SuggestionBuilder.SHARDSIZE_FIELD)) {
+                    } else if (SuggestionBuilder.SHARDSIZE_FIELD.match(currentFieldName)) {
                         shardSize = parser.intValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, RANDOM_SUFFIX_FIELD)) {
+                    } else if (RANDOM_SUFFIX_FIELD.match(currentFieldName)) {
                         suffix = parser.text();
                     }
                 } else {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
index c9896ef573c..9029a097caa 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestBuilderTests.java
@@ -19,17 +19,14 @@
 
 package org.elasticsearch.search.suggest;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.suggest.completion.CompletionSuggesterBuilderTests;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilderTests;
@@ -48,7 +45,7 @@ public class SuggestBuilderTests extends ESTestCase {
 
     private static final int NUMBER_OF_RUNS = 20;
     private static NamedWriteableRegistry namedWriteableRegistry;
-    private static Suggesters suggesters;
+    private static NamedXContentRegistry xContentRegistry;
 
     /**
      * Setup for the whole base test class.
@@ -57,13 +54,13 @@ public class SuggestBuilderTests extends ESTestCase {
     public static void init() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
         namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
-        suggesters = searchModule.getSuggesters();
+        xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
     }
 
     @AfterClass
     public static void afterClass() {
         namedWriteableRegistry = null;
-        suggesters = null;
+        xContentRegistry = null;
     }
 
     /**
@@ -78,8 +75,7 @@ public class SuggestBuilderTests extends ESTestCase {
             }
             suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
             XContentParser parser = createParser(xContentBuilder);
-            QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
-            SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
+            SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(parser);
             assertNotSame(suggestBuilder, secondSuggestBuilder);
             assertEquals(suggestBuilder, secondSuggestBuilder);
             assertEquals(suggestBuilder.hashCode(), secondSuggestBuilder.hashCode());
@@ -161,4 +157,8 @@ public class SuggestBuilderTests extends ESTestCase {
         }
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return xContentRegistry;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
index b10272329e5..95f79845621 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
@@ -459,7 +459,6 @@ public class SuggestSearchIT extends ESIntegTestCase {
                 .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
                 .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
                 .startObject("properties")
                 .startObject("body").field("type", "text").field("analyzer", "body").endObject()
                 .startObject("body_reverse").field("type", "text").field("analyzer", "reverse").endObject()
@@ -502,10 +501,6 @@ public class SuggestSearchIT extends ESIntegTestCase {
                 .put("index.analysis.filter.my_shingle.max_shingle_size", 2)
                 .put("index.number_of_shards", 1));
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                    .startObject("_all")
-                        .field("store", true)
-                        .field("term_vector", "with_positions_offsets")
-                    .endObject()
                     .startObject("properties")
                         .startObject("body").
                             field("type", "text").
@@ -655,10 +650,6 @@ public class SuggestSearchIT extends ESIntegTestCase {
         XContentBuilder mapping = XContentFactory.jsonBuilder()
                 .startObject()
                     .startObject("type1")
-                        .startObject("_all")
-                            .field("store", true)
-                            .field("term_vector", "with_positions_offsets")
-                        .endObject()
                         .startObject("properties")
                             .startObject("body")
                                 .field("type", "text")
@@ -726,7 +717,6 @@ public class SuggestSearchIT extends ESIntegTestCase {
 
         XContentBuilder mapping = XContentFactory.jsonBuilder()
                     .startObject().startObject("type1")
-                    .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
                 .startObject("properties")
                 .startObject("body").field("type", "text").field("analyzer", "body").endObject()
                 .startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject()
@@ -929,10 +919,6 @@ public class SuggestSearchIT extends ESIntegTestCase {
         XContentBuilder mapping = XContentFactory.jsonBuilder()
                 .startObject()
                     .startObject("type1")
-                        .startObject("_all")
-                            .field("store", true)
-                            .field("term_vector", "with_positions_offsets")
-                        .endObject()
                         .startObject("properties")
                             .startObject("body")
                                 .field("type", "text")
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
index a74376554f5..7c864320abe 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CategoryContextMappingTests.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -34,7 +33,6 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.completion.context.CategoryContextMapping;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -258,7 +256,7 @@ public class CategoryContextMappingTests extends ESSingleNodeTestCase {
     }
 
     private static QueryParseContext createParseContext(XContentParser parser) {
-        return new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        return new QueryParseContext(parser);
     }
 
     public void testQueryContextParsingMixed() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
index a6879ebf297..f4c65b24780 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.suggest.completion;
 
 import org.apache.lucene.index.IndexableField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -30,7 +29,6 @@ import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.completion.context.ContextBuilder;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
 import org.elasticsearch.search.suggest.completion.context.GeoContextMapping;
@@ -352,6 +350,6 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
     }
 
     private static QueryParseContext createParseContext(XContentParser parser) {
-        return new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
+        return new QueryParseContext(parser);
     };
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
index f8a0b22901b..192aa8ee1d5 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/QueryContextTestCase.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.search.suggest.completion;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -52,7 +49,7 @@ public abstract class QueryContextTestCase<QC extends ToXContent> extends ESTest
             toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
             XContentParser parser = createParser(builder);
             parser.nextToken();
-            QC fromXContext = fromXContent(new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT));
+            QC fromXContext = fromXContent(new QueryParseContext(parser));
             assertEquals(toXContent, fromXContext);
             assertEquals(toXContent.hashCode(), fromXContext.hashCode());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index 27ec964aa4b..6c30ccb4570 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -29,8 +28,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCandidateGenerator;
 import org.elasticsearch.test.ESTestCase;
 
@@ -42,13 +39,9 @@ import java.util.function.Supplier;
 
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 
-public class DirectCandidateGeneratorTests extends ESTestCase{
-
-    private static final IndicesQueriesRegistry mockRegistry = new IndicesQueriesRegistry();
+public class DirectCandidateGeneratorTests extends ESTestCase {
     private static final int NUMBER_OF_RUNS = 20;
 
-
-
     /**
      * Test serialization and deserialization of the generator
      */
@@ -113,9 +106,8 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
             }
             generator.toXContent(builder, ToXContent.EMPTY_PARAMS);
             XContentParser parser = createParser(shuffleXContent(builder));
-            QueryParseContext context = new QueryParseContext(mockRegistry, parser, ParseFieldMatcher.STRICT);
             parser.nextToken();
-            DirectCandidateGeneratorBuilder secondGenerator = DirectCandidateGeneratorBuilder.fromXContent(context);
+            DirectCandidateGeneratorBuilder secondGenerator = DirectCandidateGeneratorBuilder.PARSER.apply(parser, null);
             assertNotSame(generator, secondGenerator);
             assertEquals(generator, secondGenerator);
             assertEquals(generator.hashCode(), secondGenerator.hashCode());
@@ -177,8 +169,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
     private void assertIllegalXContent(String directGenerator, Class<? extends Exception> exceptionClass, String exceptionMsg)
             throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, directGenerator);
-        QueryParseContext context = new QueryParseContext(mockRegistry, parser, ParseFieldMatcher.STRICT);
-        Exception e = expectThrows(exceptionClass, () -> DirectCandidateGeneratorBuilder.fromXContent(context));
+        Exception e = expectThrows(exceptionClass, () -> DirectCandidateGeneratorBuilder.PARSER.apply(parser, null));
         assertEquals(exceptionMsg, e.getMessage());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
index dae87ef0570..22917c6c312 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LaplaceModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -54,7 +54,7 @@ public class LaplaceModelTests extends SmoothingModelTestCase {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return Laplace.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return Laplace.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
index b3689977840..7efbab4584f 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/LinearInterpolationModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -73,7 +73,7 @@ public class LinearInterpolationModelTests extends SmoothingModelTestCase {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return LinearInterpolation.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return LinearInterpolation.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
index 044982540a3..e75d01739cc 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
@@ -30,7 +30,6 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.store.RAMDirectory;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -38,8 +37,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
@@ -84,7 +81,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
      */
     protected abstract SmoothingModel createMutation(SmoothingModel original) throws IOException;
 
-    protected abstract SmoothingModel fromXContent(QueryParseContext context) throws IOException;
+    protected abstract SmoothingModel fromXContent(XContentParser parser) throws IOException;
 
     /**
      * Test that creates new smoothing model from a random test smoothing model and checks both for equality
@@ -99,9 +96,8 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
         testModel.innerToXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
         contentBuilder.endObject();
         XContentParser parser = createParser(shuffleXContent(contentBuilder));
-        QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
         parser.nextToken();  // go to start token, real parsing would do that in the outer element parser
-        SmoothingModel parsedModel = fromXContent(context);
+        SmoothingModel parsedModel = fromXContent(parser);
         assertNotSame(testModel, parsedModel);
         assertEquals(testModel, parsedModel);
         assertEquals(testModel.hashCode(), parsedModel.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
index 57f84c37543..c0e0abecb37 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/StupidBackoffModelTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.suggest.phrase;
 
-import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
@@ -53,7 +53,7 @@ public class StupidBackoffModelTests extends SmoothingModelTestCase {
     }
 
     @Override
-    protected SmoothingModel fromXContent(QueryParseContext context) throws IOException {
-        return StupidBackoff.innerFromXContent(context);
+    protected SmoothingModel fromXContent(XContentParser parser) throws IOException {
+        return StupidBackoff.fromXContent(parser);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
index 1df99ce83a7..f27d02fbe22 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/term/TermSuggestionBuilderTests.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase;
-import org.elasticsearch.search.suggest.DirectSpellcheckerSettings;
 import org.elasticsearch.search.suggest.SortBy;
 import org.elasticsearch.search.suggest.SuggestBuilder;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.StringDistanceImpl;
@@ -218,31 +217,10 @@ public class TermSuggestionBuilderTests extends AbstractSuggestionBuilderTestCas
                          "  }\n" +
                          "}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, suggest)) {
-            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(newParseContext(parser), suggesters);
+            final SuggestBuilder suggestBuilder = SuggestBuilder.fromXContent(parser);
             fail("Should not have been able to create SuggestBuilder from malformed JSON: " + suggestBuilder);
         } catch (Exception e) {
             assertThat(e.getMessage(), containsString("parsing failed"));
         }
     }
-
-    private void assertSpellcheckerSettings(DirectSpellcheckerSettings oldSettings, DirectSpellcheckerSettings newSettings) {
-        final double delta = 0.0d;
-        // make sure the objects aren't the same
-        assertNotSame(oldSettings, newSettings);
-        // make sure the objects aren't null
-        assertNotNull(oldSettings);
-        assertNotNull(newSettings);
-        // and now, make sure they are equal..
-        assertEquals(oldSettings.accuracy(), newSettings.accuracy(), delta);
-        assertEquals(oldSettings.maxEdits(), newSettings.maxEdits());
-        assertEquals(oldSettings.maxInspections(), newSettings.maxInspections());
-        assertEquals(oldSettings.maxTermFreq(), newSettings.maxTermFreq(), delta);
-        assertEquals(oldSettings.minDocFreq(), newSettings.minDocFreq(), delta);
-        assertEquals(oldSettings.minWordLength(), newSettings.minWordLength());
-        assertEquals(oldSettings.prefixLength(), newSettings.prefixLength());
-        assertEquals(oldSettings.sort(), newSettings.sort());
-        assertEquals(oldSettings.stringDistance().getClass(), newSettings.stringDistance().getClass());
-        assertEquals(oldSettings.suggestMode().getClass(), newSettings.suggestMode().getClass());
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
index 617fcf9ebc3..56ff111e4b0 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.snapshots;
 
 import org.elasticsearch.ElasticsearchCorruptionException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -32,7 +31,6 @@ import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -56,16 +54,13 @@ import static org.hamcrest.Matchers.greaterThan;
 
 public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
 
-    private static final ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(Settings.EMPTY);
-
     public static final String BLOB_CODEC = "blob";
 
-    private static class BlobObj implements ToXContent, FromXContentBuilder<BlobObj> {
-        public static final BlobObj PROTO = new BlobObj("");
+    private static class BlobObj implements ToXContent {
 
         private final String text;
 
-        public BlobObj(String text) {
+        BlobObj(String text) {
             this.text = text;
         }
 
@@ -73,8 +68,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
             return text;
         }
 
-        @Override
-        public BlobObj fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        public static BlobObj fromXContent(XContentParser parser) throws IOException {
             String text = null;
             XContentParser.Token token = parser.currentToken();
             if (token == null) {
@@ -114,9 +108,12 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
     public void testBlobStoreOperations() throws IOException {
         BlobStore blobStore = createTestBlobStore();
         BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
-        ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.JSON);
-        ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, XContentType.SMILE);
-        ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, true, XContentType.SMILE);
+        ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, XContentType.SMILE);
+        ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), true, XContentType.SMILE);
 
         // Write blobs in different formats
         checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
@@ -139,8 +136,10 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
         for (int i = 0; i < randomIntBetween(100, 300); i++) {
             veryRedundantText.append("Blah ");
         }
-        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
-        ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         BlobObj blobObj = new BlobObj(veryRedundantText.toString());
         checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
         checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
@@ -154,7 +153,8 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
         BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
         String testString = randomAsciiOfLength(randomInt(10000));
         BlobObj blobObj = new BlobObj(testString);
-        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         checksumFormat.write(blobObj, blobContainer, "test-path");
         assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
         randomCorruption(blobContainer, "test-path");
@@ -188,7 +188,8 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
                 return builder;
             }
         };
-        final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj.PROTO, parseFieldMatcher, randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
+        final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
+            xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
         ExecutorService threadPool = Executors.newFixedThreadPool(1);
         try {
             Future<Void> future = threadPool.submit(new Callable<Void>() {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 1d0ddf39b01..c334ba6e7a9 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.snapshots;
 
 import com.carrotsearch.hppc.IntHashSet;
 import com.carrotsearch.hppc.IntSet;
+
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
@@ -35,15 +36,24 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.indices.recovery.RecoveryState;
@@ -51,6 +61,7 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.rest.AbstractRestChannel;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.admin.cluster.RestClusterStateAction;
@@ -62,6 +73,7 @@ import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.TestCustomMetaData;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -84,13 +96,56 @@ import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
 public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
 
+    public static class TestCustomMetaDataPlugin extends Plugin {
+
+        private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
+        private final List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
+
+        public TestCustomMetaDataPlugin() {
+            registerBuiltinWritables();
+        }
+
+        private <T extends MetaData.Custom> void registerMetaDataCustom(String name, Writeable.Reader<T> reader,
+                                                                        Writeable.Reader<NamedDiff> diffReader,
+                                                                        CheckedFunction<XContentParser, T, IOException> parser) {
+            namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader));
+            namedWritables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, name, diffReader));
+            namedXContents.add(new NamedXContentRegistry.Entry(MetaData.Custom.class, new ParseField(name), parser));
+        }
+
+        private void registerBuiltinWritables() {
+            registerMetaDataCustom(SnapshottableMetadata.TYPE, SnapshottableMetadata::readFrom,
+                SnapshottableMetadata::readDiffFrom, SnapshottableMetadata::fromXContent);
+            registerMetaDataCustom(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata::readFrom,
+                NonSnapshottableMetadata::readDiffFrom, NonSnapshottableMetadata::fromXContent);
+            registerMetaDataCustom(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata::readFrom,
+                SnapshottableGatewayMetadata::readDiffFrom, SnapshottableGatewayMetadata::fromXContent);
+            registerMetaDataCustom(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata::readFrom,
+                NonSnapshottableGatewayMetadata::readDiffFrom, NonSnapshottableGatewayMetadata::fromXContent);
+            registerMetaDataCustom(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata::readFrom,
+                NonSnapshottableGatewayMetadata::readDiffFrom, SnapshotableGatewayNoApiMetadata::fromXContent);
+
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return namedWritables;
+        }
+
+        @Override
+        public List<NamedXContentRegistry.Entry> getNamedXContent() {
+            return namedXContents;
+        }
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockRepository.Plugin.class);
+        return Arrays.asList(MockRepository.Plugin.class, TestCustomMetaDataPlugin.class);
     }
 
     public void testRestorePersistentSettings() throws Exception {
@@ -631,7 +686,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         ).get();
 
         NodeClient nodeClient = internalCluster().getInstance(NodeClient.class);
-        RestGetRepositoriesAction getRepoAction = internalCluster().getInstance(RestGetRepositoriesAction.class);
+        RestGetRepositoriesAction getRepoAction = new RestGetRepositoriesAction(nodeSettings, mock(RestController.class),
+                internalCluster().getInstance(SettingsFilter.class));
         RestRequest getRepoRequest = new FakeRestRequest();
         getRepoRequest.params().put("repository", "test-repo");
         final CountDownLatch getRepoLatch = new CountDownLatch(1);
@@ -653,7 +709,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
             throw getRepoError.get();
         }
 
-        RestClusterStateAction clusterStateAction = internalCluster().getInstance(RestClusterStateAction.class);
+        RestClusterStateAction clusterStateAction = new RestClusterStateAction(nodeSettings, mock(RestController.class),
+                internalCluster().getInstance(SettingsFilter.class));
         RestRequest clusterStateRequest = new FakeRestRequest();
         final CountDownLatch clusterStateLatch = new CountDownLatch(1);
         final AtomicReference<AssertionError> clusterStateError = new AtomicReference<>();
@@ -780,33 +837,31 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
 
     }
 
-    static {
-        MetaData.registerPrototype(SnapshottableMetadata.TYPE, SnapshottableMetadata.PROTO);
-        MetaData.registerPrototype(NonSnapshottableMetadata.TYPE, NonSnapshottableMetadata.PROTO);
-        MetaData.registerPrototype(SnapshottableGatewayMetadata.TYPE, SnapshottableGatewayMetadata.PROTO);
-        MetaData.registerPrototype(NonSnapshottableGatewayMetadata.TYPE, NonSnapshottableGatewayMetadata.PROTO);
-        MetaData.registerPrototype(SnapshotableGatewayNoApiMetadata.TYPE, SnapshotableGatewayNoApiMetadata.PROTO);
-    }
-
     public static class SnapshottableMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable";
 
-        public static final SnapshottableMetadata PROTO = new SnapshottableMetadata("");
-
         public SnapshottableMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new SnapshottableMetadata(data);
+        public static SnapshottableMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshottableMetadata::new, in);
         }
 
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static SnapshottableMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshottableMetadata::new, parser);
+        }
+
+
         @Override
         public EnumSet<MetaData.XContentContext> context() {
             return MetaData.API_AND_SNAPSHOT;
@@ -816,20 +871,25 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
     public static class NonSnapshottableMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_non_snapshottable";
 
-        public static final NonSnapshottableMetadata PROTO = new NonSnapshottableMetadata("");
-
         public NonSnapshottableMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected NonSnapshottableMetadata newTestCustomMetaData(String data) {
-            return new NonSnapshottableMetadata(data);
+        public static NonSnapshottableMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(NonSnapshottableMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static NonSnapshottableMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(NonSnapshottableMetadata::new, parser);
         }
 
         @Override
@@ -841,20 +901,25 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
     public static class SnapshottableGatewayMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable_gateway";
 
-        public static final SnapshottableGatewayMetadata PROTO = new SnapshottableGatewayMetadata("");
-
         public SnapshottableGatewayMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new SnapshottableGatewayMetadata(data);
+        public static SnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshottableGatewayMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static SnapshottableGatewayMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshottableGatewayMetadata::new, parser);
         }
 
         @Override
@@ -866,20 +931,25 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
     public static class NonSnapshottableGatewayMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_non_snapshottable_gateway";
 
-        public static final NonSnapshottableGatewayMetadata PROTO = new NonSnapshottableGatewayMetadata("");
-
         public NonSnapshottableGatewayMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected NonSnapshottableGatewayMetadata newTestCustomMetaData(String data) {
-            return new NonSnapshottableGatewayMetadata(data);
+        public static NonSnapshottableGatewayMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(NonSnapshottableGatewayMetadata::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+        public static NonSnapshottableGatewayMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(NonSnapshottableGatewayMetadata::new, parser);
         }
 
         @Override
@@ -892,20 +962,21 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
     public static class SnapshotableGatewayNoApiMetadata extends TestCustomMetaData {
         public static final String TYPE = "test_snapshottable_gateway_no_api";
 
-        public static final SnapshotableGatewayNoApiMetadata PROTO = new SnapshotableGatewayNoApiMetadata("");
-
         public SnapshotableGatewayNoApiMetadata(String data) {
             super(data);
         }
 
         @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
-        @Override
-        protected SnapshotableGatewayNoApiMetadata newTestCustomMetaData(String data) {
-            return new SnapshotableGatewayNoApiMetadata(data);
+        public static SnapshotableGatewayNoApiMetadata readFrom(StreamInput in) throws IOException {
+            return readFrom(SnapshotableGatewayNoApiMetadata::new, in);
+        }
+
+        public static SnapshotableGatewayNoApiMetadata fromXContent(XContentParser parser) throws IOException {
+            return fromXContent(SnapshotableGatewayNoApiMetadata::new, parser);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
new file mode 100644
index 00000000000..d1759d83e34
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.snapshots.mockstore.MockRepository;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests for snapshot/restore that require at least 2 threads available
+ * in the thread pool (for example, tests that use the mock repository that
+ * block on master).
+ */
+public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+                   .put("thread_pool.snapshot.core", 2)
+                   .put("thread_pool.snapshot.max", 2)
+                   .build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(MockRepository.Plugin.class);
+    }
+
+    public void testConcurrentSnapshotDeletionsNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAsciiOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot twice");
+        final String index = "test-idx1";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        final String index2 = "test-idx2";
+        assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot2 = "test-snap2";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of first snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot2).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try deleting the second snapshot, should fail because the first deletion is in progress");
+        try {
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get();
+            fail("should not be able to delete snapshots concurrently");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot delete - another snapshot is currently being deleted"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until first snapshot is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> delete second snapshot, which should now work");
+        client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get();
+        assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().isEmpty());
+    }
+
+    public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAsciiOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot");
+        final String index = "test-idx";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try creating a second snapshot, should fail because the deletion is in progress");
+        final String snapshot2 = "test-snap2";
+        try {
+            client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+            fail("should not be able to create a snapshot while another is being deleted");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot snapshot while a snapshot deletion is in-progress"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until snapshot deletion is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> creating second snapshot, which should now work");
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+    }
+
+    public void testRestoreWithInProgressDeletionsNotAllowed() throws Exception {
+        logger.info("--> creating repository");
+        final String repo = "test-repo";
+        assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(
+            Settings.builder()
+                .put("location", randomRepoPath())
+                .put("random", randomAsciiOfLength(10))
+                .put("wait_after_unblock", 200)).get());
+
+        logger.info("--> snapshot");
+        final String index = "test-idx";
+        assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot1 = "test-snap1";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        final String index2 = "test-idx2";
+        assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0)));
+        for (int i = 0; i < 10; i++) {
+            index(index2, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        final String snapshot2 = "test-snap2";
+        client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get();
+        client().admin().indices().prepareClose(index, index2).get();
+
+        String blockedNode = internalCluster().getMasterName();
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true);
+        logger.info("--> start deletion of snapshot");
+        ListenableActionFuture<DeleteSnapshotResponse> future =
+            client().admin().cluster().prepareDeleteSnapshot(repo, snapshot2).execute();
+        logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
+        waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
+
+        logger.info("--> try restoring the other snapshot, should fail because the deletion is in progress");
+        try {
+            client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+            fail("should not be able to restore a snapshot while another is being deleted");
+        } catch (ConcurrentSnapshotExecutionException e) {
+            assertThat(e.getMessage(), containsString("cannot restore a snapshot while a snapshot deletion is in-progress"));
+        }
+
+        logger.info("--> unblocking blocked node [{}]", blockedNode);
+        unblockNode(repo, blockedNode);
+
+        logger.info("--> wait until snapshot deletion is finished");
+        assertAcked(future.actionGet());
+
+        logger.info("--> restoring snapshot, which should now work");
+        client().admin().cluster().prepareRestoreSnapshot(repo, snapshot1).setWaitForCompletion(true).get();
+        assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots().size());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 639b60d6d09..0f6dcec818f 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -142,30 +142,6 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
         } catch (RepositoryException ex) {
             assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo"));
         }
-
-        String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString();
-        String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/");
-        logger.info("--> trying creating url repository with unsupported url protocol");
-        try {
-            client().admin().cluster().preparePutRepository("test-repo")
-                    .setType("url").setSettings(Settings.builder().put("url", unsupportedUrl))
-                    .get();
-            fail("Shouldn't be here");
-        } catch (RepositoryException ex) {
-            assertThat(ex.toString(),
-                either(containsString("unsupported url protocol [netdoc]"))
-                    .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9
-        }
-
-        logger.info("--> trying creating url repository with location that is not registered in path.repo setting");
-        try {
-            client().admin().cluster().preparePutRepository("test-repo")
-                    .setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL()))
-                    .get();
-            fail("Shouldn't be here");
-        } catch (RepositoryException ex) {
-            assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo"));
-        }
     }
 
     public void testRepositoryAckTimeout() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesMetaDataSerializationTests.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesMetaDataSerializationTests.java
new file mode 100644
index 00000000000..101ae591292
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesMetaDataSerializationTests.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.metadata.MetaData.Custom;
+import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.AbstractDiffableSerializationTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+public class RepositoriesMetaDataSerializationTests extends AbstractDiffableSerializationTestCase<Custom> {
+
+    @Override
+    protected Custom createTestInstance() {
+        int numberOfRepositories = randomInt(10);
+        List<RepositoryMetaData> entries = new ArrayList<>();
+        for (int i = 0; i < numberOfRepositories; i++) {
+            entries.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings()));
+        }
+        entries.sort(Comparator.comparing(RepositoryMetaData::name));
+        return new RepositoriesMetaData(entries.toArray(new RepositoryMetaData[entries.size()]));
+    }
+
+    @Override
+    protected Writeable.Reader<Custom> instanceReader() {
+        return RepositoriesMetaData::new;
+    }
+
+    public Settings randomSettings() {
+        if (randomBoolean()) {
+            return Settings.EMPTY;
+        } else {
+            int numberOfSettings = randomInt(10);
+            Settings.Builder builder = Settings.builder();
+            for (int i = 0; i < numberOfSettings; i++) {
+                builder.put(randomAsciiOfLength(10), randomAsciiOfLength(20));
+            }
+            return builder.build();
+        }
+    }
+
+    @Override
+    protected Custom makeTestChanges(Custom testInstance) {
+        RepositoriesMetaData repositoriesMetaData = (RepositoriesMetaData) testInstance;
+        List<RepositoryMetaData> repos = new ArrayList<>(repositoriesMetaData.repositories());
+        if (randomBoolean() && repos.size() > 1) {
+            // remove some elements
+            int leaveElements = randomIntBetween(0, repositoriesMetaData.repositories().size() - 1);
+            repos = randomSubsetOf(leaveElements, repos.toArray(new RepositoryMetaData[leaveElements]));
+        }
+        if (randomBoolean()) {
+            // add some elements
+            int addElements = randomInt(10);
+            for (int i = 0; i < addElements; i++) {
+                repos.add(new RepositoryMetaData(randomAsciiOfLength(10), randomAsciiOfLength(10), randomSettings()));
+            }
+        }
+        return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()]));
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<Custom>> diffReader() {
+        return RepositoriesMetaData::readDiffFrom;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+    }
+
+    @Override
+    protected Custom doParseInstance(XContentParser parser) throws IOException {
+        assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
+        RepositoriesMetaData repositoriesMetaData = RepositoriesMetaData.fromXContent(parser);
+        assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
+        List<RepositoryMetaData> repos = repositoriesMetaData.repositories();
+        repos.sort(Comparator.comparing(RepositoryMetaData::name));
+        return new RepositoriesMetaData(repos.toArray(new RepositoryMetaData[repos.size()]));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index dda634023b6..6116cfa8b98 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -65,6 +65,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
@@ -445,10 +446,25 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())));
 
         logger.info("-->  creating test template");
-        assertThat(client.admin().indices().preparePutTemplate("test-template").setPatterns(Collections.singletonList("te*")).addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
-            .startObject("field1").field("type", "text").field("store", "yes").endObject()
-            .startObject("field2").field("type", "keyword").field("store", "yes").endObject()
-            .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
+        assertThat(client.admin().indices()
+            .preparePutTemplate("test-template")
+                .setPatterns(Collections.singletonList("te*"))
+                .addMapping("test-mapping", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("test-mapping")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "keyword")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
+            .get().isAcknowledged(), equalTo(true));
 
         logger.info("--> snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setIndices().setWaitForCompletion(true).get();
@@ -486,10 +502,25 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         if(testTemplate) {
             logger.info("-->  creating test template");
-            assertThat(client.admin().indices().preparePutTemplate("test-template").setPatterns(Collections.singletonList("te*")).addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
-                .startObject("field1").field("type", "text").field("store", "yes").endObject()
-                .startObject("field2").field("type", "keyword").field("store", "yes").endObject()
-                .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
+            assertThat(client.admin().indices()
+                .preparePutTemplate("test-template")
+                .setPatterns(Collections.singletonList("te*"))
+                .addMapping("test-mapping", XContentFactory.jsonBuilder()
+                    .startObject()
+                        .startObject("test-mapping")
+                            .startObject("properties")
+                                .startObject("field1")
+                                    .field("type", "text")
+                                    .field("store", true)
+                                .endObject()
+                                .startObject("field2")
+                                    .field("type", "keyword")
+                                    .field("store", true)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject())
+                .get().isAcknowledged(), equalTo(true));
         }
 
         if(testPipeline) {
@@ -503,15 +534,15 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 .endObject()
                 .endArray()
                 .endObject().bytes();
-            assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource).get());
+            assertAcked(client().admin().cluster().preparePutPipeline("barbaz", pipelineSource, XContentType.JSON).get());
         }
 
         if(testScript) {
             logger.info("-->  creating test script");
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MockScriptEngine.NAME)
+                .setLang(MockScriptEngine.NAME)
                 .setId("foobar")
-                .setSource(new BytesArray("{\"script\":\"1\"}")));
+                .setContent(new BytesArray("{\"script\":\"1\"}"), XContentType.JSON));
         }
 
         logger.info("--> snapshot without global state");
@@ -570,7 +601,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         if (testScript) {
             logger.info("--> check that script is restored");
             GetStoredScriptResponse getStoredScriptResponse = client().admin().cluster().prepareGetStoredScript(MockScriptEngine.NAME, "foobar").get();
-            assertNotNull(getStoredScriptResponse.getStoredScript());
+            assertNotNull(getStoredScriptResponse.getSource());
         }
 
         createIndex("test-idx");
@@ -614,7 +645,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
         assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
         assertFalse(client().admin().cluster().prepareGetPipeline("barbaz").get().isFound());
-        assertNull(client().admin().cluster().prepareGetStoredScript(MockScriptEngine.NAME, "foobar").get().getStoredScript());
+        assertNull(client().admin().cluster().prepareGetStoredScript(MockScriptEngine.NAME, "foobar").get().getSource());
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
     }
@@ -1412,63 +1443,6 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
     }
 
-    public void testUrlRepository() throws Exception {
-        Client client = client();
-
-        logger.info("-->  creating repository");
-        Path repositoryLocation = randomRepoPath();
-        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
-                .setType("fs").setSettings(Settings.builder()
-                        .put("location", repositoryLocation)
-                        .put("compress", randomBoolean())
-                        .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
-
-        createIndex("test-idx");
-        ensureGreen();
-
-        logger.info("--> indexing some data");
-        for (int i = 0; i < 100; i++) {
-            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
-
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
-
-        logger.info("--> delete index");
-        cluster().wipeIndices("test-idx");
-
-        logger.info("--> create read-only URL repository");
-        assertAcked(client.admin().cluster().preparePutRepository("url-repo")
-                .setType("url").setSettings(Settings.builder()
-                        .put("url", repositoryLocation.toUri().toURL())
-                        .put("list_directories", randomBoolean())));
-        logger.info("--> restore index after deletion");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> list available shapshots");
-        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
-
-        logger.info("--> delete snapshot");
-        DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
-        assertAcked(deleteSnapshotResponse);
-
-        logger.info("--> list available shapshot again, no snapshots should be returned");
-        getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
-        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
-        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
-    }
-
     @TestLogging("_root:DEBUG")  // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
     // more logs we cannot find out why
     public void testReadonlyRepository() throws Exception {
@@ -1933,7 +1907,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             Settings initialSettings = initialSettingsBuilder.build();
             logger.info("--> using initial block settings {}", initialSettings.getAsMap());
 
-            if (!initialSettings.getAsMap().isEmpty()) {
+            if (!initialSettings.isEmpty()) {
                 logger.info("--> apply initial blocks to index");
                 client().admin().indices().prepareUpdateSettings("test-idx").setSettings(initialSettingsBuilder).get();
             }
@@ -2277,6 +2251,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                                                    State.ABORTED,
                                                    Collections.singletonList(indexId),
                                                    System.currentTimeMillis(),
+                                                   repositoryData.getGenId(),
                                                    shards.build()));
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
             }
@@ -2707,6 +2682,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 Settings.builder()
                     .put("location", repoPath)
                     .put("random_control_io_exception_rate", randomIntBetween(5, 20) / 100f)
+                    // test that we can take a snapshot after a failed one, even if a partial index-N was written
+                    .put("atomic_move", false)
                     .put("random", randomAsciiOfLength(10))));
 
         logger.info("--> indexing some data");
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
index 581280a5a9b..a8e1eeaec10 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
@@ -79,7 +79,7 @@ public class SnapshotRequestsTests extends ESTestCase {
 
         BytesReference bytes = builder.endObject().bytes();
 
-        request.source(XContentHelper.createParser(bytes).mapOrdered());
+        request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2());
 
         assertEquals("test-repo", request.repository());
         assertEquals("test-snap", request.snapshot());
@@ -137,7 +137,7 @@ public class SnapshotRequestsTests extends ESTestCase {
 
         BytesReference bytes = builder.endObject().bytes();
 
-        request.source(XContentHelper.createParser(bytes).mapOrdered());
+        request.source(XContentHelper.convertToMap(bytes, true, builder.contentType()).v2());
 
         assertEquals("test-repo", request.repository());
         assertEquals("test-snap", request.snapshot());
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
new file mode 100644
index 00000000000..4b5a9677945
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotsInProgressSerializationTests.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterState.Custom;
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.cluster.SnapshotsInProgress.Entry;
+import org.elasticsearch.cluster.SnapshotsInProgress.State;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.repositories.IndexId;
+import org.elasticsearch.test.AbstractDiffableWireSerializationTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SnapshotsInProgressSerializationTests extends AbstractDiffableWireSerializationTestCase<Custom> {
+
+    @Override
+    protected Custom createTestInstance() {
+        int numberOfSnapshots = randomInt(10);
+        List<Entry> entries = new ArrayList<>();
+        for (int i = 0; i < numberOfSnapshots; i++) {
+            entries.add(randomSnapshot());
+        }
+        return new SnapshotsInProgress(entries);
+    }
+
+    private Entry randomSnapshot() {
+        Snapshot snapshot = new Snapshot(randomAsciiOfLength(10), new SnapshotId(randomAsciiOfLength(10), randomAsciiOfLength(10)));
+        boolean includeGlobalState = randomBoolean();
+        boolean partial = randomBoolean();
+        State state = randomFrom(State.values());
+        int numberOfIndices = randomIntBetween(0, 10);
+        List<IndexId> indices = new ArrayList<>();
+        for (int i = 0; i < numberOfIndices; i++) {
+            indices.add(new IndexId(randomAsciiOfLength(10), randomAsciiOfLength(10)));
+        }
+        long startTime = randomLong();
+        long repositoryStateId = randomLong();
+        ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
+        int shardsCount = randomIntBetween(0, 10);
+        for (int j = 0; j < shardsCount; j++) {
+            ShardId shardId = new ShardId(new Index(randomAsciiOfLength(10), randomAsciiOfLength(10)), randomIntBetween(0, 10));
+            String nodeId = randomAsciiOfLength(10);
+            State shardState = randomFrom(State.values());
+            builder.put(shardId, new SnapshotsInProgress.ShardSnapshotStatus(nodeId, shardState));
+        }
+        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = builder.build();
+        return new Entry(snapshot, includeGlobalState, partial, state, indices, startTime, repositoryStateId, shards);
+    }
+
+    @Override
+    protected Writeable.Reader<Custom> instanceReader() {
+        return SnapshotsInProgress::new;
+    }
+
+    @Override
+    protected Custom makeTestChanges(Custom testInstance) {
+        SnapshotsInProgress snapshots = (SnapshotsInProgress) testInstance;
+        List<Entry> entries = new ArrayList<>(snapshots.entries());
+        if (randomBoolean() && entries.size() > 1) {
+            // remove some elements
+            int leaveElements = randomIntBetween(0, entries.size() - 1);
+            entries = randomSubsetOf(leaveElements, entries.toArray(new Entry[leaveElements]));
+        }
+        if (randomBoolean()) {
+            // add some elements
+            int addElements = randomInt(10);
+            for (int i = 0; i < addElements; i++) {
+                entries.add(randomSnapshot());
+            }
+        }
+        if (randomBoolean()) {
+            // modify some elements
+            for (int i = 0; i < entries.size(); i++) {
+                if (randomBoolean()) {
+                    entries.set(i, new Entry(entries.get(i), randomFrom(State.values()), entries.get(i).shards()));
+                }
+            }
+        }
+        return new SnapshotsInProgress(entries);
+    }
+
+    @Override
+    protected Writeable.Reader<Diff<Custom>> diffReader() {
+        return SnapshotsInProgress::readDiffFrom;
+    }
+
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 7fb717cd7c2..cd29c8ea031 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -46,6 +46,7 @@ import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.Repository;
@@ -63,8 +64,8 @@ public class MockRepository extends FsRepository {
 
 
         @Override
-        public Map<String, Repository.Factory> getRepositories(Environment env) {
-            return Collections.singletonMap("mock", (metadata) -> new MockRepository(metadata, env));
+        public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+            return Collections.singletonMap("mock", (metadata) -> new MockRepository(metadata, env, namedXContentRegistry));
         }
 
         @Override
@@ -99,10 +100,13 @@ public class MockRepository extends FsRepository {
 
     private volatile boolean blockOnDataFiles;
 
+    private volatile boolean atomicMove;
+
     private volatile boolean blocked = false;
 
-    public MockRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(overrideSettings(metadata, environment), environment);
+    public MockRepository(RepositoryMetaData metadata, Environment environment,
+                          NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(overrideSettings(metadata, environment), environment, namedXContentRegistry);
         randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
         randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
         useLuceneCorruptionException = metadata.settings().getAsBoolean("use_lucene_corruption", false);
@@ -112,6 +116,7 @@ public class MockRepository extends FsRepository {
         blockOnInitialization = metadata.settings().getAsBoolean("block_on_init", false);
         randomPrefix = metadata.settings().get("random", "default");
         waitAfterUnblock = metadata.settings().getAsLong("wait_after_unblock", 0L);
+        atomicMove = metadata.settings().getAsBoolean("atomic_move", true);
         logger.info("starting mock repository with random prefix {}", randomPrefix);
         mockBlobStore = new MockBlobStore(super.blobStore());
     }
@@ -152,23 +157,7 @@ public class MockRepository extends FsRepository {
         return mockBlobStore;
     }
 
-    public void unblock() {
-        unblockExecution();
-    }
-
-    public void blockOnDataFiles(boolean blocked) {
-        blockOnDataFiles = blocked;
-    }
-
-    public void blockOnControlFiles(boolean blocked) {
-        blockOnControlFiles = blocked;
-    }
-
-    public boolean blockOnDataFiles() {
-        return blockOnDataFiles;
-    }
-
-    public synchronized void unblockExecution() {
+    public synchronized void unblock() {
         blocked = false;
         // Clean blocking flags, so we wouldn't try to block again
         blockOnDataFiles = false;
@@ -177,6 +166,10 @@ public class MockRepository extends FsRepository {
         this.notifyAll();
     }
 
+    public void blockOnDataFiles(boolean blocked) {
+        blockOnDataFiles = blocked;
+    }
+
     public boolean blocked() {
         return blocked;
     }
@@ -287,7 +280,7 @@ public class MockRepository extends FsRepository {
             }
 
 
-            public MockBlobContainer(BlobContainer delegate) {
+            MockBlobContainer(BlobContainer delegate) {
                 super(delegate);
             }
 
@@ -322,16 +315,16 @@ public class MockRepository extends FsRepository {
 
             @Override
             public void move(String sourceBlob, String targetBlob) throws IOException {
-                if (RandomizedContext.current().getRandom().nextBoolean()) {
+                if (atomicMove) {
+                    // atomic move since this inherits from FsBlobContainer which provides atomic moves
+                    maybeIOExceptionOrBlock(targetBlob);
+                    super.move(sourceBlob, targetBlob);
+                } else {
                     // simulate a non-atomic move, since many blob container implementations
                     // will not have an atomic move, and we should be able to handle that
                     maybeIOExceptionOrBlock(targetBlob);
                     super.writeBlob(targetBlob, super.readBlob(sourceBlob), 0L);
                     super.deleteBlob(sourceBlob);
-                } else {
-                    // atomic move since this inherits from FsBlobContainer which provides atomic moves
-                    maybeIOExceptionOrBlock(targetBlob);
-                    super.move(sourceBlob, targetBlob);
                 }
             }
 
diff --git a/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java b/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
index 396a7da2673..ff8fcd8fbaf 100644
--- a/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
+++ b/core/src/test/java/org/elasticsearch/tasks/TaskResultTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.tasks;
 
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -67,7 +66,7 @@ public class TaskResultTests extends ESTestCase {
             result.toXContent(builder, ToXContent.EMPTY_PARAMS);
             try (XContentBuilder shuffled = shuffleXContent(builder);
                     XContentParser parser = createParser(shuffled)) {
-                read = TaskResult.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+                read = TaskResult.PARSER.apply(parser, null);
             }
         } catch (IOException e) {
             throw new IOException("Error processing [" + result + "]", e);
diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
index e18ba0fe322..12f84f23c28 100644
--- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
+++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
@@ -56,7 +56,7 @@ public class RandomShapeGenerator extends RandomGeoGenerator {
     protected static final double xDIVISIBLE = 2;
     protected static boolean ST_VALIDATE = true;
 
-    public static enum ShapeType {
+    public enum ShapeType {
         POINT, MULTIPOINT, LINESTRING, MULTILINESTRING, POLYGON;
         private static final ShapeType[] types = values();
 
diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
index 7fc1490de73..6dd4fa384e9 100644
--- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
+++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java
@@ -21,6 +21,7 @@ package org.elasticsearch.test.search.aggregations.bucket;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
@@ -80,7 +81,7 @@ public class SharedSignificantTermsTestMethods {
         if (type.equals("text")) {
             textMappings += ",fielddata=true";
         }
-        assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings)
+        assertAcked(testCase.prepareCreate(INDEX_NAME).setSettings(settings, XContentType.JSON)
                 .addMapping("doc", "text", textMappings, CLASS_FIELD, "type=keyword"));
         String[] gb = {"0", "1"};
         List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index d065abb884c..5e7227052d3 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -29,10 +29,10 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
+import java.util.function.Function;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasToString;
 
 public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
 
@@ -95,13 +95,8 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         });
     }
 
-    @FunctionalInterface
-    private interface SizeFunction {
-        int size(int numberOfProcessors);
-    }
-
     private int expectedSize(final String threadPoolName, final int numberOfProcessors) {
-        final Map<String, SizeFunction> sizes = new HashMap<>();
+        final Map<String, Function<Integer, Integer>> sizes = new HashMap<>();
         sizes.put(ThreadPool.Names.GENERIC, n -> ThreadPool.boundedBy(4 * n, 128, 512));
         sizes.put(ThreadPool.Names.MANAGEMENT, n -> 5);
         sizes.put(ThreadPool.Names.FLUSH, ThreadPool::halfNumberOfProcessorsMaxFive);
@@ -110,7 +105,7 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         sizes.put(ThreadPool.Names.SNAPSHOT, ThreadPool::halfNumberOfProcessorsMaxFive);
         sizes.put(ThreadPool.Names.FETCH_SHARD_STARTED, ThreadPool::twiceNumberOfProcessors);
         sizes.put(ThreadPool.Names.FETCH_SHARD_STORE, ThreadPool::twiceNumberOfProcessors);
-        return sizes.get(threadPoolName).size(numberOfProcessors);
+        return sizes.get(threadPoolName).apply(numberOfProcessors);
     }
 
     public void testScalingThreadPoolIsBounded() throws InterruptedException {
@@ -198,13 +193,4 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
             terminateThreadPoolIfNeeded(threadPool);
         }
     }
-
-    private static Settings settings(final String setting, final int value) {
-        return settings(setting, Integer.toString(value));
-    }
-
-    private static Settings settings(final String setting, final String value) {
-        return Settings.builder().put(setting, value).build();
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
index 01fc15e6f75..33047b2d12a 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
@@ -78,7 +78,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
         info.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
-        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false).v2();
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map, hasKey("foo"));
         map = (Map<String, Object>) map.get("foo");
         assertThat(map, hasKey("queue_size"));
@@ -100,7 +100,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
         info.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
 
-        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false).v2();
+        Map<String, Object> map = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
         assertThat(map, hasKey("foo"));
         map = (Map<String, Object>) map.get("foo");
         assertThat(map, hasKey("queue_size"));
diff --git a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
index aabc58887b2..0b46843cdb7 100644
--- a/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
@@ -208,8 +208,8 @@ public class TCPTransportTests extends ESTestCase {
 
                 @Override
                 public NodeChannels getConnection(DiscoveryNode node) {
-                    return new NodeChannels(node, new Object[ConnectionProfile.LIGHT_PROFILE.getNumConnections()],
-                        ConnectionProfile.LIGHT_PROFILE);
+                    return new NodeChannels(node, new Object[MockTcpTransport.LIGHT_PROFILE.getNumConnections()],
+                        MockTcpTransport.LIGHT_PROFILE);
                 }
             };
             DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
new file mode 100644
index 00000000000..68dffa1ded9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+
+public class TransportActionProxyTests extends ESTestCase {
+    protected ThreadPool threadPool;
+    // we use always a non-alpha or beta version here otherwise minimumCompatibilityVersion will be different for the two used versions
+    private static final Version CURRENT_VERSION = Version.fromString(String.valueOf(Version.CURRENT.major) + ".0.0");
+    protected static final Version version0 = CURRENT_VERSION.minimumCompatibilityVersion();
+
+    protected DiscoveryNode nodeA;
+    protected MockTransportService serviceA;
+
+    protected static final Version version1 = Version.fromId(CURRENT_VERSION.id + 1);
+    protected DiscoveryNode nodeB;
+    protected MockTransportService serviceB;
+
+    protected DiscoveryNode nodeC;
+    protected MockTransportService serviceC;
+
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getClass().getName());
+        serviceA = buildService(version0); // this one supports dynamic tracer updates
+        nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+        serviceB = buildService(version1); // this one doesn't support dynamic tracer updates
+        nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
+        serviceC = buildService(version1); // this one doesn't support dynamic tracer updates
+        nodeC = new DiscoveryNode("TS_C", serviceC.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        IOUtils.close(serviceA, serviceB, serviceC, () -> {
+            try {
+                terminate(threadPool);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        });
+    }
+
+    private MockTransportService buildService(final Version version) {
+        MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, version, threadPool, null);
+            service.start();
+            service.acceptIncomingRequests();
+        return service;
+
+    }
+
+
+    public void testSendMessage() throws InterruptedException {
+        serviceA.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_A";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceA, "/test", SimpleTestResponse::new);
+        serviceA.connectToNode(nodeB);
+
+        serviceB.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_B";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceB, "/test", SimpleTestResponse::new);
+        serviceB.connectToNode(nodeC);
+        serviceC.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_C";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceC, "/test", SimpleTestResponse::new);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        serviceA.sendRequest(nodeB, TransportActionProxy.getProxyAction("/test"), TransportActionProxy.wrapRequest(nodeC,
+            new SimpleTestRequest("TS_A")), new TransportResponseHandler<SimpleTestResponse>() {
+                @Override
+                public SimpleTestResponse newInstance() {
+                    return new SimpleTestResponse();
+                }
+
+                @Override
+                public void handleResponse(SimpleTestResponse response) {
+                    try {
+                        assertEquals("TS_C", response.targetNode);
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    try {
+                    throw new AssertionError(exp);
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+            });
+        latch.await();
+    }
+
+    public void testException() throws InterruptedException {
+        serviceA.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_A";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceA, "/test", SimpleTestResponse::new);
+        serviceA.connectToNode(nodeB);
+
+        serviceB.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                assertEquals(request.sourceNode, "TS_A");
+                SimpleTestResponse response = new SimpleTestResponse();
+                response.targetNode = "TS_B";
+                channel.sendResponse(response);
+            });
+        TransportActionProxy.registerProxyAction(serviceB, "/test", SimpleTestResponse::new);
+        serviceB.connectToNode(nodeC);
+        serviceC.registerRequestHandler("/test", SimpleTestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+                throw new ElasticsearchException("greetings from TS_C");
+            });
+        TransportActionProxy.registerProxyAction(serviceC, "/test", SimpleTestResponse::new);
+
+        CountDownLatch latch = new CountDownLatch(1);
+        serviceA.sendRequest(nodeB, TransportActionProxy.getProxyAction("/test"), TransportActionProxy.wrapRequest(nodeC,
+            new SimpleTestRequest("TS_A")), new TransportResponseHandler<SimpleTestResponse>() {
+                @Override
+                public SimpleTestResponse newInstance() {
+                    return new SimpleTestResponse();
+                }
+
+                @Override
+                public void handleResponse(SimpleTestResponse response) {
+                    try {
+                        fail("expected exception");
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public void handleException(TransportException exp) {
+                    try {
+                        Throwable cause = ExceptionsHelper.unwrapCause(exp);
+                        assertEquals("greetings from TS_C", cause.getMessage());
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.SAME;
+                }
+            });
+        latch.await();
+    }
+
+    public static class SimpleTestRequest extends TransportRequest {
+        String sourceNode;
+
+        public SimpleTestRequest(String sourceNode) {
+            this.sourceNode = sourceNode;
+        }
+        public SimpleTestRequest() {}
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            sourceNode = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(sourceNode);
+        }
+    }
+
+    public static class SimpleTestResponse extends TransportResponse {
+        String targetNode;
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            targetNode = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeString(targetNode);
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
index fd756f6790e..c00f5fb07a5 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportServiceHandshakeTests.java
@@ -67,20 +67,17 @@ public class TransportServiceHandshakeTests extends ESTestCase {
                         new NamedWriteableRegistry(Collections.emptyList()),
                         new NetworkService(settings, Collections.emptyList()));
         TransportService transportService = new MockTransportService(settings, transport, threadPool,
-            TransportService.NOOP_TRANSPORT_INTERCEPTOR, null);
+            TransportService.NOOP_TRANSPORT_INTERCEPTOR, (boundAddress) -> new DiscoveryNode(
+            nodeNameAndId,
+            nodeNameAndId,
+            boundAddress.publishAddress(),
+            emptyMap(),
+            emptySet(),
+            version), null);
         transportService.start();
         transportService.acceptIncomingRequests();
-        DiscoveryNode node =
-                new DiscoveryNode(
-                        nodeNameAndId,
-                        nodeNameAndId,
-                        transportService.boundAddress().publishAddress(),
-                        emptyMap(),
-                        emptySet(),
-                        version);
-        transportService.setLocalNode(node);
         transportServices.add(transportService);
-        return new NetworkHandle(transportService, node);
+        return new NetworkHandle(transportService, transportService.getLocalNode());
     }
 
     @After
@@ -113,7 +110,7 @@ public class TransportServiceHandshakeTests extends ESTestCase {
             emptyMap(),
             emptySet(),
             Version.CURRENT.minimumCompatibilityVersion());
-        try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode, ConnectionProfile.LIGHT_PROFILE)){
+        try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode, MockTcpTransport.LIGHT_PROFILE)){
             DiscoveryNode connectedNode = handleA.transportService.handshake(connection, timeout);
             assertNotNull(connectedNode);
             // the name and version should be updated
@@ -121,16 +118,6 @@ public class TransportServiceHandshakeTests extends ESTestCase {
             assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
             assertFalse(handleA.transportService.nodeConnected(discoveryNode));
         }
-
-        DiscoveryNode connectedNode =
-            handleA.transportService.connectToNodeAndHandshake(discoveryNode, timeout);
-        assertNotNull(connectedNode);
-
-        // the name and version should be updated
-        assertEquals(connectedNode.getName(), "TS_B");
-        assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
-        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
-
     }
 
     public void testMismatchedClusterName() {
@@ -145,7 +132,7 @@ public class TransportServiceHandshakeTests extends ESTestCase {
             Version.CURRENT.minimumCompatibilityVersion());
         IllegalStateException ex = expectThrows(IllegalStateException.class, () -> {
             try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode,
-                ConnectionProfile.LIGHT_PROFILE)) {
+                MockTcpTransport.LIGHT_PROFILE)) {
                 handleA.transportService.handshake(connection, timeout);
             }
         });
@@ -166,7 +153,7 @@ public class TransportServiceHandshakeTests extends ESTestCase {
             Version.CURRENT.minimumCompatibilityVersion());
         IllegalStateException ex = expectThrows(IllegalStateException.class, () -> {
             try (Transport.Connection connection = handleA.transportService.openConnection(discoveryNode,
-                ConnectionProfile.LIGHT_PROFILE)) {
+                MockTcpTransport.LIGHT_PROFILE)) {
                 handleA.transportService.handshake(connection, timeout);
             }
         });
@@ -178,7 +165,7 @@ public class TransportServiceHandshakeTests extends ESTestCase {
         private TransportService transportService;
         private DiscoveryNode discoveryNode;
 
-        public NetworkHandle(TransportService transportService, DiscoveryNode discoveryNode) {
+        NetworkHandle(TransportService transportService, DiscoveryNode discoveryNode) {
             this.transportService = transportService;
             this.discoveryNode = discoveryNode;
         }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
index 179d977ea5d..a63d16a9364 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
@@ -22,10 +22,10 @@ package org.elasticsearch.tribe;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -33,6 +33,8 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -53,6 +55,7 @@ import org.junit.AfterClass;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -117,9 +120,37 @@ public class TribeIT extends ESIntegTestCase {
                 .build();
     }
 
+    public static class TestCustomMetaDataPlugin extends Plugin {
+
+        private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
+
+        public TestCustomMetaDataPlugin() {
+            registerBuiltinWritables();
+        }
+
+        private <T extends MetaData.Custom> void registerMetaDataCustom(String name, Writeable.Reader<? extends T> reader,
+                                                                        Writeable.Reader<NamedDiff> diffReader) {
+            namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader));
+            namedWritables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, name, diffReader));
+        }
+
+        private void registerBuiltinWritables() {
+            registerMetaDataCustom(MergableCustomMetaData1.TYPE, MergableCustomMetaData1::readFrom, MergableCustomMetaData1::readDiffFrom);
+            registerMetaDataCustom(MergableCustomMetaData2.TYPE, MergableCustomMetaData2::readFrom, MergableCustomMetaData2::readDiffFrom);
+        }
+
+        @Override
+        public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+            return namedWritables;
+        }
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return getMockPlugins();
+        ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>();
+        plugins.addAll(getMockPlugins());
+        plugins.add(TestCustomMetaDataPlugin.class);
+        return plugins;
     }
 
     @Before
@@ -456,7 +487,6 @@ public class TribeIT extends ESIntegTestCase {
     }
 
     public void testMergingRemovedCustomMetaData() throws Exception {
-        MetaData.registerPrototype(MergableCustomMetaData1.TYPE, new MergableCustomMetaData1(""));
         removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
         removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
         MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1("a");
@@ -466,13 +496,12 @@ public class TribeIT extends ESIntegTestCase {
             putCustomMetaData(cluster1, customMetaData1);
             putCustomMetaData(cluster2, customMetaData2);
             assertCustomMetaDataUpdated(internalCluster(), customMetaData2);
-            removeCustomMetaData(cluster2, customMetaData2.type());
+            removeCustomMetaData(cluster2, customMetaData2.getWriteableName());
             assertCustomMetaDataUpdated(internalCluster(), customMetaData1);
         }
     }
 
     public void testMergingCustomMetaData() throws Exception {
-        MetaData.registerPrototype(MergableCustomMetaData1.TYPE, new MergableCustomMetaData1(""));
         removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
         removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
         MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1(randomAsciiOfLength(10));
@@ -490,8 +519,6 @@ public class TribeIT extends ESIntegTestCase {
     }
 
     public void testMergingMultipleCustomMetaData() throws Exception {
-        MetaData.registerPrototype(MergableCustomMetaData1.TYPE, new MergableCustomMetaData1(""));
-        MetaData.registerPrototype(MergableCustomMetaData2.TYPE, new MergableCustomMetaData2(""));
         removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
         removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
         MergableCustomMetaData1 firstCustomMetaDataType1 = new MergableCustomMetaData1(randomAsciiOfLength(10));
@@ -521,10 +548,10 @@ public class TribeIT extends ESIntegTestCase {
             assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
 
             // test removing custom md is propagates to tribe
-            removeCustomMetaData(cluster2, secondCustomMetaDataType1.type());
+            removeCustomMetaData(cluster2, secondCustomMetaDataType1.getWriteableName());
             assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
             assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
-            removeCustomMetaData(cluster2, secondCustomMetaDataType2.type());
+            removeCustomMetaData(cluster2, secondCustomMetaDataType2.getWriteableName());
             assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
             assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
         }
@@ -534,7 +561,7 @@ public class TribeIT extends ESIntegTestCase {
                                                     TestCustomMetaData expectedCustomMetaData) throws Exception {
         assertBusy(() -> {
             ClusterState tribeState = cluster.getInstance(ClusterService.class, cluster.getNodeNames()[0]).state();
-            MetaData.Custom custom = tribeState.metaData().custom(expectedCustomMetaData.type());
+            MetaData.Custom custom = tribeState.metaData().custom(expectedCustomMetaData.getWriteableName());
             assertNotNull(custom);
             assertThat(custom, equalTo(expectedCustomMetaData));
         });
@@ -546,9 +573,9 @@ public class TribeIT extends ESIntegTestCase {
     }
 
     private void putCustomMetaData(InternalTestCluster cluster, final TestCustomMetaData customMetaData) {
-        logger.info("putting custom_md type [{}] with data[{}] from [{}]", customMetaData.type(),
+        logger.info("putting custom_md type [{}] with data[{}] from [{}]", customMetaData.getWriteableName(),
                 customMetaData.getData(), cluster.getClusterName());
-        updateMetaData(cluster, builder -> builder.putCustom(customMetaData.type(), customMetaData));
+        updateMetaData(cluster, builder -> builder.putCustom(customMetaData.getWriteableName(), customMetaData));
     }
 
     private static void updateMetaData(InternalTestCluster cluster, UnaryOperator<MetaData.Builder> addCustoms) {
@@ -556,7 +583,7 @@ public class TribeIT extends ESIntegTestCase {
         final CountDownLatch latch = new CountDownLatch(1);
         clusterService.submitStateUpdateTask("update customMetaData", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
             @Override
-            public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 latch.countDown();
             }
 
@@ -564,7 +591,7 @@ public class TribeIT extends ESIntegTestCase {
             public ClusterState execute(ClusterState currentState) throws Exception {
                 MetaData.Builder builder = MetaData.builder(currentState.metaData());
                 builder = addCustoms.apply(builder);
-                return new ClusterState.Builder(currentState).metaData(builder).build();
+                return ClusterState.builder(currentState).metaData(builder).build();
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
index 7aea02c552b..424d7415204 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
@@ -19,13 +19,16 @@
 
 package org.elasticsearch.tribe;
 
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestCustomMetaData;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -51,7 +54,7 @@ public class TribeServiceTests extends ESTestCase {
         assertEquals("false", clientSettings.get("node.ingest"));
         assertEquals("false", clientSettings.get("node.local_storage"));
         assertEquals("3707202549613653169", clientSettings.get("node.id.seed")); // should be fixed by the parent id and tribe name
-        assertEquals(9, clientSettings.getAsMap().size());
+        assertEquals(9, clientSettings.size());
     }
 
     public void testEnvironmentSettings() {
@@ -185,13 +188,16 @@ public class TribeServiceTests extends ESTestCase {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new MergableCustomMetaData1(data);
+        public String getWriteableName() {
+            return TYPE;
         }
 
-        @Override
-        public String type() {
-            return TYPE;
+        public static MergableCustomMetaData1 readFrom(StreamInput in) throws IOException {
+            return readFrom(MergableCustomMetaData1::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
         }
 
         @Override
@@ -214,15 +220,19 @@ public class TribeServiceTests extends ESTestCase {
         }
 
         @Override
-        protected TestCustomMetaData newTestCustomMetaData(String data) {
-            return new MergableCustomMetaData2(data);
-        }
-
-        @Override
-        public String type() {
+        public String getWriteableName() {
             return TYPE;
         }
 
+        public static MergableCustomMetaData2 readFrom(StreamInput in) throws IOException {
+            return readFrom(MergableCustomMetaData2::new, in);
+        }
+
+        public static NamedDiff<MetaData.Custom> readDiffFrom(StreamInput in) throws IOException {
+            return readDiffFrom(TYPE, in);
+        }
+
+
         @Override
         public EnumSet<MetaData.XContentContext> context() {
             return EnumSet.of(MetaData.XContentContext.GATEWAY);
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
index 59826608456..2d28f75ff93 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
@@ -90,7 +90,7 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase {
         private Map<String, Object> params;
         private Map<String, Object> vars = new HashMap<>(2);
 
-        public CustomScript(Map<String, Object> params) {
+        CustomScript(Map<String, Object> params) {
             this.params = params;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 2c98b393642..75c3feb038a 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -34,6 +34,7 @@ import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.DocumentMissingException;
@@ -536,7 +537,7 @@ public class UpdateIT extends ESIntegTestCase {
         // With internal versions, tt means "if object is there with version X, update it or explode. If it is not there, index.
         client().prepareUpdate(indexOrAlias(), "type", "3")
                 .setScript(new Script(ScriptType.INLINE, "put_values", "", Collections.singletonMap("text", "v2")))
-                .setVersion(10).setUpsert("{ \"text\": \"v0\" }").get();
+                .setVersion(10).setUpsert("{ \"text\": \"v0\" }", XContentType.JSON).get();
         get = get("test", "type", "3");
         assertThat(get.getVersion(), equalTo(1L));
         assertThat((String) get.getSource().get("text"), equalTo("v0"));
@@ -885,7 +886,7 @@ public class UpdateIT extends ESIntegTestCase {
             private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
             private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);
 
-            public UpdateThread(int numberOfIds, int updatesPerId) {
+            UpdateThread(int numberOfIds, int updatesPerId) {
                 this.numberOfIds = numberOfIds;
                 this.updatesPerId = updatesPerId;
             }
@@ -893,7 +894,7 @@ public class UpdateIT extends ESIntegTestCase {
             final class UpdateListener implements ActionListener<UpdateResponse> {
                 int id;
 
-                public UpdateListener(int id) {
+                UpdateListener(int id) {
                     this.id = id;
                 }
 
@@ -915,7 +916,7 @@ public class UpdateIT extends ESIntegTestCase {
             final class DeleteListener implements ActionListener<DeleteResponse> {
                 int id;
 
-                public DeleteListener(int id) {
+                DeleteListener(int id) {
                     this.id = id;
                 }
 
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
index f14d91465f6..beab15165d6 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
@@ -245,7 +245,7 @@ public class UpdateNoopIT extends ESIntegTestCase {
 
     private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) {
         UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1")
-                .setDoc(xContentBuilder.bytes().utf8ToString())
+                .setDoc(xContentBuilder)
                 .setDocAsUpsert(true)
                 .setFields("_source");
         if (detectNoop != null) {
diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
index 0a26ce32e6d..eb5c00b76d9 100644
--- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
@@ -68,7 +68,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase {
         assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("_i:d:1")).execute().actionGet().isValid(), equalTo(false));
 
         assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("foo:1")).execute().actionGet().isValid(), equalTo(true));
-        assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("bar:hey")).execute().actionGet().isValid(), equalTo(false));
+        assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("bar:hey").lenient(false)).execute().actionGet().isValid(), equalTo(false));
 
         assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryStringQuery("nonexistent:hello")).execute().actionGet().isValid(), equalTo(true));
 
@@ -108,7 +108,7 @@ public class SimpleValidateQueryIT extends ESIntegTestCase {
                     .execute().actionGet();
             assertThat(response.isValid(), equalTo(true));
             assertThat(response.getQueryExplanation().size(), equalTo(1));
-            assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("_all:foo"));
+            assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("(foo:foo | baz:foo)"));
             assertThat(response.getQueryExplanation().get(0).getError(), nullValue());
         }
     }
diff --git a/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip
new file mode 100644
index 00000000000..9edf7d57527
Binary files /dev/null and b/core/src/test/resources/indices/bwc/compressed-repo-1.7.4.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.1.1.zip b/core/src/test/resources/indices/bwc/index-5.1.1.zip
index ab13e00b66e..49e89e6ccd3 100644
Binary files a/core/src/test/resources/indices/bwc/index-5.1.1.zip and b/core/src/test/resources/indices/bwc/index-5.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.1.2.zip b/core/src/test/resources/indices/bwc/index-5.1.2.zip
new file mode 100644
index 00000000000..a8c69190586
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/index-5.2.0.zip b/core/src/test/resources/indices/bwc/index-5.2.0.zip
new file mode 100644
index 00000000000..98e753755a1
Binary files /dev/null and b/core/src/test/resources/indices/bwc/index-5.2.0.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.1.1.zip b/core/src/test/resources/indices/bwc/repo-5.1.1.zip
index 01a9645e55e..9fef3e730a7 100644
Binary files a/core/src/test/resources/indices/bwc/repo-5.1.1.zip and b/core/src/test/resources/indices/bwc/repo-5.1.1.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.1.2.zip b/core/src/test/resources/indices/bwc/repo-5.1.2.zip
new file mode 100644
index 00000000000..f5e805a6f18
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.1.2.zip differ
diff --git a/core/src/test/resources/indices/bwc/repo-5.2.0.zip b/core/src/test/resources/indices/bwc/repo-5.2.0.zip
new file mode 100644
index 00000000000..a728215c4a6
Binary files /dev/null and b/core/src/test/resources/indices/bwc/repo-5.2.0.zip differ
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json
index 3b6e373b556..8f3f8b16aa1 100644
--- a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/filebeat-5.0.template.json
@@ -1,9 +1,6 @@
 {
   "mappings": {
     "_default_": {
-      "_all": {
-        "norms": false
-      },
       "_meta": {
         "version": "5.0.0-beta1"
       },
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json
index 1cabbeb3d0c..296bef5574d 100644
--- a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/logstash-5.0.template.json
@@ -6,7 +6,6 @@
   },
   "mappings" : {
     "_default_" : {
-      "_all" : {"enabled" : true, "norms" : false},
       "dynamic_templates" : [ {
         "message_field" : {
           "path_match" : "message",
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json
index 57786288db6..68423956e7e 100644
--- a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/metricbeat-5.0.template.json
@@ -1,9 +1,6 @@
 {
   "mappings": {
     "_default_": {
-      "_all": {
-        "norms": false
-      },
       "_meta": {
         "version": "5.0.0-beta1"
       },
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json
index 22b889176aa..8a6dc9f24f5 100644
--- a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/packetbeat-5.0.template.json
@@ -1,9 +1,6 @@
 {
   "mappings": {
     "_default_": {
-      "_all": {
-        "norms": false
-      },
       "_meta": {
         "version": "5.0.0-beta1"
       },
diff --git a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json
index 017a3b0c326..52bd6466098 100644
--- a/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json
+++ b/core/src/test/resources/org/elasticsearch/action/admin/indices/template/winlogbeat-5.0.template.json
@@ -1,9 +1,6 @@
 {
   "mappings": {
     "_default_": {
-      "_all": {
-        "norms": false
-      },
       "_meta": {
         "version": "5.0.0-beta1"
       },
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_default.json b/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_default.json
deleted file mode 100644
index 9dfd924803f..00000000000
--- a/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_default.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "test": {
-        "properties": {
-            "foo": {
-                "type": "nested",
-                "properties": {
-                    "bar": {
-                        "type": "keyword",
-                        "fields": {
-                            "lower": {
-                                "analyzer": "standard",
-                                "type": "text"
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json b/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json
deleted file mode 100644
index 8d9f8036499..00000000000
--- a/core/src/test/resources/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "test": {
-        "properties": {
-            "foo": {
-                "type": "nested",
-                "include_in_all": false,
-                "properties": {
-                    "bar": {
-                        "type": "keyword",
-                        "include_in_all": false,
-                        "fields": {
-                            "lower": {
-                                "analyzer": "standard",
-                                "type": "text"
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
deleted file mode 100644
index 02354ecccdd..00000000000
--- a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_create_index.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "mappings": {
-    "type": {
-      "_all": {
-        "store": true,
-        "store_term_vectors": true,
-        "store_term_vector_offsets": true,
-        "store_term_vector_positions": true,
-        "store_term_vector_payloads": true,
-        "norms": false,
-        "analyzer": "standard",
-        "search_analyzer": "whitespace",
-        "similarity": "my_similarity"
-      }
-    }
-  },
-  "settings": {
-    "similarity": {
-      "my_similarity": {
-        "type": "DFR",
-        "basic_model": "g",
-        "after_effect": "l",
-        "normalization": "h2",
-        "normalization.h2.c": "3.0"
-      }
-    }
-  }
-}
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json b/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
deleted file mode 100644
index f9ab776ce5a..00000000000
--- a/core/src/test/resources/org/elasticsearch/index/mapper/update/all_mapping_update_with_conflicts.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
-  "type": {
-    "_all": {
-      "store": false,
-      "enabled": true,
-      "store_term_vectors": false,
-      "store_term_vector_offsets": false,
-      "store_term_vector_positions": false,
-      "store_term_vector_payloads": false,
-      "norms": true,
-      "analyzer": "whitespace",
-      "search_analyzer": "standard",
-      "similarity": "BM25"
-    }
-  }
-}
diff --git a/core/src/test/resources/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json b/core/src/test/resources/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json
deleted file mode 100644
index 5f49a6f558d..00000000000
--- a/core/src/test/resources/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json
+++ /dev/null
@@ -1 +0,0 @@
-{"type":{"_timestamp":{"enabled":false}}}
\ No newline at end of file
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json
index 05de13b4261..1a96fd71333 100644
--- a/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index-with-all.json
@@ -3,6 +3,9 @@
     "index": {
       "number_of_shards": 1,
       "number_of_replicas": 0,
+      "version": {
+        "created": "5000099"
+      },
       "analysis": {
         "analyzer": {
           "my_ngrams": {
diff --git a/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
index 1a67abcfcb3..86dde5aaf88 100644
--- a/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
+++ b/core/src/test/resources/org/elasticsearch/search/query/all-query-index.json
@@ -23,9 +23,6 @@
   },
   "mappings": {
     "doc": {
-      "_all": {
-        "enabled": false
-      },
       "properties": {
         "f1": {"type": "text"},
         "f2": {"type": "keyword"},
diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py
index a6814e52f8f..4703759e123 100644
--- a/dev-tools/create_bwc_index.py
+++ b/dev-tools/create_bwc_index.py
@@ -48,6 +48,28 @@ def rarely():
 def frequently():
   return not rarely()
 
+def capabilities_of(version):
+  current_version = parse_version(version)
+
+  return {
+    'warmers': current_version < parse_version('2.0.0-alpha1'),
+    'dots_in_field_names': current_version >= parse_version('2.4.0'),
+    'lenient_booleans': current_version < parse_version('6.0.0-alpha1')
+  }
+
+
+def falsy(lenient):
+  return random.choice(['off', 'no', '0', 0, 'false', False]) if lenient else False
+
+
+def truthy(lenient):
+  return random.choice(['on', 'yes', '1', 1, 'true', True]) if lenient else True
+
+
+def random_bool(lenient):
+  return random.choice([falsy, truthy])(lenient)
+
+
 # asserts the correctness of the given hits given they are sorted asc
 def assert_sort(hits):
   values = [hit['sort'] for hit in hits['hits']['hits']]
@@ -59,19 +81,23 @@ def assert_sort(hits):
 
 # Indexes the given number of document into the given index
 # and randomly runs refresh, optimize and flush commands
-def index_documents(es, index_name, type, num_docs, supports_dots_in_field_names):
+def index_documents(es, index_name, type, num_docs, capabilities):
   logging.info('Indexing %s docs' % num_docs)
-  index(es, index_name, type, num_docs, supports_dots_in_field_names, True)
+  index(es, index_name, type, num_docs, capabilities, flush=True)
   logging.info('Flushing index')
   es.indices.flush(index=index_name)
 
-def index(es, index_name, type, num_docs, supports_dots_in_field_names, flush=False):
+def index(es, index_name, type, num_docs, capabilities, flush=False):
   for id in range(0, num_docs):
-    body = {'string': str(random.randint(0, 100)),
-            'long_sort': random.randint(0, 100),
-            'double_sort' : float(random.randint(0, 100)),
-            'bool' : random.choice([True, False])}
-    if supports_dots_in_field_names:
+    lenient_bool = capabilities['lenient_booleans']
+    body = {
+          'string': str(random.randint(0, 100)),
+          'long_sort': random.randint(0, 100),
+          'double_sort': float(random.randint(0, 100)),
+          # be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
+          'bool': random_bool(lenient_bool) if id > 0 else random.choice([True, False])
+        }
+    if capabilities['dots_in_field_names']:
       body['field.with.dots'] = str(random.randint(0, 100))
 
     body['binary'] = base64.b64encode(bytearray(random.getrandbits(8) for _ in range(16))).decode('ascii')
@@ -83,10 +109,14 @@ def index(es, index_name, type, num_docs, supports_dots_in_field_names, flush=Fa
     if rarely() and flush:
       es.indices.flush(index=index_name, force=frequently())
 
-def reindex_docs(es, index_name, type, num_docs, supports_dots_in_field_names):
+def reindex_docs(es, index_name, type, num_docs, capabilities):
   logging.info('Re-indexing %s docs' % num_docs)
+  # TODO: Translog recovery fails on mixed representation of booleans as strings / booleans (e.g. "true", true)
+  # (see gradle :core:test -Dtests.seed=AF7BB7B3FA387AAE -Dtests.class=org.elasticsearch.index.engine.InternalEngineTests
+  #      -Dtests.method="testUpgradeOldIndex")
+  capabilities['lenient_booleans'] = False
   # reindex some docs after the flush such that we have something in the translog
-  index(es, index_name, type, num_docs, supports_dots_in_field_names)
+  index(es, index_name, type, num_docs, capabilities)
 
 def delete_by_query(es, version, index_name, doc_type):
 
@@ -200,9 +230,12 @@ def generate_index(client, version, index_name):
   client.indices.delete(index=index_name, ignore=404)
   logging.info('Create single shard test index')
 
+  capabilities = capabilities_of(version)
+  lenient_booleans = capabilities['lenient_booleans']
+
   mappings = {}
   warmers = {}
-  if parse_version(version) < parse_version('2.0.0-alpha1'):
+  if capabilities['warmers']:
     warmers['warmer1'] = {
       'source': {
         'query': {
@@ -249,7 +282,7 @@ def generate_index(client, version, index_name):
     }
     mappings['meta_fields'] = {
       '_routing': {
-        'required': 'false'
+        'required': falsy(lenient_booleans)
       },
     }
     mappings['custom_formats'] = {
@@ -266,13 +299,12 @@ def generate_index(client, version, index_name):
     }
     mappings['auto_boost'] = {
       '_all': {
-        'auto_boost': True
+        'auto_boost': truthy(lenient_booleans)
       }
     }
   mappings['doc'] = {'properties' : {}}
-  supports_dots_in_field_names = parse_version(version) >= parse_version("2.4.0")
-  if supports_dots_in_field_names:
 
+  if capabilities['dots_in_field_names']:
     if parse_version(version) < parse_version("5.0.0-alpha1"):
       mappings["doc"]['properties'].update({
         'field.with.dots': {
@@ -320,7 +352,7 @@ def generate_index(client, version, index_name):
       'properties': {
         'string_with_norms_disabled': {
           'type': 'text',
-          'norms' : False
+          'norms': False
         },
         'string_with_norms_enabled': {
           'type': 'keyword',
@@ -340,7 +372,7 @@ def generate_index(client, version, index_name):
   # test back-compat of stored binary fields
   mappings['doc']['properties']['binary'] = {
     'type': 'binary',
-    'store': True,
+    'store': truthy(lenient_booleans),
     }
 
   settings = {
@@ -372,13 +404,13 @@ def generate_index(client, version, index_name):
     # lighter index for it to keep bw tests reasonable
     # see https://github.com/elastic/elasticsearch/issues/5817
     num_docs = int(num_docs / 10)
-  index_documents(client, index_name, 'doc', num_docs, supports_dots_in_field_names)
+  index_documents(client, index_name, 'doc', num_docs, capabilities)
   if parse_version(version) < parse_version('5.1.0'):
     logging.info("Adding a alias that can't be created in 5.1+ so we can assert that we can still use it")
     client.indices.put_alias(index=index_name, name='#' + index_name)
   logging.info('Running basic asserts on the data added')
   run_basic_asserts(client, version, index_name, 'doc', num_docs)
-  return num_docs, supports_dots_in_field_names
+  return num_docs, capabilities
 
 def snapshot_index(client, version, repo_dir):
   persistent = {
@@ -400,7 +432,9 @@ def snapshot_index(client, version, repo_dir):
     },
     "mappings": {
       "type1": {
-        "_source": { "enabled" : False }
+        "_source": {
+          "enabled": falsy(capabilities_of(version)['lenient_booleans'])
+        }
       }
     },
     "aliases": {
@@ -488,7 +522,7 @@ def create_bwc_index(cfg, version):
     node = start_node(version, release_dir, data_dir, repo_dir, cfg.tcp_port, cfg.http_port)
     client = create_client(cfg.http_port)
     index_name = 'index-%s' % version.lower()
-    num_docs, supports_dots_in_field_names = generate_index(client, version, index_name)
+    num_docs, capabilities = generate_index(client, version, index_name)
     if snapshot_supported:
       snapshot_index(client, version, repo_dir)
 
@@ -497,7 +531,7 @@ def create_bwc_index(cfg, version):
     # will already have the deletions applied on upgrade.
     if version.startswith('0.') or version.startswith('1.'):
       delete_by_query(client, version, index_name, 'doc')
-    reindex_docs(client, index_name, 'doc', min(100, num_docs), supports_dots_in_field_names)
+    reindex_docs(client, index_name, 'doc', min(100, num_docs), capabilities)
 
     shutdown_node(node)
     node = None
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 2cfd7ebbbce..7c01acc1c91 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -79,7 +79,7 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':modules:') }.each
   restTestExpansions['expected.modules.count'] += 1
 }
 
-// Integ tests work over the rest http layer, so we need a transport included with the integ test zip. 
+// Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
 // All transport modules are included so that they may be randomized for testing
 task buildTransportModules(type: Sync) {
   into 'build/transport-modules'
@@ -104,6 +104,7 @@ subprojects {
   /*****************************************************************************
    *                            Rest test config                               *
    *****************************************************************************/
+  apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
   project.integTest {
     dependsOn project.assemble
@@ -116,7 +117,7 @@ subprojects {
       mustRunAfter ':distribution:integ-test-zip:integTest#stop'
     }
   }
-  
+
   processTestResources {
     inputs.properties(project(':distribution').restTestExpansions)
     MavenFilteringHack.filter(it, project(':distribution').restTestExpansions)
diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
index a63b304a1d4..9a7978f69bf 100644
--- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
+++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class DebClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
index 71ec1948662..c61b736bf6d 100644
--- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test.rest;
 
+import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.elasticsearch.client.Response;
 
@@ -47,12 +48,14 @@ public class CreatedLocationHeaderIT extends ESRestTestCase {
     public void testUpsert() throws IOException {
         locationTestCase(client().performRequest("POST", "test/test/1/_update", emptyMap(), new StringEntity("{"
                 + "\"doc\": {\"test\": \"test\"},"
-                + "\"doc_as_upsert\": true}")));
+                + "\"doc_as_upsert\": true}", ContentType.APPLICATION_JSON)));
     }
 
     private void locationTestCase(String method, String url) throws IOException {
-        locationTestCase(client().performRequest(method, url, emptyMap(), new StringEntity("{\"test\": \"test\"}")));
-        locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(), new StringEntity("{\"test\": \"test\"}")));
+        locationTestCase(client().performRequest(method, url, emptyMap(),
+            new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
+        locationTestCase(client().performRequest(method, url + "?routing=cat", emptyMap(),
+            new StringEntity("{\"test\": \"test\"}", ContentType.APPLICATION_JSON)));
     }
 
     private void locationTestCase(Response response) throws IOException {
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
index c81ff7439f9..c3a038392cb 100644
--- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class IntegTestZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
new file mode 100644
index 00000000000..ca69a95deed
--- /dev/null
+++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.ResponseListener;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Tests that wait for refresh is fired if the index is closed.
+ */
+public class WaitForRefreshAndCloseTests extends ESRestTestCase {
+    @Before
+    public void setupIndex() throws IOException {
+        try {
+            client().performRequest("DELETE", indexName());
+        } catch (ResponseException e) {
+            // If we get an error, it should be because the index doesn't exist
+            assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
+        }
+        client().performRequest("PUT", indexName(), emptyMap(), new StringEntity("{\"settings\":{\"refresh_interval\":-1}}"));
+    }
+
+    @After
+    public void cleanupIndex() throws IOException {
+        client().performRequest("DELETE", indexName());
+    }
+
+    private String indexName() {
+        return getTestName().toLowerCase(Locale.ROOT);
+    }
+
+    private String docPath() {
+        return indexName() + "/test/1";
+    }
+
+    public void testIndexAndThenClose() throws Exception {
+        closeWhileListenerEngaged(start("PUT", "", new StringEntity("{\"test\":\"test\"}")));
+    }
+
+    public void testUpdateAndThenClose() throws Exception {
+        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}"));
+        closeWhileListenerEngaged(start("POST", "/_update", new StringEntity("{\"doc\":{\"name\":\"test\"}}")));
+    }
+
+    public void testDeleteAndThenClose() throws Exception {
+        client().performRequest("PUT", docPath(), emptyMap(), new StringEntity("{\"test\":\"test\"}"));
+        closeWhileListenerEngaged(start("DELETE", "", null));
+    }
+
+    private void closeWhileListenerEngaged(ActionFuture<String> future) throws Exception {
+        // Wait for the refresh listener to start waiting
+        assertBusy(() -> {
+            Map<String, Object> stats;
+            try {
+                stats = entityAsMap(client().performRequest("GET", indexName() + "/_stats/refresh"));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            @SuppressWarnings("unchecked")
+            Map<String, Object> indices = (Map<String, Object>) stats.get("indices");
+            @SuppressWarnings("unchecked")
+            Map<String, Object> theIndex = (Map<String, Object>) indices.get(indexName());
+            @SuppressWarnings("unchecked")
+            Map<String, Object> total = (Map<String, Object>) theIndex.get("total");
+            @SuppressWarnings("unchecked")
+            Map<String, Object> refresh = (Map<String, Object>) total.get("refresh");
+            int listeners = (int) refresh.get("listeners");
+            assertEquals(1, listeners);
+        });
+
+        // Close the index. That should flush the listener.
+        client().performRequest("POST", indexName() + "/_close");
+
+        // The request shouldn't fail. It certainly shouldn't hang.
+        future.get();
+    }
+
+    private ActionFuture<String> start(String method, String path, HttpEntity body) {
+        PlainActionFuture<String> future = new PlainActionFuture<>();
+        Map<String, String> params = new HashMap<>();
+        params.put("refresh", "wait_for");
+        params.put("error_trace", "");
+        client().performRequestAsync(method, docPath() + path, params, body, new ResponseListener() {
+            @Override
+            public void onSuccess(Response response) {
+                try {
+                    future.onResponse(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
+                } catch (IOException e) {
+                    future.onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception exception) {
+                future.onFailure(exception);
+            }
+        });
+        return future;
+    }
+}
diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
index 9569dfe4d10..86b49321808 100644
--- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
+++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class RpmClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/src/main/resources/bin/elasticsearch-keystore b/distribution/src/main/resources/bin/elasticsearch-keystore
new file mode 100755
index 00000000000..d99d11aed49
--- /dev/null
+++ b/distribution/src/main/resources/bin/elasticsearch-keystore
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+CDPATH=""
+SCRIPT="$0"
+
+# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path.
+while [ -h "$SCRIPT" ] ; do
+  ls=`ls -ld "$SCRIPT"`
+  # Drop everything prior to ->
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    SCRIPT="$link"
+  else
+    SCRIPT=`dirname "$SCRIPT"`/"$link"
+  fi
+done
+
+# determine elasticsearch home
+ES_HOME=`dirname "$SCRIPT"`/..
+
+# make ELASTICSEARCH_HOME absolute
+ES_HOME=`cd "$ES_HOME"; pwd`
+
+
+# Sets the default values for elasticsearch variables used in this script
+if [ -z "$CONF_DIR" ]; then
+  CONF_DIR="${path.conf}"
+fi
+
+# The default env file is defined at building/packaging time.
+# For a ${project.name} package, the value is "${path.env}".
+ES_ENV_FILE="${path.env}"
+
+# If an include is specified with the ES_INCLUDE environment variable, use it
+if [ -n "$ES_INCLUDE" ]; then
+    ES_ENV_FILE="$ES_INCLUDE"
+fi
+
+# Source the environment file
+if [ -n "$ES_ENV_FILE" ]; then
+
+  # If the ES_ENV_FILE is not found, try to resolve the path
+  # against the ES_HOME directory
+  if [ ! -f "$ES_ENV_FILE" ]; then
+      ES_ENV_FILE="$ELASTIC_HOME/$ES_ENV_FILE"
+  fi
+
+  . "$ES_ENV_FILE"
+  if [ $? -ne 0 ]; then
+      echo "Unable to source environment file: $ES_ENV_FILE" >&2
+      exit 1
+  fi
+fi
+
+# don't let JAVA_TOOL_OPTIONS slip in (e.g. crazy agents in ubuntu)
+# works around https://bugs.launchpad.net/ubuntu/+source/jayatana/+bug/1441487
+if [ "x$JAVA_TOOL_OPTIONS" != "x" ]; then
+    echo "Warning: Ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
+    unset JAVA_TOOL_OPTIONS
+fi
+
+# CONF_FILE setting was removed
+if [ ! -z "$CONF_FILE" ]; then
+    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
+    exit 1
+fi
+
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA=$JAVA_HOME/bin/java
+else
+    JAVA=`which java`
+fi
+
+if [ ! -x "$JAVA" ]; then
+    echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
+    exit 1
+fi
+
+# full hostname passed through cut for portability on systems that do not support hostname -s
+# export on separate line for shells that do not support combining definition and export
+HOSTNAME=`hostname | cut -d. -f1`
+export HOSTNAME
+
+declare -a args=("$@")
+path_props=(-Des.path.home="$ES_HOME")
+
+if [ -e "$CONF_DIR" ]; then
+  path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR")
+fi
+
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.common.settings.KeyStoreCli "${args[@]}"
diff --git a/distribution/src/main/resources/bin/elasticsearch-keystore.bat b/distribution/src/main/resources/bin/elasticsearch-keystore.bat
new file mode 100644
index 00000000000..ceb7e96665d
--- /dev/null
+++ b/distribution/src/main/resources/bin/elasticsearch-keystore.bat
@@ -0,0 +1,30 @@
+@echo off
+
+SETLOCAL enabledelayedexpansion
+
+IF DEFINED JAVA_HOME (
+  set JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
+
+set SCRIPT_DIR=%~dp0
+for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
+
+TITLE Elasticsearch Plugin Manager ${project.version}
+
+SET path_props=-Des.path.home="%ES_HOME%"
+IF DEFINED CONF_DIR (
+  SET path_props=!path_props! -Des.path.conf="%CONF_DIR%"
+)
+
+SET args=%*
+SET HOSTNAME=%COMPUTERNAME%
+
+"%JAVA%" %ES_JAVA_OPTS% !path_props! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.common.settings.KeyStoreCli" !args!
+
+ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch-service.bat b/distribution/src/main/resources/bin/elasticsearch-service.bat
index 173ee75e8c0..d06de4c5bea 100644
--- a/distribution/src/main/resources/bin/elasticsearch-service.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-service.bat
@@ -4,7 +4,7 @@ SETLOCAL enabledelayedexpansion
 TITLE Elasticsearch Service ${project.version}
 
 IF DEFINED JAVA_HOME (
-  SET JAVA="%JAVA_HOME%"\bin\java.exe
+  SET JAVA="%JAVA_HOME%\bin\java.exe"
 ) ELSE (
   FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
 )
@@ -121,19 +121,19 @@ echo Installing service      :  "%SERVICE_ID%"
 echo Using JAVA_HOME (%ARCH%):  "%JAVA_HOME%"
 
 rem Check JVM server dll first
-if exist "%JAVA_HOME%"\jre\bin\server\jvm.dll (
+if exist "%JAVA_HOME%\jre\bin\server\jvm.dll" (
 	set JVM_DLL=\jre\bin\server\jvm.dll
 	goto foundJVM
 )
 
 rem Check 'server' JRE (JRE installed on Windows Server)
-if exist "%JAVA_HOME%"\bin\server\jvm.dll (
+if exist "%JAVA_HOME%\bin\server\jvm.dll" (
 	set JVM_DLL=\bin\server\jvm.dll
 	goto foundJVM
 )
 
 rem Fallback to 'client' JRE
-if exist "%JAVA_HOME%"\bin\client\jvm.dll (
+if exist "%JAVA_HOME%\bin\client\jvm.dll" (
 	set JVM_DLL=\bin\client\jvm.dll
 	echo Warning: JAVA_HOME points to a JRE and not JDK installation; a client (not a server^) JVM will be used...
 ) else (
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat
index d1f443aa8a3..16293ca406a 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.in.bat
@@ -1,7 +1,7 @@
 @echo off
 
 IF DEFINED JAVA_HOME (
-  set JAVA="%JAVA_HOME%"\bin\java.exe
+  set JAVA="%JAVA_HOME%\bin\java.exe"
 ) ELSE (
   FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
 )
diff --git a/distribution/src/main/resources/config/elasticsearch.yml b/distribution/src/main/resources/config/elasticsearch.yml
index 3d732e74c0e..15e841fe390 100644
--- a/distribution/src/main/resources/config/elasticsearch.yml
+++ b/distribution/src/main/resources/config/elasticsearch.yml
@@ -7,8 +7,8 @@
 # The primary way of configuring a node is via this file. This template lists
 # the most important settings you may want to configure for a production cluster.
 #
-# Please see the documentation for further information on configuration options:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
 #
 # ---------------------------------- Cluster -----------------------------------
 #
@@ -58,8 +58,7 @@
 #
 #http.port: 9200
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
+# For more information, consult the network module documentation.
 #
 # --------------------------------- Discovery ----------------------------------
 #
@@ -68,12 +67,11 @@
 #
 #discovery.zen.ping.unicast.hosts: ["host1", "host2"]
 #
-# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
+# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
 #
 #discovery.zen.minimum_master_nodes: 3
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
+# For more information, consult the zen discovery module documentation.
 #
 # ---------------------------------- Gateway -----------------------------------
 #
@@ -81,8 +79,7 @@
 #
 #gateway.recover_after_nodes: 3
 #
-# For more information, see the documentation at:
-# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
+# For more information, consult the gateway module documentation.
 #
 # ---------------------------------- Various -----------------------------------
 #
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index 11ec1cd66dc..e08ffcabbb0 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -65,9 +65,10 @@
 # use old-style file permissions on JDK9
 -Djdk.io.permissionsUseCanonicalPath=true
 
-# flags to keep Netty from being unsafe
+# flags to configure Netty
 -Dio.netty.noUnsafe=true
 -Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
 
 # log4j 2
 -Dlog4j.shutdownHookEnabled=false
@@ -96,3 +97,10 @@
 # log GC status to a file with time stamps
 # ensure the directory exists
 #-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
diff --git a/distribution/src/main/resources/config/log4j2.properties b/distribution/src/main/resources/config/log4j2.properties
index 3702afff9f3..f344d0aee55 100644
--- a/distribution/src/main/resources/config/log4j2.properties
+++ b/distribution/src/main/resources/config/log4j2.properties
@@ -11,10 +11,10 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
@@ -26,10 +26,10 @@ rootLogger.appenderRef.rolling.ref = rolling
 
 appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_rolling.layout.type = PatternLayout
 appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.deprecation_rolling.policies.size.size = 1GB
@@ -43,10 +43,10 @@ logger.deprecation.additivity = false
 
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
 appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_search_slowlog_rolling.policies.time.interval = 1
@@ -59,10 +59,10 @@ logger.index_search_slowlog_rolling.additivity = false
 
 appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
 appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_indexing_slowlog_rolling.policies.time.interval = 1
diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
index 0c811c383d0..a86e398830c 100644
--- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
+++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class TarClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
index 52581c8e765..93c31c4ab65 100644
--- a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
+++ b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class ZipClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/docs/build.gradle b/docs/build.gradle
index 5347e529fa3..225aa3957b5 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -24,20 +24,14 @@ apply plugin: 'elasticsearch.docs-test'
  * only remove entries from this list. When it is empty we'll remove it
  * entirely and have a party! There will be cake and everything.... */
 buildRestTests.expectedUnconvertedCandidates = [
-  'reference/aggregations/bucket/datehistogram-aggregation.asciidoc',
-  'reference/aggregations/bucket/daterange-aggregation.asciidoc',
-  'reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc',
-  'reference/aggregations/bucket/filter-aggregation.asciidoc',
   'reference/aggregations/bucket/geodistance-aggregation.asciidoc',
   'reference/aggregations/bucket/geohashgrid-aggregation.asciidoc',
-  'reference/aggregations/bucket/global-aggregation.asciidoc',
   'reference/aggregations/bucket/histogram-aggregation.asciidoc',
   'reference/aggregations/bucket/iprange-aggregation.asciidoc',
   'reference/aggregations/bucket/missing-aggregation.asciidoc',
   'reference/aggregations/bucket/nested-aggregation.asciidoc',
   'reference/aggregations/bucket/range-aggregation.asciidoc',
   'reference/aggregations/bucket/reverse-nested-aggregation.asciidoc',
-  'reference/aggregations/bucket/sampler-aggregation.asciidoc',
   'reference/aggregations/bucket/significantterms-aggregation.asciidoc',
   'reference/aggregations/bucket/terms-aggregation.asciidoc',
   'reference/aggregations/matrix/stats-aggregation.asciidoc',
@@ -46,15 +40,12 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/aggregations/metrics/extendedstats-aggregation.asciidoc',
   'reference/aggregations/metrics/geobounds-aggregation.asciidoc',
   'reference/aggregations/metrics/geocentroid-aggregation.asciidoc',
-  'reference/aggregations/metrics/max-aggregation.asciidoc',
-  'reference/aggregations/metrics/min-aggregation.asciidoc',
   'reference/aggregations/metrics/percentile-aggregation.asciidoc',
   'reference/aggregations/metrics/percentile-rank-aggregation.asciidoc',
   'reference/aggregations/metrics/scripted-metric-aggregation.asciidoc',
   'reference/aggregations/metrics/stats-aggregation.asciidoc',
   'reference/aggregations/metrics/sum-aggregation.asciidoc',
   'reference/aggregations/metrics/tophits-aggregation.asciidoc',
-  'reference/aggregations/metrics/valuecount-aggregation.asciidoc',
   'reference/aggregations/pipeline.asciidoc',
   'reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc',
   'reference/aggregations/pipeline/bucket-script-aggregation.asciidoc',
@@ -122,7 +113,6 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/index-modules/translog.asciidoc',
   'reference/indices/analyze.asciidoc',
   'reference/indices/flush.asciidoc',
-  'reference/indices/get-field-mapping.asciidoc',
   'reference/indices/get-settings.asciidoc',
   'reference/indices/put-mapping.asciidoc',
   'reference/indices/recovery.asciidoc',
@@ -145,6 +135,7 @@ buildRestTests.expectedUnconvertedCandidates = [
   'reference/modules/scripting/security.asciidoc',
   'reference/modules/scripting/using.asciidoc',
   'reference/modules/transport.asciidoc',
+  'reference/modules/cross-cluster-search.asciidoc', // this is hart to test since we need 2 clusters -- maybe we can trick it into referencing itself...
   'reference/query-dsl/exists-query.asciidoc',
   'reference/query-dsl/function-score-query.asciidoc',
   'reference/query-dsl/geo-shape-query.asciidoc',
@@ -169,8 +160,7 @@ integTest {
     Closure configFile = {
       extraConfigFile it, "src/test/cluster/config/$it"
     }
-    configFile 'scripts/my_script.js'
-    configFile 'scripts/my_script.py'
+    configFile 'scripts/my_script.painless'
     configFile 'scripts/my_init_script.painless'
     configFile 'scripts/my_map_script.painless'
     configFile 'scripts/my_combine_script.painless'
@@ -217,6 +207,16 @@ Closure setupTwitter = { String name, int count ->
         settings:
           number_of_shards: 1
           number_of_replicas: 1
+        mappings:
+          tweet:
+            properties:
+              user:
+                type: keyword
+                doc_values: true
+              date:
+                type: date
+              likes:
+                type: long
   - do:
     bulk:
       index: twitter
@@ -383,3 +383,59 @@ buildRestTests.setups['index_boost'] = '''
         index: index1
         name: alias1
 '''
+// Used by sampler and diversified-sampler aggregation docs
+buildRestTests.setups['stackoverflow'] = '''
+  - do:
+    indices.create:
+      index: stackoverflow
+      body:
+        settings:
+          number_of_shards: 1
+          number_of_replicas: 1
+        mappings:
+          question:
+            properties:
+              author:
+                type: keyword
+              tags:
+                type: keyword
+  - do:
+    bulk:
+      index: stackoverflow
+      type: question
+      refresh: true
+      body: |'''
+      
+// Make Kibana strongly connected to elasticsearch and logstash
+// Make Kibana rarer (and therefore higher-ranking) than Javascript
+// Make Javascript strongly connected to jquery and angular
+// Make Cabana strongly connected to elasticsearch but only as a result of a single author
+
+for (int i = 0; i < 150; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+        {"index":{}}
+        {"author": "very_relevant_$i", "tags": ["elasticsearch", "kibana"]}"""
+}
+for (int i = 0; i < 50; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+        {"index":{}}
+        {"author": "very_relevant_$i", "tags": ["logstash", "kibana"]}"""
+}
+for (int i = 0; i < 200; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+        {"index":{}}
+        {"author": "partially_relevant_$i", "tags": ["javascript", "jquery"]}"""
+}
+for (int i = 0; i < 200; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+        {"index":{}}
+        {"author": "partially_relevant_$i", "tags": ["javascript", "angular"]}"""
+}
+for (int i = 0; i < 50; i++) {
+  buildRestTests.setups['stackoverflow'] += """
+        {"index":{}}
+        {"author": "noisy author", "tags": ["elasticsearch", "cabana"]}"""
+}
+buildRestTests.setups['stackoverflow'] += """
+"""
+
diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc
index 77b1f61221a..54f3050903c 100644
--- a/docs/community-clients/index.asciidoc
+++ b/docs/community-clients/index.asciidoc
@@ -217,6 +217,9 @@ Also see the {client}/ruby-api/current/index.html[official Elasticsearch Ruby cl
 * https://github.com/ankane/searchkick[Searchkick]:
   Intelligent search made easy
 
+* https://github.com/artsy/estella[Estella]:
+  Make your Ruby models searchable
+
 The following projects appear to be abandoned:
 
 * https://github.com/wireframe/elastic_searchable/[elastic_searchable]:
diff --git a/docs/java-api/docs.asciidoc b/docs/java-api/docs.asciidoc
index 9ace406ee42..c355714bdd6 100644
--- a/docs/java-api/docs.asciidoc
+++ b/docs/java-api/docs.asciidoc
@@ -7,6 +7,7 @@ This section describes the following CRUD APIs:
 * <<java-docs-index>>
 * <<java-docs-get>>
 * <<java-docs-delete>>
+* <<java-docs-delete-by-query>>
 * <<java-docs-update>>
 
 .Multi-document APIs
diff --git a/docs/java-api/docs/delete.asciidoc b/docs/java-api/docs/delete.asciidoc
index b9efd01242d..392132aa129 100644
--- a/docs/java-api/docs/delete.asciidoc
+++ b/docs/java-api/docs/delete.asciidoc
@@ -35,3 +35,48 @@ DeleteResponse response = client.prepareDelete("twitter", "tweet", "1")
         .setOperationThreaded(false)
         .get();
 --------------------------------------------------
+
+[[java-docs-delete-by-query]]
+=== Delete By Query API
+
+The delete by query API allows one to delete a given set of documents based on
+the result of a query:
+
+[source,java]
+--------------------------------------------------
+BulkIndexByScrollResponse response =
+    DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
+        .filter(QueryBuilders.matchQuery("gender", "male")) <1>
+        .source("persons")                                  <2>
+        .get();                                             <3>
+
+long deleted = response.getDeleted();                       <4>
+--------------------------------------------------
+<1> query
+<2> index
+<3> execute the operation
+<4> number of deleted documents
+
+As it can be a long running operation, if you wish to do it asynchronously, you can call `execute` instead of `get`
+and provide a listener like:
+
+[source,java]
+--------------------------------------------------
+DeleteByQueryAction.INSTANCE.newRequestBuilder(client)
+    .filter(QueryBuilders.matchQuery("gender", "male"))                  <1>
+    .source("persons")                                                   <2>
+    .execute(new ActionListener<BulkIndexByScrollResponse>() {           <3>
+        @Override
+        public void onResponse(BulkIndexByScrollResponse response) {
+            long deleted = response.getDeleted();                        <4>
+        }
+        @Override
+        public void onFailure(Exception e) {
+            // Handle the exception
+        }
+    });
+--------------------------------------------------
+<1> query
+<2> index
+<3> listener
+<4> number of deleted documents
diff --git a/docs/java-api/docs/index_.asciidoc b/docs/java-api/docs/index_.asciidoc
index 1e48fbd431c..0b91622dd81 100644
--- a/docs/java-api/docs/index_.asciidoc
+++ b/docs/java-api/docs/index_.asciidoc
@@ -156,8 +156,8 @@ String _type = response.getType();
 String _id = response.getId();
 // Version (if it's the first time you index this document, you will get: 1)
 long _version = response.getVersion();
-// isCreated() is true if the document is a new one, false if it has been updated
-boolean created = response.isCreated();
+// status has stored current instance statement.
+RestStatus status = response.status();
 --------------------------------------------------
 
 For more information on the index operation, check out the REST
diff --git a/docs/java-api/query-dsl/percolate-query.asciidoc b/docs/java-api/query-dsl/percolate-query.asciidoc
index 186d707379e..a5651392b62 100644
--- a/docs/java-api/query-dsl/percolate-query.asciidoc
+++ b/docs/java-api/query-dsl/percolate-query.asciidoc
@@ -1,21 +1,15 @@
 [[java-query-percolate-query]]
-==== Percolate query
+==== Percolate Query
 
-See: {ref}/query-dsl-percolate-query.html[Percolate Query]
+See:
+ * {ref}/query-dsl-percolate-query.html[Percolate Query]
 
-In order to use the `percolate` query from the Java API your
-the percolator module dependency should be on the classpath and
-the transport client should be loaded with the percolator plugin:
 
 [source,java]
 --------------------------------------------------
-TransportClient transportClient = TransportClient.builder()
-        .settings(Settings.builder().put("node.name", "node"))
-        .addPlugin(PercolatorPlugin.class)
-        .build();
-transportClient.addTransportAddress(
-        new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300))
-);
+Settings settings = Settings.builder().put("cluster.name", "elasticsearch").build();
+TransportClient client = new PreBuiltTransportClient(settings);
+client.addTransportAddress(new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300)));
 --------------------------------------------------
 
 Before the `percolate` query can be used an `percolator` mapping should be added and
@@ -26,6 +20,7 @@ a document containing a percolator query should be indexed:
 // create an index with a percolator field with the name 'query':
 client.admin().indices().prepareCreate("myIndexName")
                         .addMapping("query", "query", "type=percolator")
+                        .addMapping("docs", "content", "type=text")
                         .get();
 
 //This is the query we're registering in the percolator
@@ -37,7 +32,7 @@ client.prepareIndex("myIndexName", "query", "myDesignatedQueryName")
         .startObject()
             .field("query", qb) // Register the query
         .endObject())
-    .setRefresh(true) // Needed when the query shall be available immediately
+    .setRefreshPolicy(RefreshPolicy.IMMEDIATE) // Needed when the query shall be available immediately
     .get();
 --------------------------------------------------
 
@@ -51,13 +46,14 @@ code:
 --------------------------------------------------
 //Build a document to check against the percolator
 XContentBuilder docBuilder = XContentFactory.jsonBuilder().startObject();
-docBuilder.field("doc").startObject(); //This is needed to designate the document
 docBuilder.field("content", "This is amazing!");
-docBuilder.endObject(); //End of the doc field
 docBuilder.endObject(); //End of the JSON root object
+
+PercolateQueryBuilder percolateQuery = new PercolateQueryBuilder("query", "docs", docBuilder.bytes());
+
 // Percolate, by executing the percolator query in the query dsl:
 SearchResponse response = client().prepareSearch("myIndexName")
-        .setQuery(QueryBuilders.percolateQuery("query", ""myDocumentType", docBuilder.bytes()))
+        .setQuery(percolateQuery))
         .get();
 //Iterate over the results
 for(SearchHit hit : response.getHits()) {
diff --git a/docs/java-api/search.asciidoc b/docs/java-api/search.asciidoc
index e0029930e6d..b1eb120486a 100644
--- a/docs/java-api/search.asciidoc
+++ b/docs/java-api/search.asciidoc
@@ -3,7 +3,7 @@
 
 The search API allows one to execute a search query and get back search hits
 that match the query. It can be executed across one or more indices and
-across one or more types. The query can provided using the <<java-query-dsl,query Java API>>.
+across one or more types. The query can be provided using the <<java-query-dsl,query Java API>>.
 The body of the search request is built using the `SearchSourceBuilder`. Here is an example:
 
 [source,java]
diff --git a/docs/java-rest/configuration.asciidoc b/docs/java-rest/configuration.asciidoc
index ad5a57dd94e..5fc6d37daa8 100644
--- a/docs/java-rest/configuration.asciidoc
+++ b/docs/java-rest/configuration.asciidoc
@@ -81,6 +81,29 @@ RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
         .build();
 --------------------------------------------------
 
+You can disable Preemptive Authentication, which means that every request will be sent without
+authorization headers to see if it is accepted and, upon receiving a HTTP 401 response, it will
+resend the exact same request with the basic authentication header. If you wish to do this, then
+you can do so by disabling it via the `HttpAsyncClientBuilder`:
+
+[source,java]
+--------------------------------------------------
+final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+credentialsProvider.setCredentials(AuthScope.ANY,
+        new UsernamePasswordCredentials("user", "password"));
+
+RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
+        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
+            @Override
+            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+                // disable preemptive authentication
+                httpClientBuilder.disableAuthCaching();
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+            }
+        })
+        .build();
+--------------------------------------------------
+
 === Encrypted communication
 
 Encrypted communication can also be configured through the
@@ -93,9 +116,9 @@ https://hc.apache.org/httpcomponents-asyncclient-dev/httpasyncclient/apidocs/org
 
 [source,java]
 --------------------------------------------------
-KeyStore keyStore = KeyStore.getInstance("jks");
+KeyStore keystore = KeyStore.getInstance("jks");
 try (InputStream is = Files.newInputStream(keyStorePath)) {
-    keyStore.load(is, keyStorePass.toCharArray());
+    keystore.load(is, keyStorePass.toCharArray());
 }
 RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200))
         .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
diff --git a/docs/java-rest/usage.asciidoc b/docs/java-rest/usage.asciidoc
index b3097ef9d0b..69eb16280ed 100644
--- a/docs/java-rest/usage.asciidoc
+++ b/docs/java-rest/usage.asciidoc
@@ -206,7 +206,14 @@ access to the returned response.
 NOTE: A `ResponseException` is **not** thrown for `HEAD` requests that return
 a `404` status code because it is an expected `HEAD` response that simply
 denotes that the resource is not found. All other HTTP methods (e.g., `GET`)
-throw a `ResponseException` for `404` responses.
+throw a `ResponseException` for `404` responses unless the `ignore` parameter
+contains `404`. `ignore` is a special client parameter that doesn't get sent
+to Elasticsearch and contains a comma separated list of error status codes.
+It allows to control whether some error status code should be treated as an
+expected response rather than as an exception. This is useful for instance
+with the get api as it can return `404` when the document is missing, in which
+case the response body will not contain an error but rather the usual get api
+response, just without the document as it was not found.
 
 
 === Example requests
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 26d4f321899..aaaf7d8e5db 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -120,9 +120,11 @@ The available values are:
 * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
 * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
 * `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-west-2` (`eu-west-2`) for EU (London)
 * `eu-central` (`eu-central-1`) for EU (Frankfurt)
 * `sa-east` (`sa-east-1`) for South America (São Paulo)
 * `cn-north` (`cn-north-1`) for China (Beijing)
+* `ca-central` (`ca-central-1`) for Canada (Central)
 
 [[discovery-ec2-usage-signer]]
 ===== EC2 Signer API
@@ -170,9 +172,20 @@ The following are a list of settings (prefixed with `discovery.ec2`) that can fu
 
 `host_type`::
 
-    The type of host type to use to communicate with other instances. Can be
-    one of `private_ip`, `public_ip`, `private_dns`, `public_dns`. Defaults to
-    `private_ip`.
++
+--
+The type of host type to use to communicate with other instances. Can be
+one of `private_ip`, `public_ip`, `private_dns`, `public_dns` or `tag:TAGNAME` where
+`TAGNAME` refers to a name of a tag configured for all EC2 instances. Instances which don't
+have this tag set will be ignored by the discovery process.
+
+For example if you defined a tag `my-elasticsearch-host` in ec2 and set it to `myhostname1.mydomain.com`, then
+setting `host_type: tag:my-elasticsearch-host` will tell Discovery Ec2 plugin to read the host name from the
+`my-elasticsearch-host` tag. In this case, it will be resolved to `myhostname1.mydomain.com`.
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html[Read more about EC2 Tags].
+
+Defaults to `private_ip`.
+--
 
 `availability_zones`::
 
diff --git a/docs/plugins/ingest-attachment.asciidoc b/docs/plugins/ingest-attachment.asciidoc
index a0a4d70cbcf..cb3296132d9 100644
--- a/docs/plugins/ingest-attachment.asciidoc
+++ b/docs/plugins/ingest-attachment.asciidoc
@@ -52,7 +52,8 @@ The node must be stopped before removing the plugin.
 | `field`                | yes       | -                | The field to get the base64 encoded field from
 | `target_field`         | no        | attachment       | The field that will hold the attachment information
 | `indexed_chars`        | no        | 100000           | The number of chars being used for extraction to prevent huge fields. Use `-1` for no limit.
-| `properties`           | no        | all              | Properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
+| `properties`           | no        | all properties   | Array of properties to select to be stored. Can be `content`, `title`, `name`, `author`, `keywords`, `date`, `content_type`, `content_length`, `language`
+| `ignore_missing`       | no        | `false`          | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 For example, this:
@@ -101,6 +102,139 @@ Returns this:
 --------------------------------------------------
 // TESTRESPONSE
 
+
+To specify only some fields to be extracted:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/attachment
+{
+  "description" : "Extract attachment information",
+  "processors" : [
+    {
+      "attachment" : {
+        "field" : "data",
+        "properties": [ "content", "title" ]
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// CONSOLE
+
 NOTE: Extracting contents from binary data is a resource intensive operation and
       consumes a lot of resources. It is highly recommended to run pipelines
       using this processor in a dedicated ingest node.
+      
+[[ingest-attachment-with-arrays]]
+==== Using the Attachment Processor with arrays
+
+To use the attachment processor within an array of attachments the
+{ref}/foreach-processor.html[foreach processor] is required. This
+enables the attachment processor to be run on the individual elements
+of the array.
+
+For example, given the following source:
+
+[source,js]
+--------------------------------------------------
+{
+  "attachments" : [
+    {
+      "filename" : "ipsum.txt",
+      "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo="
+    },
+    {
+      "filename" : "test.txt",
+      "data" : "VGhpcyBpcyBhIHRlc3QK"
+    }
+  ]
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+In this case, we want to process the data field in each element
+of the attachments field and insert
+the properties into the document so the following `foreach`
+processor is used:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/attachment
+{
+  "description" : "Extract attachment information from arrays",
+  "processors" : [
+    {
+      "foreach": {
+        "field": "attachments",
+        "processor": {
+          "attachment": {
+            "target_field": "_ingest._value.attachment",
+            "field": "_ingest._value.data"
+          }
+        }
+      }
+    }
+  ]
+}
+PUT my_index/my_type/my_id?pipeline=attachment
+{
+  "attachments" : [
+    {
+      "filename" : "ipsum.txt",
+      "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo="
+    },
+    {
+      "filename" : "test.txt",
+      "data" : "VGhpcyBpcyBhIHRlc3QK"
+    }
+  ]
+}
+GET my_index/my_type/my_id
+--------------------------------------------------
+// CONSOLE
+
+Returns this:
+[source,js]
+--------------------------------------------------
+{
+  "_index" : "my_index",
+  "_type" : "my_type",
+  "_id" : "my_id",
+  "_version" : 1,
+  "found" : true,
+  "_source" : {
+    "attachments" : [
+      {
+        "filename" : "ipsum.txt",
+        "data" : "dGhpcyBpcwpqdXN0IHNvbWUgdGV4dAo=",
+        "attachment" : {
+          "content_type" : "text/plain; charset=ISO-8859-1",
+          "language" : "en",
+          "content" : "this is\njust some text",
+          "content_length" : 24
+        }
+      },
+      {
+        "filename" : "test.txt",
+        "data" : "VGhpcyBpcyBhIHRlc3QK",
+        "attachment" : {
+          "content_type" : "text/plain; charset=ISO-8859-1",
+          "language" : "en",
+          "content" : "This is a test",
+          "content_length" : 16
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+
+Note that the `target_field` needs to be set, otherwise the
+default value is used which is a top level field `attachment`. The
+properties on this top level field will contain the value of the
+first attachment only. However, by specifying the
+`target_field` on to a value on `_ingest._value` it will correctly
+associate the properties with the correct attachment.
diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc
index 95e7a0442a4..3665026c01a 100644
--- a/docs/plugins/ingest-geoip.asciidoc
+++ b/docs/plugins/ingest-geoip.asciidoc
@@ -54,6 +54,7 @@ The node must be stopped before removing the plugin.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb.gz and GeoLite2-Country.mmdb.gz files.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
+| `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 *Depends on what is available in `database_field`:
diff --git a/docs/plugins/ingest-user-agent.asciidoc b/docs/plugins/ingest-user-agent.asciidoc
index 250051d17d5..a99e0b3a83a 100644
--- a/docs/plugins/ingest-user-agent.asciidoc
+++ b/docs/plugins/ingest-user-agent.asciidoc
@@ -43,11 +43,12 @@ The node must be stopped before removing the plugin.
 .User-agent options
 [options="header"]
 |======
-| Name                   | Required  | Default                                                                            | Description
-| `field`                | yes       | -                                                                                  | The field containing the user agent string.
-| `target_field`         | no        | user_agent                                                                          | The field that will be filled with the user agent details.
-| `regex_file`           | no        | -                                                                                  | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
+| Name                   | Required  | Default                                                                                         | Description
+| `field`                | yes       | -                                                                                               | The field containing the user agent string.
+| `target_field`         | no        | user_agent                                                                                      | The field that will be filled with the user agent details.
+| `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
+| `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index f060f3493ef..a1f8c6ea81d 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -102,7 +102,8 @@ The Azure repository supports following settings:
 
 `container`::
 
-    Container name. Defaults to `elasticsearch-snapshots`
+    Container name. You must create the azure container before creating the repository.
+    Defaults to `elasticsearch-snapshots`.
 
 `base_path`::
 
diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc
index f494b8089ff..bf773bef091 100644
--- a/docs/plugins/repository-hdfs.asciidoc
+++ b/docs/plugins/repository-hdfs.asciidoc
@@ -95,16 +95,3 @@ The following settings are supported:
 
     Override the chunk size. (Disabled by default)
 
-
-Alternatively, you can define the `hdfs` repository and its settings in your `elasticsearch.yml`:
-[source,yaml]
-----
-repositories:
-  hdfs:
-    uri: "hdfs://<host>:<port>/"    \# required - HDFS address only
-    path: "some/path"               \# required - path within the file-system where data is stored/loaded
-    load_defaults: "true"           \# optional - whether to load the default Hadoop configuration (default) or not
-    conf.<key> : "<value>"          \# optional - 'inlined' key=value added to the Hadoop configuration
-    compress: "false"               \# optional - whether to compress the metadata or not (default)
-    chunk_size: "10mb"              \# optional - chunk size (disabled by default)
-----
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index fcac3105930..d770853e777 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -125,9 +125,11 @@ The available values are:
 * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
 * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
 * `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-west-2` (`eu-west-2`) for EU (London)
 * `eu-central` (`eu-central-1`) for EU (Frankfurt)
 * `sa-east` (`sa-east-1`) for South America (São Paulo)
 * `cn-north` (`cn-north-1`) for China (Beijing)
+* `ca-central` (`ca-central-1`) for Canada (Central)
 
 [[repository-s3-usage-signer]]
 ===== S3 Signer API
diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc
index 2b6523725b6..586e582a05f 100644
--- a/docs/reference/aggregations.asciidoc
+++ b/docs/reference/aggregations.asciidoc
@@ -92,11 +92,15 @@ enables defining all the "dynamic" expressions in the script as parameters, whic
 between calls (this will ensure the use of the cached compiled scripts in Elasticsearch).
 ===============================
 
-Scripts can generate a single value or multiple values per document. When generating multiple values, one can use the
-`script_values_sorted` settings to indicate whether these values are sorted or not. Internally, Elasticsearch can
-perform optimizations when dealing with sorted values (for example, with the `min` aggregations, knowing the values are
-sorted, Elasticsearch will skip the iterations over all the values and rely on the first value in the list to be the
-minimum value among all other values associated with the same document).
+Elasticsearch uses the type of the field in the mapping in order to figure out
+how to run the aggregation and format the response. However there are two cases
+in which Elasticsearch cannot figure out this information: unmapped fields (for
+instance in the case of a search request across multiple indices, and only some
+of them have a mapping for the field) and pure scripts. For those cases, it is
+possible to give Elasticsearch a hint using the `value_type` option, which
+accepts the following values: `string`, `long` (works for all integer types),
+`double` (works for all decimal types like `float` or `scaled_float`), `date`,
+`ip` and `boolean`.
 
 --
 
diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc
index 851511257cf..f98fd52feba 100644
--- a/docs/reference/aggregations/bucket.asciidoc
+++ b/docs/reference/aggregations/bucket.asciidoc
@@ -13,6 +13,8 @@ aggregated for the buckets created by their "parent" bucket aggregation.
 There are different bucket aggregators, each with a different "bucketing" strategy. Some define a single bucket, some
 define fixed number of multiple buckets, and others dynamically create the buckets during the aggregation process.
 
+include::bucket/adjacency-matrix-aggregation.asciidoc[]
+
 include::bucket/children-aggregation.asciidoc[]
 
 include::bucket/datehistogram-aggregation.asciidoc[]
diff --git a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
new file mode 100644
index 00000000000..8e4102683f1
--- /dev/null
+++ b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc
@@ -0,0 +1,116 @@
+[[search-aggregations-bucket-adjacency-matrix-aggregation]]
+=== Adjacency Matrix Aggregation
+
+A bucket aggregation returning a form of https://en.wikipedia.org/wiki/Adjacency_matrix[adjacency matrix].
+The request provides a collection of named filter expressions, similar to the `filters` aggregation
+request. 
+Each bucket in the response represents a non-empty cell in the matrix of intersecting filters.
+
+experimental[The `adjacency_matrix` aggregation is a new feature and we may evolve its design as we get feedback on its use.  As a result, the API for this feature may change in non-backwards compatible ways]
+
+
+Given filters named `A`, `B` and `C` the response would return buckets with the following names:
+
+
+[options="header"]
+|=======================
+|  h|A   h|B  h|C   
+h|A |A   |A&B |A&C 
+h|B |    |B   |B&C 
+h|C |    |    |C  
+|=======================
+
+The intersecting buckets e.g `A&C` are labelled using a combination of the two filter names separated by
+the ampersand character. Note that the response does not also include a "C&A" bucket as this would be the
+same set of documents as "A&C". The matrix is said to be _symmetric_ so we only return half of it. To do this we sort 
+the filter name strings and always use the lowest of a pair as the value to the left of the "&" separator. 
+
+An alternative `separator` parameter can be passed in the request if clients wish to use a separator string 
+other than the default of the ampersand.
+
+
+Example:
+
+[source,js]
+--------------------------------------------------
+PUT /emails/message/_bulk?refresh
+{ "index" : { "_id" : 1 } }
+{ "accounts" : ["hillary", "sidney"]}
+{ "index" : { "_id" : 2 } }
+{ "accounts" : ["hillary", "donald"]}
+{ "index" : { "_id" : 3 } }
+{ "accounts" : ["vladimir", "donald"]}
+
+GET emails/message/_search
+{
+  "size": 0,
+  "aggs" : {
+    "interactions" : {
+      "adjacency_matrix" : {
+        "filters" : {
+          "grpA" : { "terms" : { "accounts" : ["hillary", "sidney"] }},
+          "grpB" : { "terms" : { "accounts" : ["donald", "mitt"] }},
+          "grpC" : { "terms" : { "accounts" : ["vladimir", "nigel"] }}
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+In the above example, we analyse email messages to see which groups of individuals 
+have exchanged messages.
+We will get counts for each group individually and also a count of messages for pairs
+of groups that have recorded interactions.
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "took": 9,
+  "timed_out": false,
+  "_shards": ...,
+  "hits": ...,
+  "aggregations": {
+    "interactions": {
+      "buckets": [
+        {
+          "key":"grpA",
+          "doc_count": 2
+        },
+        {
+          "key":"grpA&grpB",
+          "doc_count": 1
+        },
+        {
+          "key":"grpB",
+          "doc_count": 2
+        },
+        {
+          "key":"grpB&grpC",
+          "doc_count": 1
+        },
+        {
+          "key":"grpC",
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took": 9/"took": $body.took/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/]
+// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
+
+==== Usage
+On its own this aggregation can provide all of the data required to create an undirected weighted graph.
+However, when used with child aggregations such as a `date_histogram` the results can provide the
+additional levels of data required to perform https://en.wikipedia.org/wiki/Dynamic_network_analysis[dynamic network analysis]
+where examining interactions _over time_ becomes important.
+
+==== Limitations
+For N filters the matrix of buckets produced can be N²/2 and so there is a default maximum 
+imposed of 100 filters . This setting can be changed using the `index.max_adjacency_matrix_filters` index-level setting.
diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index adf6e572d4c..d2ae9052f35 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -12,9 +12,10 @@ Requesting bucket intervals of a month.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "month"
@@ -23,6 +24,8 @@ Requesting bucket intervals of a month.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Available expressions for interval: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`
 
@@ -32,9 +35,10 @@ time unit (e.g., `1.5h` could instead be specified as `90m`).
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "90m"
@@ -43,6 +47,8 @@ time unit (e.g., `1.5h` could instead be specified as `90m`).
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Keys
 
@@ -56,9 +62,10 @@ TIP: If no `format` is specified, then it will use the first date
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "articles_over_time" : {
+        "sales_over_time" : {
             "date_histogram" : {
                 "field" : "date",
                 "interval" : "1M",
@@ -68,6 +75,8 @@ TIP: If no `format` is specified, then it will use the first date
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> Supports expressive date <<date-format-pattern,format pattern>>
 
@@ -76,25 +85,31 @@ Response:
 [source,js]
 --------------------------------------------------
 {
+    ...
     "aggregations": {
-        "articles_over_time": {
+        "sales_over_time": {
             "buckets": [
                 {
-                    "key_as_string": "2013-02-02",
-                    "key": 1328140800000,
-                    "doc_count": 1
+                    "key_as_string": "2015-01-01",
+                    "key": 1420070400000,
+                    "doc_count": 3
                 },
                 {
-                    "key_as_string": "2013-03-02",
-                    "key": 1330646400000,
+                    "key_as_string": "2015-02-01",
+                    "key": 1422748800000,
                     "doc_count": 2
                 },
-                ...
+                {
+                    "key_as_string": "2015-03-01",
+                    "key": 1425168000000,
+                    "doc_count": 2
+                }
             ]
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 ==== Time Zone
 
@@ -110,12 +125,12 @@ Consider the following example:
 
 [source,js]
 ---------------------------------
-PUT my_index/log/1
+PUT my_index/log/1?refresh
 {
   "date": "2015-10-01T00:30:00Z"
 }
 
-PUT my_index/log/2
+PUT my_index/log/2?refresh
 {
   "date": "2015-10-01T01:30:00Z"
 }
@@ -132,6 +147,7 @@ GET my_index/_search?size=0
   }
 }
 ---------------------------------
+// CONSOLE
 
 UTC is used if no time zone is specified, which would result in both of these
 documents being placed into the same day bucket, which starts at midnight UTC
@@ -139,18 +155,22 @@ on 1 October 2015:
 
 [source,js]
 ---------------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-10-01T00:00:00.000Z",
-        "key":           1443657600000,
-        "doc_count":     2
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-10-01T00:00:00.000Z",
+          "key":           1443657600000,
+          "doc_count":     2
+        }
+      ]
+    }
   }
 }
 ---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 If a `time_zone` of `-01:00` is specified, then midnight starts at one hour before
 midnight UTC:
@@ -170,29 +190,36 @@ GET my_index/_search?size=0
   }
 }
 ---------------------------------
+// CONSOLE
+// TEST[continued]
 
 Now the first document falls into the bucket for 30 September 2015, while the
 second document falls into the bucket for 1 October 2015:
 
 [source,js]
 ---------------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-09-30T00:00:00.000-01:00", <1>
-        "key": 1443571200000,
-        "doc_count": 1
-      },
-      {
-        "key_as_string": "2015-10-01T00:00:00.000-01:00", <1>
-        "key": 1443657600000,
-        "doc_count": 1
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-09-30T00:00:00.000-01:00", <1>
+          "key": 1443574800000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T00:00:00.000-01:00", <1>
+          "key": 1443661200000,
+          "doc_count": 1
+        }
+      ]
+    }
   }
 }
 ---------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+
 <1> The `key_as_string` value represents midnight on each day
     in the specified time zone.
 
@@ -211,7 +238,7 @@ happens.
 
 The `offset` parameter is used to change the start value of each bucket by the
 specified positive (`+`) or negative offset (`-`) duration, such as `1h` for
-an hour, or `1M` for a month. See <<time-units>> for more possible time
+an hour, or `1d` for a day. See <<time-units>> for more possible time
 duration options.
 
 For instance, when using an interval of `day`, each bucket runs from midnight
@@ -220,12 +247,12 @@ to run from 6am to 6am:
 
 [source,js]
 -----------------------------
-PUT my_index/log/1
+PUT my_index/log/1?refresh
 {
   "date": "2015-10-01T05:30:00Z"
 }
 
-PUT my_index/log/2
+PUT my_index/log/2?refresh
 {
   "date": "2015-10-01T06:30:00Z"
 }
@@ -243,29 +270,34 @@ GET my_index/_search?size=0
   }
 }
 -----------------------------
+// CONSOLE
 
 Instead of a single bucket starting at midnight, the above request groups the
 documents into buckets starting at 6am:
 
 [source,js]
 -----------------------------
-"aggregations": {
-  "by_day": {
-    "buckets": [
-      {
-        "key_as_string": "2015-09-30T06:00:00.000Z",
-        "key": 1443592800000,
-        "doc_count": 1
-      },
-      {
-        "key_as_string": "2015-10-01T06:00:00.000Z",
-        "key": 1443679200000,
-        "doc_count": 1
-      }
-    ]
+{
+  ...
+  "aggregations": {
+    "by_day": {
+      "buckets": [
+        {
+          "key_as_string": "2015-09-30T06:00:00.000Z",
+          "key": 1443592800000,
+          "doc_count": 1
+        },
+        {
+          "key_as_string": "2015-10-01T06:00:00.000Z",
+          "key": 1443679200000,
+          "doc_count": 1
+        }
+      ]
+    }
   }
 }
 -----------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 NOTE: The start `offset` of each bucket is calculated after the `time_zone`
 adjustments have been made.
@@ -287,17 +319,20 @@ had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "publish_date" : {
+        "sale_date" : {
              "date_histogram" : {
-                 "field" : "publish_date",
+                 "field" : "date",
                  "interval": "year",
-                 "missing": "2000-01-01" <1>
+                 "missing": "2000/01/01" <1>
              }
          }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> Documents without a value in the `publish_date` field will fall into the same bucket as documents that have the value `2000-01-01`.
diff --git a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
index a69b2b3cb11..df32075583f 100644
--- a/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/daterange-aggregation.asciidoc
@@ -8,6 +8,7 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs": {
         "range": {
@@ -23,6 +24,9 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales s/now-10M\/M/10-2015/]
+
 <1> < now minus 10 months, rounded down to the start of the month.
 <2> >= now minus 10 months, rounded down to the start of the month.
 
@@ -35,25 +39,27 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "range": {
             "buckets": [
                 {
-                    "to": 1.3437792E+12,
-                    "to_as_string": "08-2012",
-                    "doc_count": 7
+                    "to": 1.4436576E12,
+                    "to_as_string": "10-2015",
+                    "doc_count": 7,
+                    "key": "*-10-2015"
                 },
                 {
-                    "from": 1.3437792E+12,
-                    "from_as_string": "08-2012",
-                    "doc_count": 2
+                    "from": 1.4436576E12,
+                    "from_as_string": "10-2015",
+                    "doc_count": 0,
+                    "key": "10-2015-*"
                 }
             ]
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 [[date-format-pattern]]
 ==== Date Format/Pattern
@@ -125,21 +131,25 @@ to round to the beginning of the day in the CET time zone, you can do the follow
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
    "aggs": {
-           "range": {
-               "date_range": {
-                   "field": "date",
-                   "time_zone": "CET",
-                   "ranges": [
-                      { "to": "2016-02-15/d" }, <1>
-                      { "from": "2016-02-15/d", "to" : "now/d" <2>},
-                      { "from": "now/d" },
-                  ]
-              }
+       "range": {
+           "date_range": {
+               "field": "date",
+               "time_zone": "CET",
+               "ranges": [
+                  { "to": "2016/02/01" }, <1>
+                  { "from": "2016/02/01", "to" : "now/d" <2>},
+                  { "from": "now/d" }
+              ]
           }
       }
-  }
+   }
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
+
 <1> This date will be converted to `2016-02-15T00:00:00.000+01:00`.
 <2> `now/d` will be rounded to the beginning of the day in the CET time zone.
diff --git a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
index f5a45da976b..acb40c5a65d 100644
--- a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
@@ -3,35 +3,51 @@
 
 experimental[]
 
-A filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents. Diversity settings are 
-used to limit the number of matches that share a common value such as an "author".
+Like the `sampler` aggregation this is a filtering aggregation used to limit any sub aggregations' processing to a sample of the top-scoring documents.
+The `diversified_sampler` aggregation adds the ability to limit the number of matches that share a common value such as an "author".
+
+NOTE: Any good market researcher will tell you that when working with samples of data it is important
+that the sample represents a healthy variety of opinions rather than being skewed by any single voice.
+The same is true with aggregations and sampling with these diversify settings can offer a way to remove the bias in your content (an over-populated geography, 
+a large spike in a timeline or an over-active forum spammer).  
+
 
 .Example use cases:
 * Tightening the focus of analytics to high-relevance matches rather than the potentially very long tail of low-quality matches
 * Removing bias from analytics by ensuring fair representation of content from different sources
 * Reducing the running cost of aggregations that can produce useful results using only samples e.g. `significant_terms`
  
+A choice of `field` or `script` setting is used to provide values used for de-duplication and the `max_docs_per_value` setting controls the maximum 
+number of documents collected on any one shard which share a common value. The default setting for `max_docs_per_value` is 1.
+
+The aggregation will throw an error if the choice of `field` or `script` produces multiple values for a single document (de-duplication using multi-valued fields is not supported due to efficiency concerns).
+
 
 Example:
 
+We might want to see which tags are strongly associated with `#elasticsearch` on StackOverflow
+forum posts but ignoring the effects of some prolific users with a tendency to misspell #Kibana as #Cabana.
+
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
     "query": {
-        "match": {
-            "text": "iphone"
+        "query_string": {
+            "query": "tags:elasticsearch"
         }
     },
     "aggs": {
-        "sample": {
+        "my_unbiased_sample": {
             "diversified_sampler": {
                 "shard_size": 200,
-                "field" : "user.id"   
+                "field" : "author"   
             },
             "aggs": {
                 "keywords": {
                     "significant_terms": {
-                        "field": "text"
+                        "field": "tags",
+                        "exclude": ["elasticsearch"]
                     }
                 }
             }
@@ -39,6 +55,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
 
 Response:
 
@@ -46,92 +64,118 @@ Response:
 --------------------------------------------------
 {
     ...
-        "aggregations": {
-        "sample": {
+    "aggregations": {
+        "my_unbiased_sample": {
             "doc_count": 1000,<1>
             "keywords": {<2>
                 "doc_count": 1000,
                 "buckets": [
-                    ...
                     {
-                        "key": "bend",
-                        "doc_count": 58,
-                        "score": 37.982536582524276,
-                        "bg_count": 103
-                    },
-                    ....
-}
---------------------------------------------------
-
-<1> 1000 documents were sampled in total because we asked for a maximum of 200 from an index with 5 shards. The cost of performing the nested significant_terms aggregation was therefore limited rather than unbounded.
-<2> The results of the significant_terms aggregation are not skewed by any single over-active Twitter user because we asked for a maximum of one tweet from any one user in our sample.
-
-
-==== shard_size
-
-The `shard_size` parameter limits how many top-scoring documents are collected in the sample processed on each shard.
-The default value is 100.
-
-==== Controlling diversity
-=`field` or `script` and `max_docs_per_value` settings are used to control the maximum number of documents collected on any one shard which share a common value.
-The choice of value (e.g. `author`) is loaded from a regular `field` or derived dynamically by a `script`.
-
-The aggregation will throw an error if the choice of field or script produces multiple values for a document.
-It is currently not possible to offer this form of de-duplication using many values, primarily due to concerns over efficiency.
-
-NOTE: Any good market researcher will tell you that when working with samples of data it is important
-that the sample represents a healthy variety of opinions rather than being skewed by any single voice.
-The same is true with aggregations and sampling with these diversify settings can offer a way to remove the bias in your content (an over-populated geography, a large spike in a timeline or an over-active forum spammer).  
-
-==== Field
-
-Controlling diversity using a field:
-
-[source,js]
---------------------------------------------------
-{
-    "aggs" : {
-        "sample" : {
-            "diversified_sampler" : {
-                "field" : "author",
-                "max_docs_per_value" : 3
+                        "key": "kibana",
+                        "doc_count": 150,
+                        "score": 2.213,
+                        "bg_count": 200
+                    }
+                ]
             }
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/1000/151/]
+// TESTRESPONSE[s/2.213/$body.aggregations.my_unbiased_sample.keywords.buckets.0.score/]
 
-Note that the `max_docs_per_value` setting applies on a per-shard basis only for the purposes of shard-local sampling.
-It is not intended as a way of providing a global de-duplication feature on search results.
+<1> 1000 documents were sampled in total because we asked for a maximum of 200 from an index with 5 shards. The cost of performing the nested significant_terms aggregation was therefore limited rather than unbounded.
+<2> The results of the significant_terms aggregation are not skewed by any single author's quirks because we asked for a maximum of one post from any one author in our sample.
 
+==== Scripted example:
 
-
-==== Script
-
-Controlling diversity using a script:
+In this scenario we might want to diversify on a combination of field values. We can use a `script` to produce a hash of the 
+multiple values in a tags field to ensure we don't have a sample that consists of the same repeated combinations of tags.
 
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
-    "aggs" : {
-        "sample" : {
-            "diversified_sampler" : {
-                "script" : { 
-                    "lang" : "painless",
-                    "inline" : "doc['author'].value + '/' + doc['genre'].value"
+    "query": {
+        "query_string": {
+            "query": "tags:kibana"
+        }
+    },
+    "aggs": {
+        "my_unbiased_sample": {
+            "diversified_sampler": {
+                "shard_size": 200,
+                "max_docs_per_value" : 3,
+                "script" : {
+                    "lang": "painless",
+                    "inline": "doc['tags'].values.hashCode()"
+                }   
+            },
+            "aggs": {
+                "keywords": {
+                    "significant_terms": {
+                        "field": "tags",
+                        "exclude": ["kibana"]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
-Note in the above example we chose to use the default `max_docs_per_value` setting of 1 and combine author and genre fields to ensure 
-each shard sample has, at most, one match for an author/genre pair.
+// CONSOLE
+// TEST[setup:stackoverflow]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "my_unbiased_sample": {
+            "doc_count": 1000,
+            "keywords": {
+                "doc_count": 1000,
+                "buckets": [
+                    {
+                        "key": "logstash",
+                        "doc_count": 3,
+                        "score": 2.213,
+                        "bg_count": 50
+                    },
+                    {
+                        "key": "elasticsearch",
+                        "doc_count": 3,
+                        "score": 1.34,
+                        "bg_count": 200
+                    },
+                ]
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/1000/6/]
+// TESTRESPONSE[s/2.213/$body.aggregations.my_unbiased_sample.keywords.buckets.0.score/]
+// TESTRESPONSE[s/1.34/$body.aggregations.my_unbiased_sample.keywords.buckets.1.score/]
+
+==== shard_size
+
+The `shard_size` parameter limits how many top-scoring documents are collected in the sample processed on each shard.
+The default value is 100.
+
+==== max_docs_per_value
+The `max_docs_per_value` is an optional parameter and limits how many documents are permitted per choice of de-duplicating value.
+The default setting is "1".
 
 
 ==== execution_hint
 
-When using the settings to control diversity, the optional `execution_hint` setting can influence the management of the values used for de-duplication.
+The optional `execution_hint` setting can influence the management of the values used for de-duplication.
 Each option will hold up to `shard_size` values in memory while performing de-duplication but the type of value held can be controlled as follows:
  
  - hold field values directly (`map`)
@@ -145,12 +189,12 @@ Please note that Elasticsearch will ignore the choice of execution hint if it is
 ==== Limitations
 
 ===== Cannot be nested under `breadth_first` aggregations
-Being a quality-based filter the sampler aggregation needs access to the relevance score produced for each document.
+Being a quality-based filter the diversified_sampler aggregation needs access to the relevance score produced for each document.
 It therefore cannot be nested under a `terms` aggregation which has the `collect_mode` switched from the default `depth_first` mode to `breadth_first` as this discards scores.
 In this situation an error will be thrown.
 
 ===== Limited de-dup logic.
-The de-duplication logic in the diversify settings applies only at a shard level so will not apply across shards.
+The de-duplication logic applies only at a shard level so will not apply across shards.
 
 ===== No specialized syntax for geo/date fields
 Currently the syntax for defining the diversifying values is defined by a choice of `field` or
diff --git a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
index be5c6ac58f2..3f8ff41304a 100644
--- a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc
@@ -7,10 +7,11 @@ Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "red_products" : {
-            "filter" : { "term": { "color": "red" } },
+        "t_shirts" : {
+            "filter" : { "term": { "type": "t-shirt" } },
             "aggs" : {
                 "avg_price" : { "avg" : { "field" : "price" } }
             }
@@ -18,6 +19,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 In the above example, we calculate the average price of all the products that are red.
 
@@ -27,12 +30,12 @@ Response:
 --------------------------------------------------
 {
     ...
-
-    "aggs" : {
-        "red_products" : {
-            "doc_count" : 100,
-            "avg_price" : { "value" : 56.3 }
+    "aggregations" : {
+        "t_shirts" : {
+            "doc_count" : 3,
+            "avg_price" : { "value" : 128.33333333333334 }
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
diff --git a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
index 4890f9dca9d..a051fc00c4f 100644
--- a/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geodistance-aggregation.asciidoc
@@ -83,7 +83,7 @@ By default, the distance unit is `m` (metres) but it can also accept: `mi` (mile
 
 <1> The distances will be computed as miles
 
-There are three distance calculation modes: `sloppy_arc` (the default), `arc` (most accurate) and `plane` (fastest). The `arc` calculation is the most accurate one but also the more expensive one in terms of performance. The `sloppy_arc` is faster but less accurate. The `plane` is the fastest but least accurate distance function. Consider using `plane` when your search context is "narrow" and spans smaller geographical areas (like cities or even countries). `plane` may return higher error mergins for searches across very large areas (e.g. cross continent search). The distance calculation type can be set using the `distance_type` parameter:
+There are two distance calculation modes: `arc` (the default), and `plane`. The `arc` calculation is the most accurate. The `plane` is the fastest but least accurate. Consider using `plane` when your search context is "narrow", and spans smaller geographical areas (~5km). `plane` will return higher error margins for searches across very large areas (e.g. cross continent search). The distance calculation type can be set using the `distance_type` parameter:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/bucket/global-aggregation.asciidoc b/docs/reference/aggregations/bucket/global-aggregation.asciidoc
index 4e5addb46c1..affabc5067e 100644
--- a/docs/reference/aggregations/bucket/global-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/global-aggregation.asciidoc
@@ -1,18 +1,22 @@
 [[search-aggregations-bucket-global-aggregation]]
 === Global Aggregation
 
-Defines a single bucket of all the documents within the search execution context. This context is defined by the indices and the document types you're searching on, but is *not* influenced by the search query itself.
+Defines a single bucket of all the documents within the search execution
+context. This context is defined by the indices and the document types you're
+searching on, but is *not* influenced by the search query itself.
 
-NOTE:   Global aggregators can only be placed as top level aggregators (it makes no sense to embed a global aggregator
-        within another bucket aggregator)
+NOTE:   Global aggregators can only be placed as top level aggregators because
+        it doesn't make sense to embed a global aggregator within another
+        bucket aggregator.
 
 Example:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "query" : {
-        "match" : { "title" : "shirt" }
+        "match" : { "type" : "t-shirt" }
     },
     "aggs" : {
         "all_products" : {
@@ -20,15 +24,21 @@ Example:
             "aggs" : { <2>
                 "avg_price" : { "avg" : { "field" : "price" } }
             }
-        }
+        },
+        "t_shirts": { "avg" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 <1> The `global` aggregation has an empty body
 <2> The sub-aggregations that are registered for this `global` aggregation
 
-The above aggregation demonstrates how one would compute aggregations (`avg_price` in this example) on all the documents in the search context, regardless of the query (in our example, it will compute the average price over all products in our catalog, not just on the "shirts").
+The above aggregation demonstrates how one would compute aggregations
+(`avg_price` in this example) on all the documents in the search context,
+regardless of the query (in our example, it will compute the average price over
+all products in our catalog, not just on the "shirts").
 
 The response for the above aggregation:
 
@@ -36,16 +46,22 @@ The response for the above aggregation:
 --------------------------------------------------
 {
     ...
-
     "aggregations" : {
         "all_products" : {
-            "doc_count" : 100, <1>
+            "doc_count" : 7, <1>
             "avg_price" : {
-                "value" : 56.3
+                "value" : 140.71428571428572 <2>
             }
+        },
+        "t_shirts": {
+            "value" : 128.33333333333334 <3>
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-<1> The number of documents that were aggregated (in our case, all documents within the search context)
+<1> The number of documents that were aggregated (in our case, all documents
+within the search context)
+<2> The average price of all products in the index
+<3> The average price of all t-shirts
diff --git a/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
index e1f3e708aa6..693df5e8144 100644
--- a/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/sampler-aggregation.asciidoc
@@ -12,12 +12,18 @@ A filtering aggregation used to limit any sub aggregations' processing to a samp
 
 Example:
 
+A query on StackOverflow data for the popular term `javascript` OR the rarer term
+`kibana` will match many documents - most of them missing the word Kibana. To focus
+the `significant_terms` aggregation on top-scoring documents that are more likely to match 
+the most interesting parts of our query we use a sample.
+
 [source,js]
 --------------------------------------------------
+POST /stackoverflow/_search?size=0
 {
     "query": {
-        "match": {
-            "text": "iphone"
+        "query_string": {
+            "query": "tags:kibana OR tags:javascript"
         }
     },
     "aggs": {
@@ -28,7 +34,8 @@ Example:
             "aggs": {
                 "keywords": {
                     "significant_terms": {
-                        "field": "text"
+                        "field": "tags",
+                        "exclude": ["kibana", "javascript"]
                     }
                 }
             }
@@ -36,6 +43,8 @@ Example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
 
 Response:
 
@@ -43,26 +52,103 @@ Response:
 --------------------------------------------------
 {
     ...
-        "aggregations": {
+    "aggregations": {
         "sample": {
             "doc_count": 1000,<1>
             "keywords": {
                 "doc_count": 1000,
                 "buckets": [
-                    ...
                     {
-                        "key": "bend",
-                        "doc_count": 58,
-                        "score": 37.982536582524276,
-                        "bg_count": 103
+                        "key": "elasticsearch",
+                        "doc_count": 150,
+                        "score": 1.078125,
+                        "bg_count": 200
                     },
-                    ....
+                    {
+                        "key": "logstash",
+                        "doc_count": 50,
+                        "score": 0.5625,
+                        "bg_count": 50
+                    }
+                ]
+            }
+        }
+    }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/1000/200/]
 
 <1> 1000 documents were sampled in total because we asked for a maximum of 200 from an index with 5 shards. The cost of performing the nested significant_terms aggregation was therefore limited rather than unbounded.
 
 
+Without the `sampler` aggregation the request query considers the full "long tail" of low-quality matches and therefore identifies
+less significant terms such as `jquery` and `angular` rather than focusing on the more insightful Kibana-related terms.
+
+
+[source,js]
+--------------------------------------------------
+POST /stackoverflow/_search?size=0
+{
+    "query": {
+        "query_string": {
+            "query": "tags:kibana OR tags:javascript"
+        }
+    },
+    "aggs": {
+             "low_quality_keywords": {
+                "significant_terms": {
+                    "field": "tags",
+                    "size": 3,
+                    "exclude":["kibana", "javascript"]
+                }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:stackoverflow]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "low_quality_keywords": {
+            "doc_count": 1000,
+            "buckets": [
+                {
+                    "key": "angular",
+                    "doc_count": 200,
+                    "score": 0.02777,
+                   "bg_count": 200
+                },
+                {
+                    "key": "jquery",
+                    "doc_count": 200,
+                    "score": 0.02777,
+                    "bg_count": 200
+                },
+                {
+                    "key": "logstash",
+                    "doc_count": 50,
+                    "score": 0.0069,
+                    "bg_count": 50
+                }
+            ]
+        }
+    }
+}
+--------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/1000/600/]
+// TESTRESPONSE[s/0.02777/$body.aggregations.low_quality_keywords.buckets.0.score/]
+// TESTRESPONSE[s/0.0069/$body.aggregations.low_quality_keywords.buckets.2.score/]
+
+
+
 ==== shard_size
 
 The `shard_size` parameter limits how many top-scoring documents are collected in the sample processed on each shard.
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 684f4aab81e..0b028c1a940 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -794,3 +794,10 @@ had a value.
 --------------------------------------------------
 
 <1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`.
+
+==== Mixing field types
+
+WARNING: When aggregating on multiple indices the type of the aggregated field may not be the same in all indices.
+Some types are compatible with each other (`integer` and `long` or `float` and `double`) but when the types are a mix
+of decimal and non-decimal number the terms aggregation will promote the non-decimal numbers to decimal numbers.
+This can result in a loss of precision in the bucket values.
diff --git a/docs/reference/aggregations/metrics/max-aggregation.asciidoc b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
index 5527c08dd27..34945bfc969 100644
--- a/docs/reference/aggregations/metrics/max-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
@@ -1,7 +1,10 @@
 [[search-aggregations-metrics-max-aggregation]]
 === Max Aggregation
 
-A `single-value` metrics aggregation that keeps track and returns the maximum value among the numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `single-value` metrics aggregation that keeps track and returns the maximum
+value among the numeric values extracted from the aggregated documents. These
+values can be extracted either from specific numeric fields in the documents,
+or be generated by a provided script.
 
 NOTE: The `min` and `max` aggregation operate on the `double` representation of
 the data. As a consequence, the result may be approximate when running on longs
@@ -11,12 +14,15 @@ Computing the max price value across all documents
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "max_price" : { "max" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -24,45 +30,52 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
         "max_price": {
-            "value": 35
+            "value": 200.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-As can be seen, the name of the aggregation (`max_price` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
+As can be seen, the name of the aggregation (`max_price` above) also serves as
+the key by which the aggregation result can be retrieved from the returned
+response.
 
 ==== Script
 
-Computing the max price value across all document, this time using a script:
+The `max` aggregation can also calculate the maximum of a script. The example
+below computes the maximum price:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "max_price" : { 
-            "max" : { 
+        "max_price" : {
+            "max" : {
                 "script" : {
-                    "inline" : "doc['price'].value",
-                    "lang" : "painless"
+                    "inline" : "doc.price.value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
+This will use the <<modules-scripting-painless, Painless>> scripting language
+and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "max_price" : { 
-            "max" : { 
+        "max_price" : {
+            "max" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
@@ -74,23 +87,28 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
+TIP: For indexed scripts replace the `file` parameter with an `id` parameter.
 
 ==== Value Script
 
-Let's say that the prices of the documents in our index are in USD, but we would like to compute the max in EURO (and for the sake of this example, lets say the conversion rate is 1.2). We can use a value script to apply the conversion rate to every value before it is aggregated:
+Let's say that the prices of the documents in our index are in USD, but we
+would like to compute the max in EURO (and for the sake of this example, let's
+say the conversion rate is 1.2). We can use a value script to apply the
+conversion rate to every value before it is aggregated:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "max_price_in_euros" : {
             "max" : {
                 "field" : "price",
                 "script" : {
-                    "lang": "painless",
-                    "inline": "_value * params.conversion_rate",
+                    "inline" : "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
@@ -100,15 +118,18 @@ Let's say that the prices of the documents in our index are in USD, but we would
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default they will be ignored but it is also possible to treat
+them as if they had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "grade_max" : {
@@ -120,5 +141,8 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-<1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`.
+<1> Documents without a value in the `grade` field will fall into the same
+bucket as documents that have the value `10`.
diff --git a/docs/reference/aggregations/metrics/min-aggregation.asciidoc b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
index 8cdd6f9e7db..813a2aff9db 100644
--- a/docs/reference/aggregations/metrics/min-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
@@ -1,7 +1,10 @@
 [[search-aggregations-metrics-min-aggregation]]
 === Min Aggregation
 
-A `single-value` metrics aggregation that keeps track and returns the minimum value among numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script.
+A `single-value` metrics aggregation that keeps track and returns the minimum
+value among numeric values extracted from the aggregated documents. These
+values can be extracted either from specific numeric fields in the documents,
+or be generated by a provided script.
 
 NOTE: The `min` and `max` aggregation operate on the `double` representation of
 the data. As a consequence, the result may be approximate when running on longs
@@ -11,12 +14,15 @@ Computing the min price value across all documents:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
         "min_price" : { "min" : { "field" : "price" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -27,42 +33,50 @@ Response:
 
     "aggregations": {
         "min_price": {
-            "value": 10
+            "value": 10.0
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
-As can be seen, the name of the aggregation (`min_price` above) also serves as the key by which the aggregation result can be retrieved from the returned response.
+As can be seen, the name of the aggregation (`min_price` above) also serves as
+the key by which the aggregation result can be retrieved from the returned
+response.
 
 ==== Script
 
-Computing the min price value across all document, this time using a script:
+The `min` aggregation can also calculate the maximum of a script. The example
+below computes the minimum price:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "min_price" : { 
-            "min" : { 
+        "min_price" : {
+            "min" : {
                 "script" : {
-                    "inline" : "doc['price'].value",
-                    "lang" : "painless"
+                    "inline" : "doc.price.value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
+This will use the <<modules-scripting-painless, Painless>> scripting language
+and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
-        "min_price" : { 
-            "min" : { 
+        "min_price" : {
+            "min" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
@@ -74,23 +88,28 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
+TIP: For indexed scripts replace the `file` parameter with an `id` parameter.
 
 ==== Value Script
 
-Let's say that the prices of the documents in our index are in USD, but we would like to compute the min in EURO (and for the sake of this example, lets say the conversion rate is 1.2). We can use a value script to apply the conversion rate to every value before it is aggregated:
+Let's say that the prices of the documents in our index are in USD, but we
+would like to compute the min in EURO (and for the sake of this example, let's
+say the conversion rate is 1.2). We can use a value script to apply the
+conversion rate to every value before it is aggregated:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "min_price_in_euros" : {
             "min" : {
                 "field" : "price",
-                "script" : 
-                    "lang" : "painless",
-                    "inline": "_value * params.conversion_rate",
+                "script" : {
+                    "inline" : "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
@@ -100,15 +119,18 @@ Let's say that the prices of the documents in our index are in USD, but we would
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 ==== Missing value
 
-The `missing` parameter defines how documents that are missing a value should be treated.
-By default they will be ignored but it is also possible to treat them as if they
-had a value.
+The `missing` parameter defines how documents that are missing a value should
+be treated. By default they will be ignored but it is also possible to treat
+them as if they had a value.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search
 {
     "aggs" : {
         "grade_min" : {
@@ -120,5 +142,8 @@ had a value.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-<1> Documents without a value in the `grade` field will fall into the same bucket as documents that have the value `10`. 
+<1> Documents without a value in the `grade` field will fall into the same
+bucket as documents that have the value `10`.
diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index f0ada57b37f..b7193b827cd 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -156,9 +156,9 @@ PUT /transactions/stock/_bulk?refresh
 {"type": "sale","amount": 80}
 {"index":{"_id":2}}
 {"type": "cost","amount": 10}
-{"index":{"_id":2}}
+{"index":{"_id":3}}
 {"type": "cost","amount": 30}
-{"index":{"_id":2}}
+{"index":{"_id":4}}
 {"type": "sale","amount": 130}
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 20312a7de64..6935354c0ac 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -33,27 +33,26 @@ only the title field is being included in the source.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs": {
-        "top-tags": {
+        "top_tags": {
             "terms": {
-                "field": "tags",
+                "field": "type",
                 "size": 3
             },
             "aggs": {
-                "top_tag_hits": {
+                "top_sales_hits": {
                     "top_hits": {
                         "sort": [
                             {
-                                "last_activity_date": {
+                                "date": {
                                     "order": "desc"
                                 }
                             }
                         ],
                         "_source": {
-                            "includes": [
-                                "title"
-                            ]
+                            "includes": [ "date", "price" ]
                         },
                         "size" : 1
                     }
@@ -63,90 +62,105 @@ only the title field is being included in the source.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
-Possible response snippet:
+Possible response:
 
 [source,js]
 --------------------------------------------------
-"aggregations": {
-  "top-tags": {
-     "buckets": [
-        {
-           "key": "windows-7",
-           "doc_count": 25365,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 25365,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602679",
-                       "_score": 1,
-                       "_source": {
-                          "title": "Windows port opening"
-                       },
-                       "sort": [
-                          1370143231177
-                       ]
-                    }
-                 ]
-              }
-           }
-        },
-        {
-           "key": "linux",
-           "doc_count": 18342,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 18342,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602672",
-                       "_score": 1,
-                       "_source": {
-                          "title": "Ubuntu RFID Screensaver lock-unlock"
-                       },
-                       "sort": [
-                          1370143379747
-                       ]
-                    }
-                 ]
-              }
-           }
-        },
-        {
-           "key": "windows",
-           "doc_count": 18119,
-           "top_tags_hits": {
-              "hits": {
-                 "total": 18119,
-                 "max_score": 1,
-                 "hits": [
-                    {
-                       "_index": "stack",
-                       "_type": "question",
-                       "_id": "602678",
-                       "_score": 1,
-                       "_source": {
-                          "title": "If I change my computers date / time, what could be affected?"
-                       },
-                       "sort": [
-                          1370142868283
-                       ]
-                    }
-                 ]
-              }
-           }
-        }
-     ]
+{
+  ...
+  "aggregations": {
+    "top_tags": {
+       "doc_count_error_upper_bound": 0,
+       "sum_other_doc_count": 0,
+       "buckets": [
+          {
+             "key": "hat",
+             "doc_count": 3,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 3,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmauCQpcRyxw6ChK",
+                         "_source": {
+                            "date": "2015/03/01 00:00:00",
+                            "price": 200
+                         },
+                         "sort": [
+                            1425168000000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          },
+          {
+             "key": "t-shirt",
+             "doc_count": 3,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 3,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmauCQpcRyxw6ChL",
+                         "_source": {
+                            "date": "2015/03/01 00:00:00",
+                            "price": 175
+                         },
+                         "sort": [
+                            1425168000000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          },
+          {
+             "key": "bag",
+             "doc_count": 1,
+             "top_sales_hits": {
+                "hits": {
+                   "total": 1,
+                   "max_score": null,
+                   "hits": [
+                      {
+                         "_index": "sales",
+                         "_type": "sale",
+                         "_id": "AVnNBmatCQpcRyxw6ChH",
+                         "_source": {
+                            "date": "2015/01/01 00:00:00",
+                            "price": 150
+                         },
+                         "sort": [
+                            1420070400000
+                         ],
+                         "_score": null
+                      }
+                   ]
+                }
+             }
+          }
+       ]
+    }
   }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChK/$body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._id/]
+// TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChL/$body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._id/]
+// TESTRESPONSE[s/AVnNBmatCQpcRyxw6ChH/$body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._id/]
+
 
 ==== Field collapse example
 
@@ -184,7 +198,6 @@ relevancy order of the most relevant document in a bucket.
         "top_hit" : {
           "max": {
             "script": {
-              "lang": "painless",
               "inline": "_score"
             }
           }
diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
index 925f5426187..8b67d64c877 100644
--- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
@@ -8,12 +8,15 @@ one might be interested in the number of values the average is computed over.
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
     "aggs" : {
-        "grades_count" : { "value_count" : { "field" : "grade" } }
+        "types_count" : { "value_count" : { "field" : "type" } }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 Response:
 
@@ -21,14 +24,14 @@ Response:
 --------------------------------------------------
 {
     ...
-
     "aggregations": {
-        "grades_count": {
-            "value": 10
+        "types_count": {
+            "value": 7
         }
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
 
 The name of the aggregation (`grades_count` above) also serves as the key by which the aggregation result can be
 retrieved from the returned response.
@@ -39,36 +42,35 @@ Counting the values generated by a script:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "grades_count" : { 
-            "value_count" : { 
+        "type_count" : {
+            "value_count" : {
                 "script" : {
-                    "inline" : "doc['grade'].value",
-                    "lang" : "painless"
+                    "inline" : "doc['type'].value"
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
+POST /sales/_search?size=0
 {
-    ...,
-
     "aggs" : {
-        "grades_count" : { 
-            "value_count" : { 
+        "grades_count" : {
+            "value_count" : {
                 "script" : {
                     "file": "my_script",
                     "params" : {
-                        "field" : "grade"
+                        "field" : "type"
                     }
                 }
             }
@@ -76,5 +78,7 @@ This will interpret the `script` parameter as an `inline` script with the `painl
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:sales]
 
 TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc
index 42b1ef65d24..a8299f45e19 100644
--- a/docs/reference/analysis.asciidoc
+++ b/docs/reference/analysis.asciidoc
@@ -112,6 +112,8 @@ include::analysis/testing.asciidoc[]
 
 include::analysis/analyzers.asciidoc[]
 
+include::analysis/normalizers.asciidoc[]
+
 include::analysis/tokenizers.asciidoc[]
 
 include::analysis/tokenfilters.asciidoc[]
diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
index 983a719e4f4..02843789c4b 100644
--- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
@@ -55,9 +55,9 @@ functionality is implemented by adding the
 with the `keywords` set to the value of the `stem_exclusion` parameter.
 
 The following analyzers support setting custom `stem_exclusion` list:
-`arabic`, `armenian`, `basque`, `catalan`, `bulgarian`, `catalan`,
-`czech`, `finnish`, `dutch`, `english`, `finnish`, `french`, `galician`,
-`german`, `irish`, `hindi`, `hungarian`, `indonesian`, `italian`, `latvian`,
+`arabic`, `armenian`, `basque`, `bulgarian`, `catalan`, `czech`,
+`dutch`, `english`, `finnish`, `french`, `galician`,
+`german`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `latvian`,
 `lithuanian`, `norwegian`, `portuguese`, `romanian`, `russian`, `sorani`,
 `spanish`, `swedish`, `turkish`.
 
diff --git a/docs/reference/analysis/charfilters.asciidoc b/docs/reference/analysis/charfilters.asciidoc
index cd24f5bf571..47390fe0ae8 100644
--- a/docs/reference/analysis/charfilters.asciidoc
+++ b/docs/reference/analysis/charfilters.asciidoc
@@ -6,8 +6,8 @@ is passed to the <<analysis-tokenizers,tokenizer>>.
 
 A character filter receives the original text as a stream of characters and
 can transform the stream by adding, removing, or changing characters.  For
-instance, a character filter could be used to convert Arabic numerals
-(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Latin equivalents (0123456789), or to strip HTML
+instance, a character filter could be used to convert Hindu-Arabic numerals
+(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Arabic-Latin equivalents (0123456789), or to strip HTML
 elements like `<b>` from the stream.
 
 
diff --git a/docs/reference/analysis/normalizers.asciidoc b/docs/reference/analysis/normalizers.asciidoc
new file mode 100644
index 00000000000..0287f140c74
--- /dev/null
+++ b/docs/reference/analysis/normalizers.asciidoc
@@ -0,0 +1,57 @@
+[[analysis-normalizers]]
+== Normalizers
+
+experimental[]
+
+Normalizers are similar to analyzers except that they may only emit a single
+token. As a consequence, they do not have a tokenizer and only accept a subset
+of the available char filters and token filters. Only the filters that work on
+a per-character basis are allowed. For instance a lowercasing filter would be
+allowed, but not a stemming filter, which needs to look at the keyword as a
+whole.
+
+[float]
+=== Custom analyzers
+
+Elasticsearch does not ship with built-in normalizers so far, so the only way
+to get one is by building a custom one. Custom normalizers take a list of char
+<<analysis-charfilters, character filters>> and a list of
+<<analysis-tokenfilters,token filters>>.
+
+[source,js]
+--------------------------------
+PUT index
+{
+  "settings": {
+    "analysis": {
+      "char_filter": {
+        "quote": {
+          "type": "mapping",
+          "mappings": [
+            "« => \"",
+            "» => \""
+          ]
+        }
+      },
+      "normalizer": {
+        "my_normalizer": {
+          "type": "custom",
+          "char_filter": ["quote"],
+          "filter": ["lowercase", "asciifolding"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword",
+          "normalizer": "my_normalizer"
+        }
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
diff --git a/docs/reference/analysis/tokenfilters.asciidoc b/docs/reference/analysis/tokenfilters.asciidoc
index 227947fb45e..eae6ceabed7 100644
--- a/docs/reference/analysis/tokenfilters.asciidoc
+++ b/docs/reference/analysis/tokenfilters.asciidoc
@@ -13,6 +13,8 @@ include::tokenfilters/standard-tokenfilter.asciidoc[]
 
 include::tokenfilters/asciifolding-tokenfilter.asciidoc[]
 
+include::tokenfilters/flatten-graph-tokenfilter.asciidoc[]
+
 include::tokenfilters/length-tokenfilter.asciidoc[]
 
 include::tokenfilters/lowercase-tokenfilter.asciidoc[]
diff --git a/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
new file mode 100644
index 00000000000..90b61368983
--- /dev/null
+++ b/docs/reference/analysis/tokenfilters/flatten-graph-tokenfilter.asciidoc
@@ -0,0 +1,20 @@
+[[analysis-flatten-graph-tokenfilter]]
+=== Flatten Graph Token Filter
+
+experimental[]
+
+The `flatten_graph` token filter accepts an arbitrary graph token
+stream, such as that produced by
+<<analysis-synonym-graph-tokenfilter>>, and flattens it into a single
+linear chain of tokens suitable for indexing.
+
+This is a lossy process, as separate side paths are squashed on top of
+one another, but it is necessary if you use a graph token stream
+during indexing because a Lucene index cannot currently represent a
+graph.  For this reason, it's best to apply graph analyzers only at
+search time because that preserves the full graph structure and gives
+correct matches for proximity queries.
+
+For more information on this topic and its various complexities,
+please read the http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's
+TokenStreams are actually graphs] blog post.
diff --git a/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
index ccde46a3fd2..b233081d540 100644
--- a/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/pattern-capture-tokenfilter.asciidoc
@@ -53,7 +53,7 @@ curl -XPUT localhost:9200/test/  -d '
          "filter" : {
             "code" : {
                "type" : "pattern_capture",
-               "preserve_original" : 1,
+               "preserve_original" : true,
                "patterns" : [
                   "(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)",
                   "(\\d+)"
@@ -94,7 +94,7 @@ curl -XPUT localhost:9200/test/  -d '
          "filter" : {
             "email" : {
                "type" : "pattern_capture",
-               "preserve_original" : 1,
+               "preserve_original" : true,
                "patterns" : [
                   "([^@]+)",
                   "(\\p{L}+)",
diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
index 16758ad6ad2..5ab498802d9 100644
--- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc
@@ -8,9 +8,8 @@ including multi-word synonyms correctly during the analysis process.
 
 In order to properly handle multi-word synonyms this token filter
 creates a "graph token stream" during processing.  For more information
-on this topic and it's various complexities, please read
-http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's TokenStreams are actually graphs!]
-by Michael McCandless.
+on this topic and its various complexities, please read the
+http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's TokenStreams are actually graphs] blog post.
 
 ["NOTE",id="synonym-graph-index-note"]
 ===============================
@@ -19,15 +18,6 @@ only.  If you want to apply synonyms during indexing please use the
 standard <<analysis-synonym-tokenfilter,synonym token filter>>.
 ===============================
 
-["NOTE",id="synonym-graph-query-note"]
-===============================
-The graph token stream created by this token filter requires special
-query handling. Currently only the <<query-dsl-match-query, Match>> and 
-<<query-dsl-multi-match-query, Multi Match>> queries can do this.  Using 
-it with any other type of analyzed query will potentially result in 
-incorrect search results.
-===============================
-
 Synonyms are configured using a configuration file.
 Here is an example:
 
diff --git a/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
index 14652f46342..4c28ddba381 100644
--- a/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/truncate-tokenfilter.asciidoc
@@ -2,9 +2,7 @@
 === Truncate Token Filter
 
 The `truncate` token filter can be used to truncate tokens into a
-specific length. This can come in handy with keyword (single token)
-based mapped fields that are used for sorting in order to reduce memory
-usage.
+specific length.
 
 It accepts a `length` parameter which control the number of characters
 to truncate to, defaults to `10`.
diff --git a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
index 9d06ea28d55..55ce590031f 100644
--- a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
@@ -1,5 +1,5 @@
 [[analysis-whitespace-tokenizer]]
-=== Whitespace Analyzer
+=== Whitespace Tokenizer
 
 The `whitespace` tokenizer breaks text into terms whenever it encounters a
 whitespace character.
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index ef0b8f37958..7e8ed1c1c81 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -118,6 +118,7 @@ The special characters used for date rounding must be URI encoded as follows:
 `|`:: `%7C`
 `+`:: `%2B`
 `:`:: `%3A`
+`,`:: `%2C`
 ======================================================
 
 The following example shows different forms of date math index names and the final index names
@@ -475,9 +476,8 @@ convention of using underscore casing.
 === Boolean Values
 
 All REST APIs parameters (both request parameters and JSON body) support
-providing boolean "false" as the values: `false`, `0`, `no` and `off`.
-All other values are considered "true". Note, this is not related to
-fields within a document indexed treated as boolean fields.
+providing boolean "false" as the value `false` and boolean "true" as the
+value `true`. All other values will raise an error.
 
 [float]
 === Number Values
@@ -670,7 +670,27 @@ The response looks like:
 
 For libraries that don't accept a request body for non-POST requests,
 you can pass the request body as the `source` query string parameter
-instead.
+instead. When using this method, the `source_content_type` parameter
+should also be passed with a media type value that indicates the format
+of the source, such as `application/json`.
+
+[float]
+=== Content-Type auto-detection
+
+deprecated[5.3.0, Provide the proper Content-Type header]
+
+The content sent in a request body or using the `source` query string
+parameter is inspected to automatically determine the content type
+(JSON, YAML, SMILE, or CBOR).
+
+A strict mode can be enabled which disables auto-detection and requires
+that all requests with a body have a Content-Type header that maps to
+a supported format. To enabled this strict mode, add the following
+setting to the `elasticsearch.yml` file:
+
+    http.content_type.required: true
+
+The default value is `false`.
 
 [[url-access-control]]
 == URL-based access control
@@ -682,7 +702,7 @@ the choice of specifying an index in the URL and on each individual request
 within the request body. This can make URL-based access control challenging.
 
 To prevent the user from overriding the index which has been specified in the
-URL, add this setting to the `config.yml` file:
+URL, add this setting to the `elasticsearch.yml` file:
 
     rest.action.multi.allow_explicit_index: false
 
diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc
index cca24c66a36..a87fe4e5e4a 100644
--- a/docs/reference/cat/health.asciidoc
+++ b/docs/reference/cat/health.asciidoc
@@ -22,7 +22,7 @@ It has one option `ts` to disable the timestamping:
 
 [source,js]
 --------------------------------------------------
-GET /_cat/health?v&ts=0
+GET /_cat/health?v&ts=false
 --------------------------------------------------
 // CONSOLE
 // TEST[s/^/PUT twitter\n{"settings":{"number_of_replicas": 0}}\n/]
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index 2d9312a7849..1978c26b7fd 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -164,11 +164,11 @@ phase |37ms
 operations |7
 |`search.open_contexts` |`so`, `searchOpenContexts` |No |Open search
 contexts |0
-|`search.query_current` |`sqc`, `searchFetchCurrent` |No |Current query
+|`search.query_current` |`sqc`, `searchQueryCurrent` |No |Current query
 phase operations |0
-|`search.query_time` |`sqti`, `searchFetchTime` |No |Time spent in query
+|`search.query_time` |`sqti`, `searchQueryTime` |No |Time spent in query
 phase |43ms
-|`search.query_total` |`sqto`, `searchFetchTotal` |No |Number of query
+|`search.query_total` |`sqto`, `searchQueryTotal` |No |Number of query
 operations |9
 |`search.scroll_current` |`scc`, `searchScrollCurrent` |No |Open scroll contexts |2
 |`search.scroll_time` |`scti`, `searchScrollTime` |No |Time scroll contexts held open|2m
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index 972a918b145..0116b055698 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -1,203 +1,291 @@
 [[cluster-allocation-explain]]
 == Cluster Allocation Explain API
 
-The cluster allocation explanation API is designed to assist in answering the
-question "why is this shard unassigned?". To explain the allocation (on
-unassigned state) of a shard, issue a request like:
+The purpose of the cluster allocation explain API is to provide
+explanations for shard allocations in the cluster.  For unassigned shards,
+the explain API provides an explanation for why the shard is unassigned.
+For assigned shards, the explain API provides an explanation for why the
+shard is remaining on its current moved and has not moved or rebalanced to
+another node.  This API can be very useful when attempting to diagnose why
+a shard is unassigned or why a shard continues to remain on its current node
+when you might expect otherwise.
 
-experimental[The cluster allocation explain API is new and should still be considered experimental. The API may change in ways that are not backwards compatible]
+[float]
+=== Explain API Request
+
+To explain the allocation of a shard, issue a request:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain' -d'{
+$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain' -d '{
   "index": "myindex",
   "shard": 0,
-  "primary": false
+  "primary": true
 }'
 --------------------------------------------------
 
 Specify the `index` and `shard` id of the shard you would like an explanation
-for, as well as the `primary` flag to indicate whether to explain a primary or
-replica shard.
+for, as well as the `primary` flag to indicate whether to explain the primary
+shard for the given shard id or one of its replica shards.  These three request
+parameters are required.
 
-The response looks like:
+You may also specify an optional `current_node` request parameter to only explain
+a shard that is currently located on `current_node`.  The `current_node` can be
+specified as either the node id or node name.
 
 [source,js]
 --------------------------------------------------
-{
-  "shard" : {
-    "index" : "myindex",
-    "index_uuid" : "KnW0-zELRs6PK84l0r38ZA",
-    "id" : 0,
-    "primary" : false
-  },
-  "assigned" : false,                             <1>
-  "shard_state_fetch_pending": false,             <2>
-  "unassigned_info" : {
-    "reason" : "INDEX_CREATED",                   <3>
-    "at" : "2016-03-22T20:04:23.620Z"
-  },
-  "allocation_delay_ms" : 0,                      <4>
-  "remaining_delay_ms" : 0,                       <5>
-  "nodes" : {
-    "V-Spi0AyRZ6ZvKbaI3691w" : {
-      "node_name" : "H5dfFeA",
-      "node_attributes" : {                       <6>
-        "bar" : "baz"
-      },
-      "store" : {
-        "shard_copy" : "NONE"                     <7>
-      },
-      "final_decision" : "NO",                    <8>
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 0.06666675,                      <9>
-      "decisions" : [ {                           <10>
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    },
-    "Qc6VL8c5RWaw1qXZ0Rg57g" : {
-      "node_name" : "bGG90GE",
-      "node_attributes" : {
-        "bar" : "baz",
-        "foo" : "bar"
-      },
-      "store" : {
-        "shard_copy" : "AVAILABLE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : -1.3833332,
-      "decisions" : [ {
-        "decider" : "same_shard",
-        "decision" : "NO",
-        "explanation" : "the shard cannot be allocated on the same node id [Qc6VL8c5RWaw1qXZ0Rg57g] on which it already exists"
-      } ]
-    },
-    "PzdyMZGXQdGhqTJHF_hGgA" : {
-      "node_name" : "DKDM97B",
-      "node_attributes" : { },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 2.3166666,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    }
-  }
-}
+$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain' -d '{
+  "index": "myindex",
+  "shard": 0,
+  "primary": false,
+  "current_node": "nodeA"                         <1>
+}'
 --------------------------------------------------
-<1> Whether the shard is assigned or unassigned
-<2> Whether information about the shard is still being fetched
-<3> Reason for the shard originally becoming unassigned
-<4> Configured delay before the shard can be allocated
-<5> Remaining delay before the shard can be allocated
-<6> User-added attributes the node has
-<7> The shard copy information for this node and error (if applicable)
-<8> Final decision and explanation of whether the shard can be allocated to this node
-<9> Weight for how much the allocator would like to allocate the shard to this node
-<10> List of node decisions factoring into final decision about the shard
-
-For a shard that is already assigned, the output looks similar to:
-
-[source,js]
---------------------------------------------------
-{
-  "shard" : {
-    "index" : "only-foo",
-    "index_uuid" : "KnW0-zELRs6PK84l0r38ZA",
-    "id" : 0,
-    "primary" : true
-  },
-  "assigned" : true,
-  "assigned_node_id" : "Qc6VL8c5RWaw1qXZ0Rg57g",      <1>
-  "shard_state_fetch_pending": false,
-  "allocation_delay_ms" : 0,
-  "remaining_delay_ms" : 0,
-  "nodes" : {
-    "V-Spi0AyRZ6ZvKbaI3691w" : {
-      "node_name" : "bGG90GE",
-      "node_attributes" : {
-        "bar" : "baz"
-      },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 1.4499999,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    },
-    "Qc6VL8c5RWaw1qXZ0Rg57g" : {
-      "node_name" : "I8hydUG",
-      "node_attributes" : {
-        "bar" : "baz",
-        "foo" : "bar"
-      },
-      "store" : {
-        "shard_copy" : "AVAILABLE"
-      },
-      "final_decision" : "ALREADY_ASSIGNED",        <2>
-      "final_explanation" : "the shard is already assigned to this node",
-      "weight" : 0.0,
-      "decisions" : [ {
-        "decider" : "same_shard",
-        "decision" : "NO",
-        "explanation" : "the shard cannot be allocated on the same node id [Qc6VL8c5RWaw1qXZ0Rg57g] on which it already exists"
-      } ]
-    },
-    "PzdyMZGXQdGhqTJHF_hGgA" : {
-      "node_name" : "H5dfFeA",
-      "node_attributes" : { },
-      "store" : {
-        "shard_copy" : "NONE"
-      },
-      "final_decision" : "NO",
-      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 3.6999998,
-      "decisions" : [ {
-        "decider" : "filter",
-        "decision" : "NO",
-        "explanation" : "node does not match index include filters [foo:\"bar\"]"
-      } ]
-    }
-  }
-}
---------------------------------------------------
-<1> Node the shard is currently assigned to
-<2> The decision is "ALREADY_ASSIGNED" because the shard is currently assigned to this node
+<1> The node where shard 0 currently has a replica on
 
 You can also have Elasticsearch explain the allocation of the first unassigned
-shard it finds by sending an empty body, such as:
+shard that it finds by sending an empty body for the request:
 
 [source,js]
 --------------------------------------------------
 $ curl -XGET 'http://localhost:9200/_cluster/allocation/explain'
 --------------------------------------------------
 
-If you would like to include all decisions that were factored into the final
-decision, the `include_yes_decisions` parameter will return all decisions:
+[float]
+=== Explain API Response
+
+This section includes examples of the cluster allocation explain API response output
+under various scenarios.
+
+The API response for an unassigned shard:
+
+[source,js]
+--------------------------------------------------
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "unassigned",                 <1>
+  "unassigned_info" : {
+    "reason" : "INDEX_CREATED",                   <2>
+    "at" : "2017-01-04T18:08:16.600Z",
+    "last_allocation_status" : "no"
+  },
+  "can_allocate" : "no",                          <3>
+  "allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "8qt2rY-pT6KNZB3-hGfLnw",
+      "node_name" : "node_t1",
+      "transport_address" : "127.0.0.1:9401",
+      "node_decision" : "no",                     <4>
+      "weight_ranking" : 1,
+      "deciders" : [
+        {
+          "decider" : "filter",                   <5>
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"  <6>
+        }
+      ]
+    },
+    {
+      "node_id" : "7Wr-QxLXRLKDxhzNm50pFA",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "weight_ranking" : 2,
+      "deciders" : [
+        {
+          "decider" : "filter",
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+        }
+      ]
+    }
+  ]
+}
+--------------------------------------------------
+<1> The current state of the shard
+<2> The reason for the shard originally becoming unassigned
+<3> Whether to allocate the shard
+<4> Whether to allocate the shard to the particular node
+<5> The decider which led to the `no` decision for the node
+<6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision
+
+You can return information gathered by the cluster info service about disk usage
+and shard sizes by setting the `include_disk_info` parameter to `true`:
+
+[source,js]
+--------------------------------------------------
+$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain?include_disk_info=true'
+--------------------------------------------------
+
+Additionally, if you would like to include all decisions that were factored into the final
+decision, the `include_yes_decisions` parameter will return all decisions for each node:
 
 [source,js]
 --------------------------------------------------
 $ curl -XGET 'http://localhost:9200/_cluster/allocation/explain?include_yes_decisions=true'
 --------------------------------------------------
 
-Additionally, you can return information gathered by the cluster info service
-about disk usage and shard sizes by setting the `include_disk_info` parameter to
-`true`:
+The default value for `include_yes_decisions` is `false`, which will only
+include the `no` decisions in the response.  This is generally what you would
+want, as the `no` decisions indicate why a shard is unassigned or cannot be moved,
+and including all decisions include the `yes` ones adds a lot of verbosity to the
+API's response output.
+
+The API response output for an unassigned primary shard that had previously been
+allocated to a node in the cluster:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/_cluster/allocation/explain?include_disk_info=true'
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "reason" : "NODE_LEFT",
+    "at" : "2017-01-04T18:03:28.464Z",
+    "details" : "node_left[OIWe8UhhThCK0V5XfmdrmQ]",
+    "last_allocation_status" : "no_valid_shard_copy"
+  },
+  "can_allocate" : "no_valid_shard_copy",
+  "allocate_explanation" : "cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster"
+}
 --------------------------------------------------
+
+The API response output for a replica that is unassigned due to delayed allocation:
+
+[source,js]
+--------------------------------------------------
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : false,
+  "current_state" : "unassigned",
+  "unassigned_info" : {
+    "reason" : "NODE_LEFT",
+    "at" : "2017-01-04T18:53:59.498Z",
+    "details" : "node_left[G92ZwuuaRY-9n8_tc-IzEg]",
+    "last_allocation_status" : "no_attempt"
+  },
+  "can_allocate" : "allocation_delayed",
+  "allocate_explanation" : "cannot allocate because the cluster is still waiting 59.8s for the departed node holding a replica to rejoin, despite being allowed to allocate the shard to at least one other node",
+  "configured_delay" : "1m",                      <1>
+  "configured_delay_in_millis" : 60000,
+  "remaining_delay" : "59.8s",                    <2>
+  "remaining_delay_in_millis" : 59824,
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "pmnHu_ooQWCPEFobZGbpWw",
+      "node_name" : "node_t2",
+      "transport_address" : "127.0.0.1:9402",
+      "node_decision" : "yes"
+    },
+    {
+      "node_id" : "3sULLVJrRneSg0EfBB-2Ew",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "store" : {                                 <3>
+        "matching_size" : "4.2kb",
+        "matching_size_in_bytes" : 4325
+      },
+      "deciders" : [
+        {
+          "decider" : "same_shard",
+          "decision" : "NO",
+          "explanation" : "the shard cannot be allocated to the same node on which a copy of the shard already exists [[idx][0], node[3sULLVJrRneSg0EfBB-2Ew], [P], s[STARTED], a[id=eV9P8BN1QPqRc3B4PLx6cg]]"
+        }
+      ]
+    }
+  ]
+}
+--------------------------------------------------
+<1> The configured delay before allocating a replica shard that does not exist due to the node holding it leaving the cluster
+<2> The remaining delay before allocating the replica shard
+<3> Information about the shard data found on a node
+
+The API response output for an assigned shard that is not allowed to
+remain on its current node and is required to move:
+
+[source,js]
+--------------------------------------------------
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "started",
+  "current_node" : {
+    "id" : "8lWJeJ7tSoui0bxrwuNhTA",
+    "name" : "node_t1",
+    "transport_address" : "127.0.0.1:9401"
+  },
+  "can_remain_on_current_node" : "no",            <1>
+  "can_remain_decisions" : [                      <2>
+    {
+      "decider" : "filter",
+      "decision" : "NO",
+      "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+    }
+  ],
+  "can_move_to_other_node" : "no",                <3>
+  "move_explanation" : "cannot move shard to another node, even though it is not allowed to remain on its current node",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "_P8olZS8Twax9u6ioN-GGA",
+      "node_name" : "node_t0",
+      "transport_address" : "127.0.0.1:9400",
+      "node_decision" : "no",
+      "weight_ranking" : 1,
+      "deciders" : [
+        {
+          "decider" : "filter",
+          "decision" : "NO",
+          "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]"
+        }
+      ]
+    }
+  ]
+}
+--------------------------------------------------
+<1> Whether the shard is allowed to remain on its current node
+<2> The deciders that factored into the decision of why the shard is not allowed to remain on its current node
+<3> Whether the shard is allowed to be allocated to another node
+
+The API response output for an assigned shard that remains on its current node
+because moving the shard to another node does not form a better cluster balance:
+
+[source,js]
+--------------------------------------------------
+{
+  "index" : "idx",
+  "shard" : 0,
+  "primary" : true,
+  "current_state" : "started",
+  "current_node" : {
+    "id" : "wLzJm4N4RymDkBYxwWoJsg",
+    "name" : "node_t0",
+    "transport_address" : "127.0.0.1:9400",
+    "weight_ranking" : 1
+  },
+  "can_remain_on_current_node" : "yes",
+  "can_rebalance_cluster" : "yes",                <1>
+  "can_rebalance_to_other_node" : "no",           <2>
+  "rebalance_explanation" : "cannot rebalance as no target node exists that can both allocate this shard and improve the cluster balance",
+  "node_allocation_decisions" : [
+    {
+      "node_id" : "oE3EGFc8QN-Tdi5FFEprIA",
+      "node_name" : "node_t1",
+      "transport_address" : "127.0.0.1:9401",
+      "node_decision" : "worse_balance",          <3>
+      "weight_ranking" : 1
+    }
+  ]
+}
+--------------------------------------------------
+<1> Whether rebalancing is allowed on the cluster
+<2> Whether the shard can be rebalanced to another node
+<3> The reason the shard cannot be rebalanced to the node, in this case indicating that it offers no better balance than the current node
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index ce550a689bf..e087eebd9c7 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -79,8 +79,57 @@ GET _tasks?parent_task_id=parentTaskId:1 <1>
 
 <1> This won't return a 404 if the parent isn't found.
 
-The task API can be also used to wait for completion of a particular task. The following call will
-block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is completed.
+You can also use the `detailed` request parameter to get more information about
+the running tasks. This is useful for telling one task from another but is more
+costly to execute. For example, fetching all searches using the `detailed`
+request parameter:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?actions=*search&detailed
+--------------------------------------------------
+// CONSOLE
+
+might look like:
+
+[source,js]
+--------------------------------------------------
+{
+  "nodes" : {
+    "oTUltX4IQMOUUVeiohTt8A" : {
+      "name" : "H5dfFeA",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1:9300",
+      "tasks" : {
+        "oTUltX4IQMOUUVeiohTt8A:464" : {
+          "node" : "oTUltX4IQMOUUVeiohTt8A",
+          "id" : 464,
+          "type" : "transport",
+          "action" : "indices:data/read/search",
+          "description" : "indices[test], types[test], search_type[QUERY_THEN_FETCH], source[{\"query\":...}]",
+          "start_time_in_millis" : 1483478610008,
+          "running_time_in_nanos" : 13991383,
+          "cancellable" : true
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+The new `description` field contains human readable text that identifies the
+particular request that the task is performing such as identifying the search
+request being performed by a search task like the example above. Other kinds of
+task have have different descriptions, like <<docs-reindex,`_reindex`>> which
+has the search and the destination, or <<docs-bulk,`_bulk`>> which just has the
+number of requests and the destination indices. Many requests will only have an
+empty description because more detailed information about the request is not
+easily available or particularly helpful in identifying the request.
+
+The task API can also be used to wait for completion of a particular task. The
+following call will block for 10 seconds or until the task with id
+`oTUltX4IQMOUUVeiohTt8A:12345` is completed.
 
 [source,js]
 --------------------------------------------------
@@ -98,12 +147,13 @@ GET _tasks?actions=*reindex&wait_for_completion=true&timeout=10s
 --------------------------------------------------
 // CONSOLE
 
-Tasks can be also listed using _cat version of the list tasks command, which accepts the same arguments
-as the standard list tasks command.
+Tasks can be also listed using _cat version of the list tasks command, which
+accepts the same arguments as the standard list tasks command.
 
 [source,js]
 --------------------------------------------------
 GET _cat/tasks
+GET _cat/tasks?detailed
 --------------------------------------------------
 // CONSOLE
 
diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index 04049663e84..a8ab282853c 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -4,7 +4,8 @@
 [partintro]
 --
 
-This section describes the following CRUD APIs:
+This section starts with a short introduction to Elasticsearch's <<docs-replication,data replication model>>, followed by a
+detailed description of the following CRUD APIs:
 
 .Single document APIs
 * <<docs-index_>>
@@ -15,6 +16,7 @@ This section describes the following CRUD APIs:
 .Multi-document APIs
 * <<docs-multi-get>>
 * <<docs-bulk>>
+* <<docs-delete-by-query>>
 * <<docs-update-by-query>>
 * <<docs-reindex>>
 
@@ -23,6 +25,8 @@ index name, or an `alias` which points to a single index.
 
 --
 
+include::docs/data-replication.asciidoc[]
+
 include::docs/index_.asciidoc[]
 
 include::docs/get.asciidoc[]
diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc
new file mode 100644
index 00000000000..47af258204f
--- /dev/null
+++ b/docs/reference/docs/data-replication.asciidoc
@@ -0,0 +1,149 @@
+
+[[docs-replication]]
+== Reading and Writing documents
+
+[float]
+=== Introduction
+
+Each index in Elasticsearch is <<getting-started-shards-and-replicas,divided into shards>>
+and each shard can have multiple copies. These copies are known as a _replication group_ and must be kept in sync when documents
+are added or removed. If we fail to do so, reading from one copy will result in very different results than reading from another.
+The process of keeping the shard copies in sync and serving reads from them is what we call the _data replication model_.
+
+Elasticsearch’s data replication model is based on the _primary-backup model_ and is described very well in the 
+https://www.microsoft.com/en-us/research/publication/pacifica-replication-in-log-based-distributed-storage-systems/[PacificA paper] of
+Microsoft Research. That model is based on having a single copy from the replication group that acts as the primary shard.
+The other copies are called _replica shards_. The primary serves as the main entry point for all indexing operations. It is in charge of
+validating them and making sure they are correct. Once an index operation has been accepted by the primary, the primary is also
+responsible for replicating the operation to the other copies.
+
+This purpose of this section is to give a high level overview of the Elasticsearch replication model and discuss the implications
+it has for various interactions between write and read operations.
+
+[float]
+=== Basic write model
+
+Every indexing operation in Elasticsearch is first resolved to a replication group using <<index-routing,routing>>, 
+typically based on the document ID. Once the replication group has been determined,
+the operation is forwarded internally to the current _primary shard_ of the group. The primary shard is responsible
+for validating the operation and forwarding it to the other replicas. Since replicas can be offline, the primary
+is not required to replicate to all replicas. Instead, Elasticsearch maintains a list of shard copies that should
+receive the operation. This list is called the _in-sync copies_ and is maintained by the master node. As the name implies,
+these are the set of "good" shard copies that are guaranteed to have processed all of the index and delete operations that
+have been acknowledged to the user. The primary is responsible for maintaining this invariant and thus has to replicate all
+operations to each copy in this set.
+
+The primary shard follows this basic flow:
+
+. Validate incoming operation and reject it if structurally invalid (Example: have an object field where a number is expected)
+. Execute the operation locally i.e. indexing or deleting the relevant document. This will also validate the content of fields
+   and reject if needed (Example: a keyword value is too long for indexing in Lucene).
+. Forward the operation to each replica in the current in-sync copies set. If there are multiple replicas, this is done in parallel.
+. Once all replicas have successfully performed the operation and responded to the primary, the primary acknowledges the successful
+   completion of the request to the client.
+
+[float]
+==== Failure handling
+
+Many things can go wrong during indexing -- disks can get corrupted, nodes can be disconnected from each other, or some
+configuration mistake could cause an operation to fail on a replica despite it being successful on the primary. These
+are infrequent but the primary has to respond to them.
+
+In the case that the primary itself fails, the node hosting the primary will send a message to the master about it. The indexing
+operation will wait (up to 1 minute, by <<dynamic-index-settings,default>>) for the master to promote one of the replicas to be a 
+new primary. The operation will then be forwarded to the new primary for processing. Note that the master also monitors the
+health of the nodes and may decide to proactively demote a primary. This typically happens when the node holding the primary
+is isolated from the cluster by a networking issue. See <<demoted-primary,here>> for more details.
+
+Once the operation has been successfully performed on the primary, the primary has to deal with potential failures
+when executing it on the replica shards. This may be caused by an actual failure on the replica or due to a network
+issue preventing the operation from reaching the replica (or preventing the replica from responding). All of these
+share the same end result: a replica which is part of the in-sync replica set misses an operation that is about to
+be acknowledged. In order to avoid violating the invariant, the primary sends a message to the master requesting
+that the problematic shard be removed from the in-sync replica set. Only once removal of the shard has been acknowledged 
+by the master does the primary acknowledge the operation. Note that the master will also instruct another node to start 
+building a new shard copy in order to restore the system to a healthy state.
+
+[[demoted-primary]]
+While forwarding an operation to the replicas, the primary will use the replicas to validate that it is still the
+active primary. If the primary has been isolated due to a network partition (or a long GC) it may continue to process
+incoming indexing operations before realising that it has been demoted. Operations that come from a stale primary
+will be rejected by the replicas. When the primary receives a response from the replica rejecting its request because
+it is no longer the primary then it will reach out to the master and will learn that it has been replaced. The
+operation is then routed to the new primary.
+
+.What happens if there are no replicas? 
+************
+This is a valid scenario that can happen due to index configuration or simply
+because all the replicas have failed. In that case the primary is processing operations without any external validation,
+which may seem problematic. On the other hand, the primary cannot fail other shards on its own but request the master to do
+so on its behalf. This means that the master knows that the primary is the only single good copy. We are therefore guaranteed 
+that the master will not promote any other (out-of-date) shard copy to be a  new primary and that any operation indexed 
+into the primary will not be lost. Of course, since at that point we are running with only single copy of the data, physical hardware
+issues can cause data loss. See <<index-wait-for-active-shards>> for some mitigation options.
+************
+
+[float]
+=== Basic read model
+
+Reads in Elasticsearch can be very lightweight lookups by ID or a heavy search request with complex aggregations that
+take non-trivial CPU power. One of the beauties of the primary-backup model is that it keeps all shard copies identical
+(with the exception of in-flight operations). As such, a single in-sync copy is sufficient to serve read requests.
+
+When a read request is received by a node, that node is responsible for forwarding it to the nodes that hold the relevant shards,
+collating the responses, and responding to the client. We call that node the _coordinating node_ for that request. The basic flow 
+is as follows:
+
+. Resolve the read requests to the relevant shards. Note that since most searches will be sent to one or more indices,
+   they typically need to read from multiple shards, each representing a different subset of the data.
+. Select an active copy of each relevant shard, from the shard replication group. This can be either the primary or
+   a replica. By default, Elasticsearch will simply round robin between the shard copies.
+. Send shard level read requests to the selected copies.
+. Combine the results and respond. Note that in the case of get by ID look up, only one shard is relevant and this step can be skipped.
+
+[float]
+==== Failure handling
+
+When a shard fails to respond to a read request, the coordinating node will select another copy from the same replication group
+and send the shard level search request to that copy instead. Repetitive failures can result in no shard copies being available.
+In some cases, such as `_search`, Elasticsearch will prefer to respond fast, albeit with partial results, instead of waiting 
+for the issue to be resolved (partial results are indicated in the `_shards` header of the response).
+
+[float]
+=== A few simple implications
+
+Each of these basic flows determines how Elasticsearch behaves as a system for both reads and writes. Furthermore, since read
+and write requests can be executed concurrently, these two basic flows interact with each other. This has a few inherent implications:
+
+Efficient reads:: Under normal operation each read operation is performed once for each relevant replication group.
+   Only under failure conditions do multiple copies of the same shard execute the same search.
+
+Read unacknowledged:: Since the primary first indexes locally and then replicates the request, it is possible for a
+  concurrent read to already see the change before it has been acknowledged.
+
+Two copies by default:: This model can be fault tolerant while maintaining only two copies of the data. This is in contrast to
+  quorum-based system where the minimum number of copies for fault tolerance is 3.
+
+[float]
+=== Failures
+
+Under failures, the following is possible:
+
+A single shard can slow down indexing:: Because the primary waits for all replicas in the in-sync copies set during each operation,
+  a single slow shard can slow down the entire replication group. This is the price we pay for the read efficiency mentioned above.
+  Of course a single slow shard will also slow down unlucky searches that have been routed to it.
+
+Dirty reads:: An isolated primary can expose writes that will not be acknowledged. This is caused by the fact that an isolated
+  primary will only realize that it is isolated once it sends requests to its replicas or when reaching out to the master.
+  At that point the operation is already indexed into the primary and can be read by a concurrent read. Elasticsearch mitigates
+  this risk by pinging the master every second (by default) and rejecting indexing operations if no master is known.
+
+[float]
+=== The Tip of the Iceberg
+
+This document provides a high level overview of how Elasticsearch deals with data. Of course, there is much much more 
+going on under the hood. Things like primary terms, cluster state publishing and master election all play a role in 
+keeping this system behaving correctly. This document also doesn't cover known and important
+bugs (both closed and open). We recognize that https://github.com/elastic/elasticsearch/issues?q=label%3Aresiliency[GitHub is hard to keep up with].
+To help people stay on top of those and we maintain a dedicated https://www.elastic.co/guide/en/elasticsearch/resiliency/current/index.html[resiliency page]
+on our website. We strongly advise reading it.
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index 7ccaf241ab3..bd75ce1694d 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -294,7 +294,7 @@ GET /_tasks/taskId:1
 
 The advantage of this API is that it integrates with `wait_for_completion=false`
 to transparently return the status of completed tasks. If the task is completed
-and `wait_for_completion=false` was set on it them it'll come back with a
+and `wait_for_completion=false` was set on it then it'll come back with 
 `results` or an `error` field. The cost of this feature is the document that
 `wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
 you to delete that document.
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index a1f868358eb..0e95c951cb3 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -891,6 +891,7 @@ Which results in a sensible `total` like this one:
 
 Adding `slices` to `_reindex` just automates the manual process used in the
 section above, creating sub-requests which means it has some quirks:
+
 * You can see these requests in the <<docs-reindex-task-api,Tasks APIs>>. These
 sub-requests are "child" tasks of the task for the request with `slices`.
 * Fetching the status of the task for the request with `slices` only contains
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 4b88cad7246..d6fb08a2adb 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -501,6 +501,7 @@ Which results in a sensible `total` like this one:
 
 Adding `slices` to `_update_by_query` just automates the manual process used in
 the section above, creating sub-requests which means it has some quirks:
+
 * You can see these requests in the
 <<docs-update-by-query-task-api,Tasks APIs>>. These sub-requests are "child"
 tasks of the task for the request with `slices`.
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index 0d2b564d1ca..130d95acb91 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -67,6 +67,7 @@ A document is a basic unit of information that can be indexed. For example, you
 
 Within an index/type, you can store as many documents as you want. Note that although a document physically resides in an index, a document actually must be indexed/assigned to a type inside an index.
 
+[[getting-started-shards-and-replicas]]
 [float]
 === Shards & Replicas
 
@@ -91,7 +92,7 @@ Replication is important for two primary reasons:
 
 
 To summarize, each index can be split into multiple shards. An index can also be replicated zero (meaning no replicas) or more times. Once replicated, each index will have primary shards (the original shards that were replicated from) and replica shards (the copies of the primary shards).
-The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number shards after-the-fact.
+The number of shards and replicas can be defined per index at the time the index is created. After the index is created, you may change the number of replicas dynamically anytime but you cannot change the number of shards after-the-fact.
 
 By default, each index in Elasticsearch is allocated 5 primary shards and 1 replica which means that if you have at least two nodes in your cluster, your index will have 5 primary shards and another 5 replica shards (1 complete replica) for a total of 10 shards per index.
 
@@ -215,6 +216,7 @@ And the response:
 epoch      timestamp cluster       status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
 1475247709 17:01:49  elasticsearch green           1         1      0   0    0    0        0             0                  -                100.0%
 --------------------------------------------------
+// TESTRESPONSE[s/0             0/0             [01]/]
 // TESTRESPONSE[s/1475247709 17:01:49  elasticsearch/\\d+ \\d+:\\d+:\\d+ docs_integTest/ _cat]
 
 We can see that our cluster named "elasticsearch" is up with a green status.
@@ -550,7 +552,7 @@ POST /customer/external/_bulk?pretty
 
 Note above that for the delete action, there is no corresponding source document after it since deletes only require the ID of the document to be deleted.
 
-The bulk API executes all the actions sequentially and in order. If a single action fails for whatever reason, it will continue to process the remainder of the actions after it. When the bulk API returns, it will provide a status for each action (in the same order it was sent in) so that you can check if a specific action failed or not.
+The Bulk API does not fail due to failures in one of the actions. If a single action fails for whatever reason, it will continue to process the remainder of the actions after it. When the bulk API returns, it will provide a status for each action (in the same order it was sent in) so that you can check if a specific action failed or not.
 
 == Exploring Your Data
 
@@ -940,7 +942,7 @@ In the above example, the `bool must_not` clause specifies a list of queries non
 
 We can combine `must`, `should`, and `must_not` clauses simultaneously inside a `bool` query. Furthermore, we can compose `bool` queries inside any of these `bool` clauses to mimic any complex multi-level boolean logic.
 
-This example returns all accounts of anybody who is 40 years old but don't live in ID(aho):
+This example returns all accounts of anybody who is 40 years old but doesn't live in ID(aho):
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc
index d709e17bb4e..accdb8e8915 100644
--- a/docs/reference/how-to.asciidoc
+++ b/docs/reference/how-to.asciidoc
@@ -4,7 +4,7 @@
 [partintro]
 --
 Elasticsearch ships with defaults which are intended to give a good out of
-the box experience. Full text search, highlighting, aggregations, indexing
+the box experience. Full text search, highlighting, aggregations, and indexing
 should all just work without the user having to change anything.
 
 Once you better understand how you want to use Elasticsearch, however,
diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc
index b0bd5fef802..6d7f66c2cd6 100644
--- a/docs/reference/how-to/indexing-speed.asciidoc
+++ b/docs/reference/how-to/indexing-speed.asciidoc
@@ -17,17 +17,17 @@ it is advisable to avoid going beyond a couple tens of megabytes per request
 even if larger requests seem to perform better.
 
 [float]
-=== Use multiple workers/threads to send data to elasticsearch
+=== Use multiple workers/threads to send data to Elasticsearch
 
 A single thread sending bulk requests is unlikely to be able to max out the
-indexing capacity of an elasticsearch cluster. In order to use all resources
+indexing capacity of an Elasticsearch cluster. In order to use all resources
 of the cluster, you should send data from multiple threads or processes. In
 addition to making better use of the resources of the cluster, this should
 help reduce the cost of each fsync.
 
 Make sure to watch for `TOO_MANY_REQUESTS (429)` response codes
 (`EsRejectedExecutionException` with the Java client), which is the way that
-elasticsearch tells you that it cannot keep up with the current indexing rate.
+Elasticsearch tells you that it cannot keep up with the current indexing rate.
 When it happens, you should pause indexing a bit before trying again, ideally
 with randomized exponential backoff.
 
@@ -39,7 +39,7 @@ number of workers until either I/O or CPU is saturated on the cluster.
 === Increase the refresh interval
 
 The default <<dynamic-index-settings,`index.refresh_interval`>> is `1s`, which
-forces elasticsearch to create a new segment every second.
+forces Elasticsearch to create a new segment every second.
 Increasing this value (to say, `30s`) will allow larger segments to flush and
 decreases future merge pressure.
 
diff --git a/docs/reference/how-to/recipes.asciidoc b/docs/reference/how-to/recipes.asciidoc
index 0bb158f88e8..4d1a4b67a2b 100644
--- a/docs/reference/how-to/recipes.asciidoc
+++ b/docs/reference/how-to/recipes.asciidoc
@@ -169,9 +169,9 @@ the query need to be matched exactly while other parts should still take
 stemming into account?
 
 Fortunately, the `query_string` and `simple_query_string` queries have a feature
-that allows to solve exactly this problem: `quote_field_suffix`. It allows to
-tell Elasticsearch that words that appear in between quotes should be redirected
-to a different field, see below:
+that solve this exact problem: `quote_field_suffix`. This tell Elasticsearch
+that the words that appear in between quotes are to be redirected to a different
+field, see below:
 
 [source,js]
 --------------------------------------------------
@@ -218,7 +218,7 @@ GET index/_search
 --------------------------------------------------
 // TESTRESPONSE[s/"took": 2,/"took": "$body.took",/]
 
-In that case, since `ski` was in-between quotes, it was searched on the
+In the above case, since `ski` was in-between quotes, it was searched on the
 `body.exact` field due to the `quote_field_suffix` parameter, so only document
 `1` matched. This allows users to mix exact search with stemmed search as they
 like.
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 28e9e6a114e..2d2b6600faf 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -79,6 +79,13 @@ Checking shards may take a lot of time on large indices.
     which uses https://en.wikipedia.org/wiki/DEFLATE[DEFLATE] for a higher
     compression ratio, at the expense of slower stored fields performance.
 
+[[routing-partition-size]] `index.routing_partition_size`::
+
+    The number of shards a custom <<mapping-routing-field,routing>> value can go to.
+    Defaults to 1 and can only be set at index creation time. This value must be less
+    than the `index.number_of_shards` unless the `index.number_of_shards` value is also 1.
+    See <<routing-index-partition>> for more details about how this setting is used.
+
 [float]
 [[dynamic-index-settings]]
 === Dynamic index settings
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 7a2e2628102..f4629c51b09 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -20,6 +20,27 @@ release-state can be: released | prerelease | unreleased
 :issue:          https://github.com/elastic/elasticsearch/issues/
 :pull:           https://github.com/elastic/elasticsearch/pull/
 
+
+///////
+Javadoc roots used to generate links from Painless's API reference
+///////
+:java8-javadoc: https://docs.oracle.com/javase/8/docs/api
+:java9-javadoc: http://download.java.net/java/jigsaw/docs/api
+:joda-time-javadoc: http://www.joda.org/joda-time/apidocs
+:lucene-core-javadoc: http://lucene.apache.org/core/6_4_0/core
+
+ifeval::["{release-state}"=="unreleased"]
+:elasticsearch-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/elasticsearch/{version}-SNAPSHOT
+:painless-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}-SNAPSHOT
+endif::[]
+
+ifeval::["{release-state}"!="unreleased"]
+:elasticsearch-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/elasticsearch/{version}
+:painless-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/painless/lang-painless/{version}
+endif::[]
+
+
+
 include::getting-started.asciidoc[]
 
 include::setup.asciidoc[]
@@ -62,4 +83,6 @@ include::glossary.asciidoc[]
  include::release-notes.asciidoc[]
 //////
 
+include::painless-api-reference.asciidoc[]
+
 include::redirects.asciidoc[]
diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index a04fe0ca3f6..11f3dac9567 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -135,7 +135,7 @@ what happened:
 // TESTRESPONSE
 
 `acknowledged` indicates whether the index was successfully created in the cluster, while 
-`shards_acknowledged` indices whether the requisite number of shard copies were started for
+`shards_acknowledged` indicates whether the requisite number of shard copies were started for
 each shard in the index before timing out. Note that it is still possible for either 
 `acknowledged` or `shards_acknowledged` to be `false`, but the index creation was successful. 
 These values simply indicate whether the operation completed before the timeout. If 
diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc
index 224e74605f4..d2f0d5a7508 100644
--- a/docs/reference/indices/get-field-mapping.asciidoc
+++ b/docs/reference/indices/get-field-mapping.asciidoc
@@ -5,34 +5,53 @@ The get field mapping API allows you to retrieve mapping definitions for one or
 This is useful when you do not need the complete type mapping returned by
 the <<indices-get-mapping>> API.
 
-The following returns the mapping of the field `text` only:
+For example, consider the following mapping:
 
 [source,js]
 --------------------------------------------------
-GET /twitter/_mapping/tweet/field/message
+PUT publications
+{
+    "mappings": {
+        "article": {
+            "properties": {
+                "id": { "type": "text" },
+                "title":  { "type": "text"},
+                "abstract": { "type": "text"},
+                "author": {
+                    "properties": {
+                        "id": { "type": "text" },
+                        "name": { "type": "text" }
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// TESTSETUP
+// CONSOLE
+
+The following returns the mapping of the field `title` only:
+
+[source,js]
+--------------------------------------------------
+GET publications/_mapping/article/field/title
 --------------------------------------------------
 // CONSOLE
-// TEST[setup:twitter]
 
-For which the response is (assuming `text` is a default string field):
+For which the response is:
 
 [source,js]
 --------------------------------------------------
 {
-   "twitter": {
+   "publications": {
       "mappings": {
-         "tweet": {
-            "message": {
-               "full_name": "message",
+         "article": {
+            "title": {
+               "full_name": "title",
                "mapping": {
-                  "message": {
-                     "type": "text",
-                     "fields": {
-                        "keyword": {
-                           "type": "keyword",
-                           "ignore_above": 256
-                        }
-                     }
+                  "title": {
+                     "type": "text"
                   }
                }
             }
@@ -43,7 +62,6 @@ For which the response is (assuming `text` is a default string field):
 --------------------------------------------------
 // TESTRESPONSE
 
-
 [float]
 === Multiple Indices, Types and Fields
 
@@ -69,46 +87,15 @@ GET /_all/_mapping/tw*/field/*.id
 [float]
 === Specifying fields
 
-The get mapping api allows you to specify one or more fields separated with by a comma.
-You can also use wildcards. The field names can be any of the following:
+The get mapping api allows you to specify a comma-separated list of fields.
 
-[horizontal]
-Full names:: the full path, including any parent object name the field is
-   part of (ex. `user.id`).
-Field names:: the name of the field without the path to it (ex. `id` for `{ "user" : { "id" : 1 } }`).
-
-The above options are specified in the order the `field` parameter is resolved.
-The first field found which matches is returned. This is especially important
-if index names or field names are used as those can be ambiguous.
-
-For example, consider the following mapping:
+For instance to select the `id` of the `author` field, you must use its full name `author.id`.
 
 [source,js]
 --------------------------------------------------
- {
-     "article": {
-         "properties": {
-             "id": { "type": "text" },
-             "title":  { "type": "text"},
-             "abstract": { "type": "text"},
-             "author": {
-                 "properties": {
-                     "id": { "type": "text" },
-                     "name": { "type": "text" }
-                 }
-             }
-         }
-     }
- }
---------------------------------------------------
-
-To select the `id` of the `author` field, you can use its full name `author.id`. `name` will return
-the field `author.name`:
-
-[source,js]
---------------------------------------------------
-curl -XGET "http://localhost:9200/publications/_mapping/article/field/author.id,abstract,name"
+GET publications/_mapping/article/field/author.id,abstract,name
 --------------------------------------------------
+// CONSOLE
 
 returns:
 
@@ -116,33 +103,77 @@ returns:
 --------------------------------------------------
 {
    "publications": {
-      "article": {
-         "abstract": {
-            "full_name": "abstract",
-            "mapping": {
-               "abstract": { "type": "text" }
-            }
-         },
-         "author.id": {
-            "full_name": "author.id",
-            "mapping": {
-               "id": { "type": "text" }
-            }
-         },
-         "name": {
-            "full_name": "author.name",
-            "mapping": {
-               "name": { "type": "text" }
+      "mappings": {
+         "article": {
+            "author.id": {
+               "full_name": "author.id",
+               "mapping": {
+                  "id": {
+                     "type": "text"
+                  }
+               }
+            },
+            "abstract": {
+               "full_name": "abstract",
+               "mapping": {
+                  "abstract": {
+                     "type": "text"
+                  }
+               }
             }
          }
       }
    }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
-Note how the response always use the same fields specified in the request as keys.
-The `full_name` in every entry contains the full name of the field whose mapping were returned.
-This is useful when the request can refer to to multiple fields.
+The get field mapping API also supports wildcard notation.
+
+[source,js]
+--------------------------------------------------
+GET publications/_mapping/article/field/a*
+--------------------------------------------------
+// CONSOLE
+
+returns:
+
+[source,js]
+--------------------------------------------------
+{
+   "publications": {
+      "mappings": {
+         "article": {
+            "author.name": {
+               "full_name": "author.name",
+               "mapping": {
+                  "name": {
+                     "type": "text"
+                  }
+               }
+            },
+            "abstract": {
+               "full_name": "abstract",
+               "mapping": {
+                  "abstract": {
+                     "type": "text"
+                  }
+               }
+            },
+            "author.id": {
+               "full_name": "author.id",
+               "mapping": {
+                  "id": {
+                     "type": "text"
+                  }
+               }
+            }
+         }
+      }
+   }
+}
+--------------------------------------------------
+// TESTRESPONSE
 
 [float]
 === Other options
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index db78104be12..9ae8c72a93c 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -108,6 +108,8 @@ PUT logs_write/log/1
   "message": "a dummy log"
 }
 
+POST logs_write/_refresh
+
 # Wait for a day to pass
 
 POST /logs_write/_rollover <2>
diff --git a/docs/reference/indices/shadow-replicas.asciidoc b/docs/reference/indices/shadow-replicas.asciidoc
index 3a0b23852b0..625165d5bdd 100644
--- a/docs/reference/indices/shadow-replicas.asciidoc
+++ b/docs/reference/indices/shadow-replicas.asciidoc
@@ -1,7 +1,7 @@
 [[indices-shadow-replicas]]
 == Shadow replica indices
 
-experimental[]
+deprecated[5.2.0, Shadow replicas don't see much usage and we are planning to remove them]
 
 If you would like to use a shared filesystem, you can use the shadow replicas
 settings to choose where on disk the data for an index should be kept, as well
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 3cdcfd5d2cd..d6acabd87f1 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -143,7 +143,7 @@ This should give a small response that makes it both easy and inexpensive to par
 [[delete-pipeline-api]]
 === Delete Pipeline API
 
-The delete pipeline API deletes pipelines by ID.
+The delete pipeline API deletes pipelines by ID or wildcard match (`my-*`, `*`).
 
 [source,js]
 --------------------------------------------------
@@ -152,6 +152,36 @@ DELETE _ingest/pipeline/my-pipeline-id
 // CONSOLE
 // TEST[continued]
 
+////
+Hidden setup for wildcard test:
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/wild-one
+{
+  "description" : "first pipeline to be wildcard deleted",
+  "processors" : [ ]
+}
+
+PUT _ingest/pipeline/wild-two
+{
+  "description" : "second pipeline to be wildcard deleted",
+  "processors" : [ ]
+}
+
+DELETE _ingest/pipeline/*
+--------------------------------------------------
+// CONSOLE
+
+Hidden expected response:
+[source,js]
+--------------------------------------------------
+{
+"acknowledged": true
+}
+--------------------------------------------------
+// TESTRESPONSE
+////
+
 [[simulate-pipeline-api]]
 === Simulate Pipeline API
 
@@ -1136,6 +1166,10 @@ In this example, if the `remove` processor does fail, then
 the array elements that have been processed thus far will
 be updated.
 
+Another advanced example can be found in the {plugins}/ingest-attachment-with-arrays.html[attachment processor documentation].
+
+
+
 [[grok-processor]]
 === Grok Processor
 
@@ -1485,6 +1519,38 @@ Converts a JSON string into a structured JSON object.
 }
 --------------------------------------------------
 
+[[kv-processor]]
+=== KV Processor
+This processor helps automatically parse messages (or specific event fields) which are of the foo=bar variety.
+
+For example, if you have a log message which contains `ip=1.2.3.4 error=REFUSED`, you can parse those automatically by configuring:
+
+
+[source,js]
+--------------------------------------------------
+{
+  "kv": {
+    "field": "message",
+    "field_split": " ",
+    "value_split": "="
+  }
+}
+--------------------------------------------------
+
+[[kv-options]]
+.Kv Options
+[options="header"]
+|======
+| Name             | Required  | Default  | Description
+| `field`          | yes       | -        | The field to be parsed
+| `field_split`    | yes       | -        | Regex pattern to use for splitting key-value pairs
+| `value_split`    | yes       | -        | Regex pattern to use for splitting the key from the value within a key-value pair
+| `target_field`   | no        | `null`   | The field to insert the extracted keys into. Defaults to the root of the document
+| `include_keys`   | no        | `null`   | List of keys to filter and insert into document. Defaults to including all keys
+| `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+|======
+
+
 [[lowercase-processor]]
 === Lowercase Processor
 Converts a string to its lowercase equivalent.
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index dffb4f94ff5..a5cbb452220 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -155,23 +155,21 @@ PUT my_index <1>
 {
   "mappings": {
     "user": { <2>
-      "_all":       { "enabled": false  }, <3>
-      "properties": { <4>
-        "title":    { "type": "text"  }, <5>
-        "name":     { "type": "text"  }, <5>
-        "age":      { "type": "integer" }  <5>
+      "properties": { <3>
+        "title":    { "type": "text"  }, <4>
+        "name":     { "type": "text"  }, <4>
+        "age":      { "type": "integer" }  <4>
       }
     },
     "blogpost": { <2>
-      "_all":       { "enabled": false  }, <3>
-      "properties": { <4>
-        "title":    { "type": "text"  }, <5>
-        "body":     { "type": "text"  }, <5>
+      "properties": { <3>
+        "title":    { "type": "text"  }, <4>
+        "body":     { "type": "text"  }, <4>
         "user_id":  {
-          "type":   "keyword" <5>
+          "type":   "keyword" <4>
         },
         "created":  {
-          "type":   "date", <5>
+          "type":   "date", <4>
           "format": "strict_date_optional_time||epoch_millis"
         }
       }
@@ -182,9 +180,8 @@ PUT my_index <1>
 // CONSOLE
 <1> Create an index called `my_index`.
 <2> Add mapping types called `user` and `blogpost`.
-<3> Disable the `_all` <<mapping-fields,meta field>> for the `user` mapping type.
-<4> Specify fields or _properties_ in each mapping type.
-<5> Specify the data `type` and mapping for each field.
+<3> Specify fields or _properties_ in each mapping type.
+<4> Specify the data `type` and mapping for each field.
 
 
 --
diff --git a/docs/reference/mapping/dynamic/default-mapping.asciidoc b/docs/reference/mapping/dynamic/default-mapping.asciidoc
index de96667b7e3..926c66ced51 100644
--- a/docs/reference/mapping/dynamic/default-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/default-mapping.asciidoc
@@ -14,13 +14,13 @@ PUT my_index
 {
   "mappings": {
     "_default_": { <1>
-      "_all": {
+      "_source": {
         "enabled": false
       }
     },
     "user": {}, <2>
     "blogpost": { <3>
-      "_all": {
+      "_source": {
         "enabled": true
       }
     }
@@ -28,9 +28,9 @@ PUT my_index
 }
 --------------------------------------------------
 // CONSOLE
-<1> The `_default_` mapping defaults the <<mapping-all-field,`_all`>> field to disabled.
+<1> The `_default_` mapping defaults the <<mapping-source-field,`_source`>> field to disabled.
 <2> The `user` type inherits the settings from `_default_`.
-<3> The `blogpost` type overrides the defaults and enables the <<mapping-all-field,`_all`>> field.
+<3> The `blogpost` type overrides the defaults and enables the <<mapping-source-field,`_source`>> field.
 
 NOTE: When updating the `_default_` mapping with the
 <<indices-put-mapping,PUT mapping>> API, the new mapping is not merged with
@@ -53,7 +53,7 @@ PUT _template/logging
   "settings": { "number_of_shards": 1 }, <2>
   "mappings": {
     "_default_": {
-      "_all": { <3>
+      "_field_names": { <3>
         "enabled": false
       },
       "dynamic_templates": [
@@ -82,5 +82,5 @@ PUT logs-2015.10.01/event/1
 // CONSOLE
 <1> The `logging` template will match any indices beginning with `logs-`.
 <2> Matching indices will be created with a single primary shard.
-<3> The `_all` field will be disabled by default for new type mappings.
+<3> The `_field_names` field will be disabled by default for new type mappings.
 <4> String fields will be created with a `text` main field, and a `keyword` `.raw` field.
diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index 7bed12b5b78..2b0afa1562d 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -4,7 +4,8 @@
 By default, when a previously unseen field is found in a document,
 Elasticsearch will add the new field to the type mapping.   This behaviour can
 be disabled, both at the document and at the <<object,`object`>>  level, by
-setting the <<dynamic,`dynamic`>> parameter to `false` or to `strict`.
+setting the <<dynamic,`dynamic`>> parameter to `false` (to ignore new fields) or to `strict` (to throw
+an exception if an unknown field is encountered).
 
 Assuming `dynamic` field mapping is enabled, some simple rules are used to
 determine which datatype the field should have:
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index dde3defff67..eb9eb1a60d9 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -399,18 +399,18 @@ You can override the default mappings for all indices and all types
 by specifying a `_default_` type mapping in an index template
 which matches all indices.
 
-For example, to disable the `_all` field by default for all types in all
+For example, to disable the `_field_names` field by default for all types in all
 new indices, you could create the following index template:
 
 [source,js]
 --------------------------------------------------
-PUT _template/disable_all_field
+PUT _template/disable_field_names
 {
   "order": 0,
   "index_patterns": ["*"], <1>
   "mappings": {
     "_default_": { <2>
-      "_all": { <3>
+      "_field_names": { <3>
         "enabled": false
       }
     }
@@ -421,4 +421,4 @@ PUT _template/disable_all_field
 <1> Applies the mappings to an `index` which matches the pattern `*`, in other
     words, all new indices.
 <2> Defines the `_default_` type mapping types within the index.
-<3> Disables the `_all` field by default.
+<3> Disables the `_field_names` field by default.
diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc
index 8bd86cd0085..cc803ef5666 100644
--- a/docs/reference/mapping/fields.asciidoc
+++ b/docs/reference/mapping/fields.asciidoc
@@ -42,7 +42,7 @@ can be customised when a mapping type is created.
 
 <<mapping-all-field,`_all`>>::
 
-    A _catch-all_  field that indexes the values of all other fields.
+    A _catch-all_  field that indexes the values of all other fields. Disabled by default.
 
 <<mapping-field-names-field,`_field_names`>>::
 
diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc
index aa957247a3f..18d7d5ec5df 100644
--- a/docs/reference/mapping/fields/all-field.asciidoc
+++ b/docs/reference/mapping/fields/all-field.asciidoc
@@ -1,6 +1,8 @@
 [[mapping-all-field]]
 === `_all` field
 
+deprecated[6.0.0, `_all` may no longer be enabled for indices created in 6.0+, use a custom field and the mapping `copy_to` parameter]
+
 The `_all` field is a special _catch-all_ field which concatenates the values
 of all of the other fields into one big string, using space as a delimiter, which is then
 <<analysis,analyzed>> and indexed, but not stored.  This means that it can be
@@ -12,24 +14,37 @@ started with a new dataset. For instance:
 
 [source,js]
 --------------------------------
-PUT my_index/user/1 <1>
+PUT /my_index
+{
+  "mapping": {
+    "user": {
+      "_all": {
+        "enabled": true   <1>
+      }
+    }
+  }
+}
+
+PUT /my_index/user/1      <2>
 {
   "first_name":    "John",
   "last_name":     "Smith",
   "date_of_birth": "1970-10-24"
 }
 
-GET my_index/_search
+GET /my_index/_search
 {
   "query": {
     "match": {
-      "_all": "john smith new york"
+      "_all": "john smith 1970"
     }
   }
 }
 --------------------------------
+// TEST[skip:_all is no longer allowed]
 // CONSOLE
-<1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"1970"`, `"10"`, `"24"` ]
+<1> Enabling the `_all` field
+<2> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"1970"`, `"10"`, `"24"` ]
 
 [NOTE]
 .All values treated as strings
@@ -56,15 +71,16 @@ and long fields (less relevant). For use cases where search relevance is
 important, it is better to query individual fields specifically.
 
 The `_all` field is not free: it requires extra CPU cycles and uses more disk
-space. If not needed, it can be completely <<disabling-all-field,disabled>> or
-customised on a <<include-in-all,per-field basis>>.
+space. For this reason, it is disabled by default. If not needed, it can be
+<<enabling-all-field,enabled>> or customised on a <<include-in-all,per-field
+basis>>.
 
 [[querying-all-field]]
 ==== Using the `_all` field in queries
 
 The <<query-dsl-query-string-query,`query_string`>> and
-<<query-dsl-simple-query-string-query,`simple_query_string`>> queries query
-the `_all` field by default, unless another field is specified:
+<<query-dsl-simple-query-string-query,`simple_query_string`>> queries query the
+`_all` field by default if it is enabled, unless another field is specified:
 
 [source,js]
 --------------------------------
@@ -88,15 +104,13 @@ GET _search?q=john+smith+new+york
 --------------------------------
 
 Other queries, such as the <<query-dsl-match-query,`match`>> and
-<<query-dsl-term-query,`term`>> queries require you to specify
-the `_all` field explicitly, as per the
-<<mapping-all-field,first example>>.
+<<query-dsl-term-query,`term`>> queries require you to specify the `_all` field
+explicitly, as per the <<mapping-all-field,first example>>.
 
-[[disabling-all-field]]
-==== Disabling the `_all` field
+[[enabling-all-field]]
+==== Enabling the `_all` field
 
-The `_all` field can be completely disabled per-type by setting `enabled` to
-`false`:
+The `_all` field can be enabled per-type by setting `enabled` to `true`:
 
 [source,js]
 --------------------------------
@@ -108,23 +122,24 @@ PUT my_index
     },
     "type_2": { <2>
       "_all": {
-        "enabled": false
+        "enabled": true
       },
       "properties": {...}
     }
   }
 }
 --------------------------------
-// CONSOLE
 // TEST[s/\.\.\.//]
+// TEST[skip:_all is no longer allowed]
+// CONSOLE
 
-<1> The `_all` field in `type_1` is enabled.
-<2> The `_all` field in `type_2` is completely disabled.
+<1> The `_all` field in `type_1` is disabled.
+<2> The `_all` field in `type_2` is enabled.
 
-If the `_all` field is disabled, then URI search requests and the
-`query_string` and `simple_query_string` queries will not be able to use it
-for queries (see <<querying-all-field>>).  You can configure them to use a
-different field with the `index.query.default_field` setting:
+If the `_all` field is enabled, then URI search requests and the `query_string`
+and `simple_query_string` queries can automatically use it for queries (see
+<<querying-all-field>>). You can configure them to use a different field with
+the `index.query.default_field` setting:
 
 [source,js]
 --------------------------------
@@ -132,9 +147,6 @@ PUT my_index
 {
   "mappings": {
     "my_type": {
-      "_all": {
-        "enabled": false <1>
-      },
       "properties": {
         "content": {
           "type": "text"
@@ -143,14 +155,13 @@ PUT my_index
     }
   },
   "settings": {
-    "index.query.default_field": "content" <2>
+    "index.query.default_field": "content" <1>
   }
 }
 --------------------------------
 // CONSOLE
 
-<1> The `_all` field is disabled for the `my_type` type.
-<2> The `query_string` query will default to querying the `content` field in this index.
+<1> The `query_string` query will default to querying the `content` field in this index.
 
 [[excluding-from-all]]
 ==== Excluding fields from `_all`
@@ -171,6 +182,7 @@ PUT myindex
 {
   "mappings": {
     "mytype": {
+      "_all": {"enabled": true},
       "properties": {
         "title": { <1>
           "type": "text",
@@ -184,6 +196,7 @@ PUT myindex
   }
 }
 --------------------------------
+// TEST[skip:_all is no longer allowed]
 // CONSOLE
 
 <1> When querying the `_all` field, words that originated in the
@@ -252,8 +265,8 @@ A field can only be used for <<search-request-highlighting,highlighting>>  if
 the original string value is available, either from the
 <<mapping-source-field,`_source`>>  field or as a stored field.
 
-The `_all` field is not present in the `_source` field and it is not stored by
-default, and so cannot be highlighted. There are two options. Either
+The `_all` field is not present in the `_source` field and it is not stored or
+enabled by default, and so cannot be highlighted. There are two options. Either
 <<all-field-store,store the `_all` field>> or highlight the
 <<all-highlight-fields,original fields>>.
 
@@ -270,6 +283,7 @@ PUT myindex
   "mappings": {
     "mytype": {
       "_all": {
+        "enabled": true,
         "store": true
       }
     }
@@ -296,11 +310,12 @@ GET _search
   }
 }
 --------------------------------
+// TEST[skip:_all is no longer allowed]
 // CONSOLE
 
-Of course, storing the `_all` field will use significantly more disk space
-and, because it is a combination of other fields, it may result in odd
-highlighting results.
+Of course, enabling and storing the `_all` field will use significantly more
+disk space and, because it is a combination of other fields, it may result in
+odd highlighting results.
 
 The `_all` field also accepts the `term_vector` and `index_options`
 parameters, allowing the use of the fast vector highlighter and the postings
@@ -317,7 +332,7 @@ PUT myindex
 {
   "mappings": {
     "mytype": {
-      "_all": {}
+      "_all": {"enabled": true}
     }
   }
 }
@@ -344,6 +359,7 @@ GET _search
   }
 }
 --------------------------------
+// TEST[skip:_all is no longer allowed]
 // CONSOLE
 
 <1> The query inspects the `_all` field to find matching documents.
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index eddf9ef3a25..9197c7184ea 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -59,16 +59,18 @@ See the <<query-dsl-has-child-query,`has_child`>> and
 the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation,
 and <<parent-child-inner-hits,inner hits>> for more information.
 
-The value of the `_parent` field is accessible in queries, aggregations,
-and scripts:
+The value of the `_parent` field is accessible in aggregations
+and scripts, and may be queried with the
+<<query-dsl-parent-id-query, `parent_id` query>>:
 
 [source,js]
 --------------------------
 GET my_index/_search
 {
   "query": {
-    "terms": {
-      "_parent": [ "1" ] <1>
+    "parent_id": { <1>
+      "type": "my_child",
+      "id": "1"
     }
   },
   "aggs": {
@@ -82,7 +84,6 @@ GET my_index/_search
   "script_fields": {
     "parent": {
       "script": {
-         "lang": "painless",
          "inline": "doc['_parent']" <3>
       }
     }
@@ -92,7 +93,7 @@ GET my_index/_search
 // CONSOLE
 // TEST[continued]
 
-<1> Querying on the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
+<1> Querying the id of the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
 <2> Aggregating on the `_parent` field (also see the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation)
 <3> Accessing the `_parent` field in scripts
 
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index bbdef3370c4..ebb659af344 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -109,3 +109,29 @@ documents with the same `_id` might end up on different shards if indexed with
 different `_routing` values.
 
 It is up to the user to ensure that IDs are unique across the index.
+
+[[routing-index-partition]]
+==== Routing to an index partition
+
+An index can be configured such that custom routing values will go to a subset of the shards rather
+than a single shard. This helps mitigate the risk of ending up with an imbalanced cluster while still
+reducing the impact of searches.
+
+This is done by providing the index level setting <<routing-partition-size,`index.routing_partition_size`>> at index creation.
+As the partition size increases, the more evenly distributed the data will become at the
+expense of having to search more shards per request.
+
+When this setting is present, the formula for calculating the shard becomes:
+
+    shard_num = (hash(_routing) + hash(_id) % routing_partition_size) % num_primary_shards
+
+That is, the `_routing` field is used to calculate a set of shards within the index and then the
+`_id` is used to pick a shard within that set.
+
+To enable this feature, the `index.routing_partition_size` should have a value greater than 1 and
+less than `index.number_of_shards`.
+
+Once enabled, the partitioned index will have the following limitations:
+
+*   Mappings with parent-child relationships cannot be created within it.
+*   All mappings within the index must have the `_routing` field marked as required.
\ No newline at end of file
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index a84b2508de4..07f62e016e9 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -21,7 +21,7 @@ PUT my_index/type_2/2?refresh=true
   "text": "Document with type 2"
 }
 
-GET my_index/type_*/_search
+GET my_index/_search
 {
   "query": {
     "terms": {
diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc
index e7d2d7ac0c8..24220356233 100644
--- a/docs/reference/mapping/params.asciidoc
+++ b/docs/reference/mapping/params.asciidoc
@@ -8,6 +8,7 @@ parameters that are used by <<mapping-types,field mappings>>:
 The following mapping parameters are common to some or all field datatypes:
 
 * <<analyzer,`analyzer`>>
+* <<normalizer, `normalizer`>>
 * <<mapping-boost,`boost`>>
 * <<coerce,`coerce`>>
 * <<copy-to,`copy_to`>>
@@ -34,6 +35,8 @@ The following mapping parameters are common to some or all field datatypes:
 
 include::params/analyzer.asciidoc[]
 
+include::params/normalizer.asciidoc[]
+
 include::params/boost.asciidoc[]
 
 include::params/coerce.asciidoc[]
diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc
index d9d6bbc9afc..afe325310c7 100644
--- a/docs/reference/mapping/params/doc-values.asciidoc
+++ b/docs/reference/mapping/params/doc-values.asciidoc
@@ -44,8 +44,3 @@ PUT my_index
 <1> The `status_code` field has `doc_values` enabled by default.
 <2> The `session_id` has `doc_values` disabled, but can still be queried.
 
-TIP: The `doc_values` setting is allowed to have different settings for fields
-of the same name in the same index.  It can be disabled (set to `false`) on
-existing fields using the <<indices-put-mapping,PUT mapping API>>.
-
-
diff --git a/docs/reference/mapping/params/index.asciidoc b/docs/reference/mapping/params/index.asciidoc
index e097293d142..270c930e284 100644
--- a/docs/reference/mapping/params/index.asciidoc
+++ b/docs/reference/mapping/params/index.asciidoc
@@ -4,3 +4,4 @@
 The `index` option controls whether field values are indexed. It accepts `true`
 or `false`. Fields that are not indexed are not queryable.
 
+NOTE: For the legacy mapping type <<string,`string`>> the `index` option only accepts legacy values `analyzed` (default, treat as full-text field), `not_analyzed` (treat as keyword field) and `no`.
diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc
new file mode 100644
index 00000000000..c0636763e52
--- /dev/null
+++ b/docs/reference/mapping/params/normalizer.asciidoc
@@ -0,0 +1,163 @@
+[[normalizer]]
+=== `normalizer`
+
+The `normalizer` property of <<keyword,`keyword`>> fields is similar to
+<<analyzer,`analyzer`>> except that it guarantees that the analysis chain
+produces a single token.
+
+The `normalizer` is applied prior to indexing the keyword, as well as at
+search-time when the `keyword` field is searched via a query parser such as
+the <<query-dsl-match-query,`match`>> query.
+
+[source,js]
+--------------------------------
+PUT index
+{
+  "settings": {
+    "analysis": {
+      "normalizer": {
+        "my_normalizer": {
+          "type": "custom",
+          "char_filter": [],
+          "filter": ["lowercase", "asciifolding"]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword",
+          "normalizer": "my_normalizer"
+        }
+      }
+    }
+  }
+}
+
+PUT index/type/1
+{
+  "foo": "BÀR"
+}
+
+PUT index/type/2
+{
+  "foo": "bar"
+}
+
+PUT index/type/3
+{
+  "foo": "baz"
+}
+
+POST index/_refresh
+
+GET index/_search
+{
+  "query": {
+    "match": {
+      "foo": "BAR"
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+
+The above query matches documents 1 and 2 since `BÀR` is converted to `bar` at
+both index and query time.
+
+[source,js]
+----------------------------
+{
+  "took": $body.took,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 2,
+    "max_score": 0.2876821,
+    "hits": [
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "2",
+        "_score": 0.2876821,
+        "_source": {
+          "foo": "bar"
+        }
+      },
+      {
+        "_index": "index",
+        "_type": "type",
+        "_id": "1",
+        "_score": 0.2876821,
+        "_source": {
+          "foo": "BÀR"
+        }
+      }
+    ]
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
+
+Also, the fact that keywords are converted prior to indexing also means that
+aggregations return normalized values:
+
+[source,js]
+----------------------------
+GET index/_search
+{
+  "size": 0,
+  "aggs": {
+    "foo_terms": {
+      "terms": {
+        "field": "foo"
+      }
+    }
+  }
+}
+--------------------------------
+// CONSOLE
+// TEST[continued]
+
+returns
+
+[source,js]
+----------------------------
+{
+  "took": 43,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 3,
+    "max_score": 0.0,
+    "hits": []
+  },
+  "aggregations": {
+    "foo_terms": {
+      "doc_count_error_upper_bound": 0,
+      "sum_other_doc_count": 0,
+      "buckets": [
+        {
+          "key": "bar",
+          "doc_count": 2
+        },
+        {
+          "key": "baz",
+          "doc_count": 1
+        }
+      ]
+    }
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index d273dddb27d..5f521922655 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -2,16 +2,16 @@
 === Boolean datatype
 
 Boolean fields accept JSON `true` and `false` values, but can also accept
-strings and numbers which are interpreted as either true or false:
+strings which are interpreted as either true or false:
 
 [horizontal]
 False values::
 
-    `false`, `"false"`, `"off"`, `"no"`, `"0"`, `""` (empty string), `0`, `0.0`
+    `false`, `"false"`
 
 True values::
 
-    Anything that isn't false.
+    `true`, `"true"`
 
 For example:
 
@@ -32,7 +32,7 @@ PUT my_index
 
 POST my_index/my_type/1
 {
-  "is_published": 1 <1>
+  "is_published": "true" <1>
 }
 
 GET my_index/_search
@@ -45,7 +45,7 @@ GET my_index/_search
 }
 --------------------------------------------------
 // CONSOLE
-<1> Indexing a document with `1`, which is interpreted as `true`.
+<1> Indexing a document with `"true"`, which is interpreted as `true`.
 <2> Searching for documents with a JSON `true`.
 
 Aggregations like the <<search-aggregations-bucket-terms-aggregation,`terms`
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index d0a889f15d9..691387ba8ab 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -74,7 +74,7 @@ PUT my_index
       "properties": {
         "date": {
           "type":   "date",
-          "format": "yyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
+          "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
         }
       }
     }
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index 7c09ef46e55..316c92a73e6 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -109,6 +109,12 @@ The following parameters are accepted by `keyword` fields:
     Which scoring algorithm or _similarity_ should be used. Defaults
     to `classic`, which uses TF/IDF.
 
+<<normalizer,`normalizer`>>::
+
+    experimental[]
+    How to pre-process the keyword prior to indexing. Defaults to `null`,
+    meaning the keyword is kept as-is.
+
 NOTE: Indexes imported from 2.x do not support `keyword`. Instead they will
 attempt to downgrade `keyword` into `string`. This allows you to merge modern
 mappings with legacy mappings. Long lived indexes will have to be recreated
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 6fdc0c806a4..48a74cd1760 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -39,6 +39,12 @@ PUT my_index
 --------------------------------------------------
 // CONSOLE
 
+NOTE: The `double`, `float` and `half_float` types consider that `-0.0` and
+`+0.0` are different values. As a consequence, doing a `term` query on
+`-0.0` will not match `+0.0` and vice-versa. Same is true for range queries:
+if the upper bound is `-0.0` then `+0.0` will not match, and if the lower
+bound is `+0.0` then `-0.0` will not match.
+
 ==== Which type should I use?
 
 As far as integer types (`byte`, `short`, `integer` and `long`) are concerned,
diff --git a/docs/reference/mapping/types/string.asciidoc b/docs/reference/mapping/types/string.asciidoc
index b6a5fef86eb..c4be060cc40 100644
--- a/docs/reference/mapping/types/string.asciidoc
+++ b/docs/reference/mapping/types/string.asciidoc
@@ -12,7 +12,7 @@ to use `text` or `keyword`.
 
 Indexes imported from 2.x *only* support `string` and not `text` or `keyword`.
 To ease the migration from 2.x Elasticsearch will downgrade `text` and `keyword`
-mappings applied to indexes imported to 2.x into `string`. While long lived
+mappings applied to indexes imported from 2.x into `string`. While long lived
 indexes will eventually need to be recreated against 5.x before eventually
 upgrading to 6.x, this downgrading smooths the process before you find time for
 it.
diff --git a/docs/reference/migration/migrate_6_0.asciidoc b/docs/reference/migration/migrate_6_0.asciidoc
index abc476a7d1b..2bfe5d88f2d 100644
--- a/docs/reference/migration/migrate_6_0.asciidoc
+++ b/docs/reference/migration/migrate_6_0.asciidoc
@@ -36,6 +36,7 @@ way to reindex old indices is to use the `reindex` API.
 * <<breaking_60_indices_changes>>
 * <<breaking_60_scripting_changes>>
 * <<breaking_60_ingest_changes>>
+* <<breaking_60_percolator_changes>>
 
 include::migrate_6_0/cat.asciidoc[]
 
@@ -60,3 +61,5 @@ include::migrate_6_0/indices.asciidoc[]
 include::migrate_6_0/scripting.asciidoc[]
 
 include::migrate_6_0/ingest.asciidoc[]
+
+include::migrate_6_0/percolator.asciidoc[]
diff --git a/docs/reference/migration/migrate_6_0/indices.asciidoc b/docs/reference/migration/migrate_6_0/indices.asciidoc
index be726ce155a..7062ac7cb1e 100644
--- a/docs/reference/migration/migrate_6_0/indices.asciidoc
+++ b/docs/reference/migration/migrate_6_0/indices.asciidoc
@@ -27,3 +27,8 @@ PUT _template/template_2
 }
 --------------------------------------------------
 // CONSOLE
+
+
+=== Shadow Replicas are deprecated
+
+<<indices-shadow-replicas,Shadow Replicas>> don't see much usage and we are planning to remove them.
diff --git a/docs/reference/migration/migrate_6_0/mappings.asciidoc b/docs/reference/migration/migrate_6_0/mappings.asciidoc
index 97584aa1903..b5fa55fdd5c 100644
--- a/docs/reference/migration/migrate_6_0/mappings.asciidoc
+++ b/docs/reference/migration/migrate_6_0/mappings.asciidoc
@@ -1,6 +1,23 @@
 [[breaking_60_mappings_changes]]
 === Mapping changes
 
+==== Coercion of boolean fields
+
+Previously, Elasticsearch recognized the strings `true`, `false`, `on`, `off`, `yes`, `no`, `0`, `1` as booleans. Elasticsearch 6.0
+recognizes only `true` and `false` as boolean and will throw an error otherwise. For backwards compatibility purposes, during the 6.x
+series the previous coercion rules will continue to work on pre-6.0 indices. This means that you do not need to change affected existing
+mappings immediately. However, it is not possible to create new indices from existing index templates that violate the strict `boolean`
+coercion rules.
+
+==== The `_all` meta field is now disabled by default
+
+On new mappings, the `_all` meta field that contains a copy of the text from
+each field is now disabled by default. The `query_string` and
+`simple_query_string` queries that previously used `_all` to search will now
+check if `_all` is enabled/disabled and switch to executing the query across all
+fields if `_all` is disabled. `_all` can no longer be configured for indices
+created with Elasticsearch version 6.0 or later.
+
 ==== Unrecognized `match_mapping_type` options not silently ignored
 
 Previously Elastiscearch would silently ignore any dynamic templates that
diff --git a/docs/reference/migration/migrate_6_0/percolator.asciidoc b/docs/reference/migration/migrate_6_0/percolator.asciidoc
new file mode 100644
index 00000000000..d31a1857ce9
--- /dev/null
+++ b/docs/reference/migration/migrate_6_0/percolator.asciidoc
@@ -0,0 +1,6 @@
+[[breaking_60_percolator_changes]]
+=== Percolator changes
+
+==== Deprecated percolator and mpercolate apis have been removed
+
+Instead the `percolate` query should be used via either the search or msearch apis.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_6_0/plugins.asciidoc b/docs/reference/migration/migrate_6_0/plugins.asciidoc
index bf73dc10e94..be650a71bd0 100644
--- a/docs/reference/migration/migrate_6_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_6_0/plugins.asciidoc
@@ -5,3 +5,23 @@
 
 * The mapper attachments plugin has been deprecated in elasticsearch 5.0 and is now removed.
 You can use {plugins}/ingest-attachment.html[ingest attachment plugin] instead.
+
+==== S3 Repository plugin
+
+* The bucket an s3 repository is configured with will no longer be created automatically.
+It must exist before the s3 repository is created.
+
+* Support for specifying s3 credentials through environment variables and
+system properties has been removed. Use the `elasticsearch-keystore` tool
+to securely store the credentials.
+
+* Specifying region has been removed. This includes the settings `cloud.aws.region`,
+`cloud.aws.s3.region`, `repositories.s3.region`, and specifying
+region inside the repository settings. Instead, specify the full endpoint if a custom
+s3 location is needed, or rely on the default behavior which automatically locates
+the region of the configured bucket.
+
+==== Azure Repository plugin
+
+* The container an azure repository is configured with will no longer be created automatically.
+It must exist before the azure repository is created.
diff --git a/docs/reference/migration/migrate_6_0/rest.asciidoc b/docs/reference/migration/migrate_6_0/rest.asciidoc
index dcbc289300e..f7b52b914da 100644
--- a/docs/reference/migration/migrate_6_0/rest.asciidoc
+++ b/docs/reference/migration/migrate_6_0/rest.asciidoc
@@ -13,6 +13,17 @@ has been removed in Elasticsearch 6.0.0.
 In previous versions of Elasticsearch, documents were allowed to contain duplicate keys. Elasticsearch 6.0.0
  enforces that all keys are unique. This applies to all content types: JSON, CBOR, Yaml and Smile.
 
+==== Content-Type Auto-detection
+
+In previous versions of Elasticsearch, having a proper Content-Type for the data in a request was not enforced.
+Elasticsearch 6.0.0 enforces that all requests with a body must have a supported Content-Type and this type will
+be used when parsing the data.
+
+==== Boolean API parameters
+
+All REST APIs parameters (both request parameters and JSON body) support providing boolean "false" as the
+value `false` and boolean "true" as the value `true`. All other values will raise an error.
+
 ==== Analyze API changes
 
 The deprecated request parameters and plain text in request body has been removed. Define parameters in request body.
@@ -26,4 +37,4 @@ defaults.
 === `timestamp` and `ttl` in index requests
 
 `timestamp` and `ttl` are not accepted anymore as parameters of index/update
-requests.
+requests.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_6_0/search.asciidoc b/docs/reference/migration/migrate_6_0/search.asciidoc
index bc90b90c8b2..3a8072f7cc0 100644
--- a/docs/reference/migration/migrate_6_0/search.asciidoc
+++ b/docs/reference/migration/migrate_6_0/search.asciidoc
@@ -26,7 +26,52 @@
 * Support for empty query objects (`{ }`) has been removed from the query DSL.
   An error is thrown whenever an empty query object is provided.
 
+* The deprecated `minimum_number_should_match` parameter in the `bool` query has
+  been removed, use `minimum_should_match` instead.
+
+* The `query_string` query now correctly parses the maximum number of
+  states allowed when
+  "https://en.wikipedia.org/wiki/Powerset_construction#Complexity[determinizing]"
+  a regex as `max_determinized_states` instead of the typo
+  `max_determined_states`.
+
+* For `geo_distance` queries, sorting, and aggregations the `sloppy_arc` and `factor` options
+  have been removed from the `distance_type` parameter.
+
 ==== Search shards API
 
 The search shards API no longer accepts the `type` url parameter, which didn't
 have any effect in previous versions.
+
+==== Changes to the Profile API
+
+* The `"time"` field showing human readable timing output has been replaced by the `"time_in_nanos"`
+  field which displays the elapsed time in nanoseconds. The `"time"` field can be turned on by adding
+  `"?human=true"` to the request url. It will display a rounded, human readable time value.
+
+==== Scoring changes
+
+===== Query normalization
+
+Query normalization has been removed. This means that the TF-IDF similarity no
+longer tries to make scores comparable across queries and that boosts are now
+integrated into scores as simple multiplicative factors.
+
+Other similarities are not affected as they did not normalize scores and
+already integrated boosts into scores as multiplicative factors.
+
+See https://issues.apache.org/jira/browse/LUCENE-7347[`LUCENE-7347`] for more
+information.
+
+===== Coordination factors
+
+Coordination factors have been removed from the scoring formula. This means that
+boolean queries no longer score based on the number of matching clauses.
+Instead, they always return the sum of the scores of the matching clauses.
+
+As a consequence, use of the TF-IDF similarity is now discouraged as this was
+an important component of the quality of the scores that this similarity
+produces. BM25 is recommended instead.
+
+See https://issues.apache.org/jira/browse/LUCENE-7347[`LUCENE-7347`] for more
+information.
diff --git a/docs/reference/migration/migrate_6_0/settings.asciidoc b/docs/reference/migration/migrate_6_0/settings.asciidoc
index c2f20de7d21..ec25ffb601c 100644
--- a/docs/reference/migration/migrate_6_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_6_0/settings.asciidoc
@@ -1,6 +1,13 @@
 [[breaking_60_settings_changes]]
 === Settings changes
 
+==== Coercion of boolean settings
+
+Previously, Elasticsearch recognized the strings `true`, `false`, `on`, `off`, `yes`, `no`, `0`, `1` as booleans. Elasticsearch 6.0
+recognizes only `true` and `false` as boolean and will throw an error otherwise. For backwards compatibility purposes, during the 6.x series
+index settings on pre-6.0 indices will continue to work. Note that this does not apply to node-level settings that are stored
+in `elasticsearch.yml`.
+
 ==== Snapshot settings
 
 The internal setting `cluster.routing.allocation.snapshot.relocation_enabled` that allowed shards with running snapshots to be reallocated to
@@ -20,3 +27,10 @@ recognized anymore.
 The `default` `index.store.type` has been removed. If you were using it, we
 advise that you simply remove it from your index settings and Elasticsearch
 will use the best `store` implementation for your operating system.
+
+==== Network settings
+
+The blocking TCP client, blocking TCP server, and blocking HTTP server have been removed.
+As a consequence, the `network.tcp.blocking_server`, `network.tcp.blocking_client`,
+`network.tcp.blocking`,`transport.tcp.blocking_client`, `transport.tcp.blocking_server`,
+and `http.tcp.blocking_server` settings are not recognized anymore.
\ No newline at end of file
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 2ab1232d3b5..ec6bd4593f5 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -77,6 +77,11 @@ The modules in this section are:
 
     A tribe node joins one or more clusters and acts as a federated
     client across them.
+
+<<modules-cross-cluster-search, Cross cluster Search>>::
+
+    Cross cluster search enables executing search requests across more than one cluster without joining them and acts
+    as a federated client across them.
 --
 
 
@@ -105,3 +110,6 @@ include::modules/threadpool.asciidoc[]
 include::modules/transport.asciidoc[]
 
 include::modules/tribe.asciidoc[]
+
+include::modules/cross-cluster-search.asciidoc[]
+
diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc
new file mode 100644
index 00000000000..5fadda35e1c
--- /dev/null
+++ b/docs/reference/modules/cross-cluster-search.asciidoc
@@ -0,0 +1,127 @@
+[[modules-cross-cluster-search]]
+== Cross cluster search
+
+experimental[]
+
+The _cross cluster search_ feature allows any node to act as a federated client across
+multiple clusters. In contrast to the _tribe_ feature, a  _cross cluster search_ node won't
+join the remote cluster, instead it connects to a remote cluster in a light fashion in order to execute
+federated search requests.
+
+The _cross cluster search_ feature works by configuring a remote cluster in the cluster state and connects only to a
+limited number of nodes in the remote cluster. Each remote cluster is referenced by a name and a list of seed nodes.
+Those seed nodes are used to discover other nodes eligible as so-called _gateway nodes_. Each node in a cluster that
+has remote clusters configured connects to one or more _gateway nodes_ and uses them to federate search requests to
+the remote cluster.
+
+Remote clusters can either be configured as part of the `elasticsearch.yml` file or be dynamically updated via
+the <<cluster-update-settings,cluster settings API>>. If a remote cluster is configured via `elasticsearch.yml` only
+the nodes with the configuration set will be connecting to the remote cluster in which case federated search requests
+will have to be sent specifically to those nodes. Remote clusters set via the
+<<cluster-update-settings,cluster settings API>> will be available on every node in the cluster.
+
+The `elasticsearch.yml` config file for a _cross cluster search_ node just needs to list the
+remote clusters that should be connected to, for instance:
+
+[source,yaml]
+--------------------------------
+search:
+    remote:
+        cluster_one: <1>
+            seeds: 127.0.0.1:9300
+        cluster_two: <1>
+            seeds: 127.0.0.1:9301
+
+--------------------------------
+<1> `cluster_one` and `cluster_two` are arbitrary cluster aliases representing the connection to each cluster.
+These names are subsequently used to distinguish between local and remote indices.
+
+[float]
+=== Using cross cluster search
+
+To search the `twitter` index on remote cluster `cluster_1` the index name must be prefixed with the cluster alias
+separated by a `:` character:
+
+[source,js]
+--------------------------------------------------
+POST /cluster_one:twitter/tweet/_search
+{
+    "match_all": {}
+}
+--------------------------------------------------
+
+In contrast to the `tribe` feature cross cluster search can also search indices with the same name on different
+clusters:
+
+[source,js]
+--------------------------------------------------
+POST /cluster_one:twitter,twitter/tweet/_search
+{
+    "match_all": {}
+}
+--------------------------------------------------
+
+Search results are disambiguated the same way as the indices are disambiguated in the request. Even if index names are
+identical these indices will be treated as different indices when results are merged. All results retrieved from a
+remote index
+will be prefixed with their remote cluster name:
+
+[source,js]
+--------------------------------------------------
+ {
+  "took" : 89,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 10,
+    "successful" : 10,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 2,
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "cluster_one:twitter",
+        "_type" : "tweet",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "user" : "kimchy",
+          "postDate" : "2009-11-15T14:12:12",
+          "message" : "trying out Elasticsearch"
+        }
+      },
+      {
+        "_index" : "twitter",
+        "_type" : "tweet",
+        "_id" : "1",
+        "_score" : 1.0,
+        "_source" : {
+          "user" : "kimchy",
+          "postDate" : "2009-11-15T14:12:12",
+          "message" : "trying out Elasticsearch"
+        }
+      }
+    ]
+  }
+}
+--------------------------------------------------
+
+[float]
+=== Cross cluster search settings
+
+`search.remote.connections_per_cluster`::
+
+  The number of nodes to connect to per remote cluster. The default is `3`.
+
+`search.remote.initial_connect_timeout`::
+
+  The time to wait for remote connections to be established when the node starts. The default is `30s`.
+
+`search.remote.node.attr`::
+
+  A node attribute to filter out nodes that are eligible as a gateway node in
+  the remote cluster. For instance a node can have a node attribute
+  `node.attr.gateway: true`such that only nodes with this attribute will be
+  connected to if `search.remote.node.attr` is set to `gateway`.
+
diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index 1caace2216e..ea50d11e39e 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -39,8 +39,8 @@ Unicast discovery provides the following settings with the `discovery.zen.ping.u
 |=======================================================================
 |Setting |Description
 |`hosts` |Either an array setting or a comma delimited setting. Each
-value should be in the form of `host:port` or `host` (where `port` defaults to `9300`). Note that IPv6 hosts must be
-bracketed. Defaults to `127.0.0.1, [::1]`
+          value should be in the form of `host:port` or `host` (where `port` defaults to the setting `transport.profiles.default.port`
+          falling back to `transport.tcp.port` if not set). Note that IPv6 hosts must be bracketed. Defaults to `127.0.0.1, [::1]`
 |`hosts.resolve_timeout` |The amount of time to wait for DNS lookups on each round of pinging. Specified as
 <<time-units, time units>>. Defaults to 5s.
 |=======================================================================
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
index 065c91349c2..3682234f5e6 100644
--- a/docs/reference/modules/http.asciidoc
+++ b/docs/reference/modules/http.asciidoc
@@ -100,6 +100,9 @@ simple message will be returned. Defaults to `true`
 
 |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`.
 
+|`http.content_type.required`|Enables or disables strict checking and usage of
+the `Content-Type` header for all requests with content, defaults to `false`.
+
 |=======================================================================
 
 It also uses the common
diff --git a/docs/reference/modules/indices/recovery.asciidoc b/docs/reference/modules/indices/recovery.asciidoc
index 8de3309347c..84352cfe689 100644
--- a/docs/reference/modules/indices/recovery.asciidoc
+++ b/docs/reference/modules/indices/recovery.asciidoc
@@ -1,22 +1,10 @@
 [[recovery]]
 === Indices Recovery
 
-The following _expert_ settings can be set to manage the recovery policy.
-
-`indices.recovery.file_chunk_size`::
-    Defaults to `512kb`.
-
-`indices.recovery.translog_ops`::
-    Defaults to `1000`.
-
-`indices.recovery.translog_size`::
-    Defaults to `512kb`.
-
-`indices.recovery.compress`::
-    Defaults to `true`.
+The following _expert_ setting can be set to manage the recovery policy.
 
 `indices.recovery.max_bytes_per_sec`::
     Defaults to `40mb`.
 
-These settings can be dynamically updated on a live cluster with the
+This setting can be dynamically updated on a live cluster with the
 <<cluster-update-settings,cluster-update-settings>> API:
diff --git a/docs/reference/modules/scripting/native.asciidoc b/docs/reference/modules/scripting/native.asciidoc
index 8fb49b03d96..ba8b154846f 100644
--- a/docs/reference/modules/scripting/native.asciidoc
+++ b/docs/reference/modules/scripting/native.asciidoc
@@ -36,6 +36,10 @@ public class MyNativeScriptPlugin extends Plugin implements ScriptPlugin {
         public boolean needsScores() {
             return false;
         }
+        @Override
+        public String getName() {
+            return "my_script";
+        }        
     }
 
     public static class MyNativeScript extends AbstractDoubleSearchScript {
@@ -66,7 +70,7 @@ curl -XPOST localhost:9200/_search -d '{
         {
           "script_score": {
             "script": {
-                "id": "my_script",
+                "inline": "my_script",
                 "lang" : "native"
             }
           }
diff --git a/docs/reference/modules/scripting/painless-debugging.asciidoc b/docs/reference/modules/scripting/painless-debugging.asciidoc
index eafe291d259..5bd7fe73541 100644
--- a/docs/reference/modules/scripting/painless-debugging.asciidoc
+++ b/docs/reference/modules/scripting/painless-debugging.asciidoc
@@ -38,15 +38,16 @@ POST /hockey/player/1/_explain
 // we have to override it to get a normal shaped response
 
 Which shows that the class of `doc.first` is
-`org.elasticsearch.index.fielddata.ScriptDocValues$Longs` by responding with:
+`org.elasticsearch.index.fielddata.ScriptDocValues.Longs` by responding with:
 
 [source,js]
 ---------------------------------------------------------
 {
    "error": {
       "type": "script_exception",
-      "class": "org.elasticsearch.index.fielddata.ScriptDocValues$Longs",
       "to_string": "[1, 9, 27]",
+      "painless_class": "org.elasticsearch.index.fielddata.ScriptDocValues.Longs",
+      "java_class": "org.elasticsearch.index.fielddata.ScriptDocValues$Longs",
       ...
    },
    "status": 500
@@ -54,7 +55,7 @@ Which shows that the class of `doc.first` is
 ---------------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"script_stack": $body.error.script_stack, "script": $body.error.script, "lang": $body.error.lang, "caused_by": $body.error.caused_by, "root_cause": $body.error.root_cause, "reason": $body.error.reason/]
 
-You can use the same trick to see that `_source` is a `java.util.LinkedHashMap`
+You can use the same trick to see that `_source` is a `LinkedHashMap`
 in the `_update` API:
 
 [source,js]
@@ -78,8 +79,9 @@ The response looks like:
     "reason": "failed to execute script",
     "caused_by": {
       "type": "script_exception",
-      "class": "java.util.LinkedHashMap",
       "to_string": "{gp=[26, 82, 1], last=gaudreau, assists=[17, 46, 0], first=johnny, goals=[9, 27, 1]}",
+      "painless_class": "LinkedHashMap",
+      "java_class": "java.util.LinkedHashMap",
       ...
     }
   },
@@ -90,19 +92,5 @@ The response looks like:
 // TESTRESPONSE[s/\.\.\./"script_stack": $body.error.caused_by.script_stack, "script": $body.error.caused_by.script, "lang": $body.error.caused_by.lang, "caused_by": $body.error.caused_by.caused_by, "reason": $body.error.caused_by.reason/]
 // TESTRESPONSE[s/"to_string": ".+"/"to_string": $body.error.caused_by.to_string/]
 
-// TODO we should build some javadoc like mashup so people don't have to jump through these hoops.
-
-Once you have the class of an object you can go
-https://github.com/elastic/elasticsearch/tree/{branch}/modules/lang-painless/src/main/resources/org/elasticsearch/painless[here]
-and check the available methods. Painless uses a strict whitelist to prevent
-scripts that don't work well with Elasticsearch and all whitelisted methods
-are listed in a file named after the package of the object (everything before
-the last `.`). So `java.util.Map` is listed in a file named `java.util.txt`
-starting on the line that looks like `class Map -> java.util.Map {`.
-
-With the list of whitelisted methods in hand you can turn to either
-https://docs.oracle.com/javase/8/docs/api/[Javadoc],
-https://github.com/elastic/elasticsearch/tree/{branch}[Elasticsearch's source tree]
-or, for whitelisted methods ending in `*`, the
-https://github.com/elastic/elasticsearch/blob/{branch}/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java[Augmentation]
-class.
+Once you have a class you can go to <<painless-api-reference>> to see a list of
+available methods.
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 7995e45e7b9..0f8cfc5d0e2 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -10,8 +10,9 @@ default.
 
 The Painless syntax is similar to http://groovy-lang.org/index.html[Groovy].
 
-You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter
-to `painless`.
+You can use Painless anywhere a script can be used in Elasticsearch. It is the
+default if you don't set the `lang` parameter but if you want to be explicit you
+can set the `lang` parameter to `painless`.
 
 [[painless-features]]
 [float]
@@ -19,7 +20,8 @@ to `painless`.
 
 * Fast performance: https://benchmarks.elastic.co/index.html#search_qps_scripts[several times faster] than the alternatives.
 
-* Safety: Fine-grained <<painless-api, whitelist>> with method call/field granularity.
+* Safety: Fine-grained whitelist with method call/field granularity. See
+<<painless-api-reference>> for a complete list of available classes and methods.
 
 * Optional typing: Variables and parameters can use explicit types or the dynamic `def` type.
 
@@ -37,27 +39,27 @@ To illustrate how Painless works, let's load some hockey stats into an Elasticse
 ----------------------------------------------------------------
 PUT hockey/player/_bulk?refresh
 {"index":{"_id":1}}
-{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1],"born":"1993/08/13"}
 {"index":{"_id":2}}
-{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82]}
+{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82],"born":"1994/10/12"}
 {"index":{"_id":3}}
-{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79]}
+{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79],"born":"1984/01/04"}
 {"index":{"_id":4}}
-{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82]}
+{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82],"born":"1988/02/17"}
 {"index":{"_id":5}}
-{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0]}
+{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0],"born":"1996/06/20"}
 {"index":{"_id":6}}
-{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82]}
+{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82],"born":"1983/03/20"}
 {"index":{"_id":7}}
-{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34]}
+{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34],"born":"1984/08/10"}
 {"index":{"_id":8}}
-{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82]}
+{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82],"born":"1990/06/07"}
 {"index":{"_id":39}}
-{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63]}
+{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63],"born":"1983/10/03"}
 {"index":{"_id":10}}
-{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82]}
+{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82],"born":"1989/03/17"}
 {"index":{"_id":11}}
-{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
+{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82],"born":"1990/01/30"}
 ----------------------------------------------------------------
 // CONSOLE
 // TESTSETUP
@@ -192,6 +194,40 @@ POST hockey/player/1/_update
 ----------------------------------------------------------------
 // CONSOLE
 
+[float]
+[[modules-scripting-painless-dates]]
+=== Regular expressions
+
+Dates are a little different to work with than regular values. Here is an
+example returning the year of every player's birth:
+
+[source,js]
+----------------------------------------------------------------
+GET hockey/_search
+{
+  "script_fields": {
+    "birth_year": {
+      "script": {
+        "inline": "doc.born.date.year"
+      }
+    }
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+The key here is that instead of indexing directly into `doc.born` like you would
+a normal field you have to call `doc.born.date` to get a
+<<painless-api-reference-org-joda-time-ReadableDateTime, `ReadableDateTime`>>.
+From there you can call methods like
+<<painless-api-reference-org-joda-time-ReadableDateTime-getYear-0, `getYear`>>,
+and <<painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0, `getDayOfWeek`>>.
+In the example above `year` is a shortcut to `getYear()`.
+
+If the date field is a list then `date` will always return the first date. To
+access all the dates use `dates` instead of `date`.
+
+
 [float]
 [[modules-scripting-painless-regex]]
 === Regular expressions
@@ -320,28 +356,3 @@ Note: all of the `_update_by_query` examples above could really do with a
 <<query-dsl-script-query>> it wouldn't be as efficient as using any other query
 because script queries aren't able to use the inverted index to limit the
 documents that they have to check.
-
-[float]
-[[painless-api]]
-== Painless API
-
-The following Java packages are available for use in the Painless language:
-
-* https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[java.lang]
-* https://docs.oracle.com/javase/8/docs/api/java/math/package-summary.html[java.math]
-* https://docs.oracle.com/javase/8/docs/api/java/text/package-summary.html[java.text]
-* https://docs.oracle.com/javase/8/docs/api/java/time/package-summary.html[java.time]
-* https://docs.oracle.com/javase/8/docs/api/java/time/chrono/package-summary.html[java.time.chrono]
-* https://docs.oracle.com/javase/8/docs/api/java/time/format/package-summary.html[java.time.format]
-* https://docs.oracle.com/javase/8/docs/api/java/time/temporal/package-summary.html[java.time.temporal]
-* https://docs.oracle.com/javase/8/docs/api/java/time/zone/package-summary.html[java.time.zone]
-* https://docs.oracle.com/javase/8/docs/api/java/util/package-summary.html[java.util]
-* https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[java.util.function]
-* https://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html[java.util.regex]
-* https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html[java.util.stream]
-
-Note that unsafe classes and methods are not included, there is no support for:
-
-* Manipulation of processes and threads
-* Input/Output
-* Reflection
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index 3b40937c2d8..110e65e52c8 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -51,8 +51,8 @@ Each of these settings takes one of these values:
 
 
 [horizontal]
-`false`::   Scripting is enabled.
-`true`::    Scripting is disabled.
+`false`::   Scripting is disabled.
+`true`::    Scripting is enabled.
 
 The default values are the following:
 
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 5f2132d142d..7c89201c7fc 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -54,11 +54,11 @@ GET my_index/_search
     setting `script.default_lang` to the appropriate language.
 
 
-`inline`, `id`, `file`::
+`inline`, `stored`, `file`::
 
     Specifies the source of the script.  An `inline` script is specified
-    `inline` as in the example above, a stored script with the specified `id`
-    is retrieved from the cluster state (see <<modules-scripting-stored-scripts,Stored Scripts>>),
+    `inline` as in the example above, a `stored` script is specified `stored`
+    and is retrieved from the cluster state (see <<modules-scripting-stored-scripts,Stored Scripts>>),
     and a `file` script is retrieved from a file in the `config/scripts`
     directory (see <<modules-scripting-file-scripts, File Scripts>>).
 +
@@ -164,7 +164,6 @@ hierarchy of directories is flattened and concatenated with underscores.  A
 script in `group1/group2/my_script.painless` should use `group1_group2_myscript`
 as the `file` name.
 
-
 [[reload-scripts]]
 [float]
 ==== Automatic script reloading
@@ -181,23 +180,51 @@ Script reloading can be completely disabled by setting
 === Stored Scripts
 
 Scripts may be stored in and retrieved from the cluster state using the
-`_scripts` end-point:
+`_scripts` end-point.
+
+==== Deprecated Namespace
+
+The namespace for stored scripts using both `lang` and `id` as a unique
+identifier has been deprecated.  The new namespace for stored scripts will
+only use `id`. Stored scripts with the same `id`, but different `lang`'s
+will no longer be allowed in 6.0.  To comply with the new namespace for
+stored scripts, existing stored scripts should be deleted and put again.
+Any scripts that share an `id` but have different `lang`s will need to
+be re-named.  For example, take the following:
+
+"id": "example", "lang": "painless"
+"id": "example", "lang": "expressions"
+
+The above scripts will conflict under the new namespace since the id's are
+the same.  At least one will have to be re-named to comply with the new
+namespace of only `id`.
+
+As a final caveat, stored search templates and stored scripts share
+the same namespace, so if a search template has the same `id` as a
+stored script, one of the two will have to be re-named as well using
+delete and put requests.
+
+==== Request Examples
+
+The following are examples of stored script requests:
 
 [source,js]
 -----------------------------------
-/_scripts/{lang}/{id} <1> <2>
+/_scripts/{id} <1>
 -----------------------------------
-<1> The `lang` represents the script language.
-<2> The `id` is a unique identifier or script name.
+<1> The `id` is a unique identifier for the stored script.
 
 This example stores a Painless script called `calculate-score` in the cluster
 state:
 
 [source,js]
 -----------------------------------
-POST _scripts/painless/calculate-score
+POST _scripts/calculate-score
 {
-  "script": "Math.log(_score * 2) + params.my_modifier"
+  "script": {
+    "lang": "painless",
+    "code": "Math.log(_score * 2) + params.my_modifier"
+  }
 }
 -----------------------------------
 // CONSOLE
@@ -206,12 +233,12 @@ This same script can be retrieved with:
 
 [source,js]
 -----------------------------------
-GET _scripts/painless/calculate-score
+GET _scripts/calculate-score
 -----------------------------------
 // CONSOLE
 // TEST[continued]
 
-Stored scripts can be used by specifying the `lang` and `stored` parameters as follows:
+Stored scripts can be used by specifying the `stored` parameters as follows:
 
 [source,js]
 --------------------------------------------------
@@ -220,7 +247,6 @@ GET _search
   "query": {
     "script": {
       "script": {
-        "lang": "painless",
         "stored": "calculate-score",
         "params": {
           "my_modifier": 2
@@ -237,7 +263,7 @@ And deleted with:
 
 [source,js]
 -----------------------------------
-DELETE _scripts/painless/calculate-score
+DELETE _scripts/calculate-score
 -----------------------------------
 // CONSOLE
 // TEST[continued]
diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index 4e8b5c61efd..8f3a524dbdd 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -35,11 +35,6 @@ There are several thread pools, but the important ones include:
     queue_size of `50`.  The maximum size for this pool
     is `1 + # of available processors`.
 
-`percolate`::
-    For percolate operations. Thread pool type is `fixed`
-    with a size of `# of available processors`,
-    queue_size of `1000`.
-
 `snapshot`::
     For snapshot/restore operations. Thread pool type is `scaling` with a
     keep-alive of `5m` and a max of `min(5, (# of available processors)/2)`.
diff --git a/docs/reference/painless-api-reference.asciidoc b/docs/reference/painless-api-reference.asciidoc
new file mode 100644
index 00000000000..f729c52a1bc
--- /dev/null
+++ b/docs/reference/painless-api-reference.asciidoc
@@ -0,0 +1,17 @@
+["appendix",id="painless-api-reference"]
+= Painless API Reference
+
+<<modules-scripting-painless,Painless>> has a strict whitelist for methods and
+classes to make sure that all painless scripts are secure and fast. Most of
+these methods are exposed directly from the JRE while others are part of
+Elasticsearch or Painless itself. Below is a list of all available methods
+grouped under the classes on which you can call them. Clicking on the method
+name takes you to the documentation for the method.
+
+NOTE: Methods defined in the JRE also have a `(java 9)` link which can be used
+to see the method's documentation in Java 9 while clicking on the method's name
+goes to the Java 8 documentation. Usually these aren't different but it is
+worth going to the version that matches the version of Java you are using to
+run Elasticsearch just in case.
+
+include::painless-api-reference/index.asciidoc[]
diff --git a/docs/reference/painless-api-reference/AbstractChronology.asciidoc b/docs/reference/painless-api-reference/AbstractChronology.asciidoc
new file mode 100644
index 00000000000..3cc90ee65cd
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractChronology.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractChronology]]++AbstractChronology++::
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/AbstractCollection.asciidoc b/docs/reference/painless-api-reference/AbstractCollection.asciidoc
new file mode 100644
index 00000000000..313c327ac07
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractCollection.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractCollection]]++AbstractCollection++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractList.asciidoc b/docs/reference/painless-api-reference/AbstractList.asciidoc
new file mode 100644
index 00000000000..9ce7c9a6830
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractList.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractList]]++AbstractList++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractMap.SimpleEntry.asciidoc b/docs/reference/painless-api-reference/AbstractMap.SimpleEntry.asciidoc
new file mode 100644
index 00000000000..327939e009d
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractMap.SimpleEntry.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap-SimpleEntry]]++AbstractMap.SimpleEntry++::
+* ++[[painless-api-reference-AbstractMap-SimpleEntry-AbstractMap.SimpleEntry-1]]link:{java8-javadoc}/java/util/AbstractMap$SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.util.Map$Entry%2D[AbstractMap.SimpleEntry](<<painless-api-reference-Map-Entry,Map.Entry>>)++ (link:{java9-javadoc}/java/util/AbstractMap$SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.util.Map$Entry%2D[java 9])
+* ++[[painless-api-reference-AbstractMap-SimpleEntry-AbstractMap.SimpleEntry-2]]link:{java8-javadoc}/java/util/AbstractMap$SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.lang.Object%2Djava.lang.Object%2D[AbstractMap.SimpleEntry](def, def)++ (link:{java9-javadoc}/java/util/AbstractMap$SimpleEntry.html#AbstractMap.SimpleEntry%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map-Entry,Map.Entry>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc b/docs/reference/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc
new file mode 100644
index 00000000000..94a4b0c307a
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractMap.SimpleImmutableEntry.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap-SimpleImmutableEntry]]++AbstractMap.SimpleImmutableEntry++::
+* ++[[painless-api-reference-AbstractMap-SimpleImmutableEntry-AbstractMap.SimpleImmutableEntry-1]]link:{java8-javadoc}/java/util/AbstractMap$SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.util.Map$Entry%2D[AbstractMap.SimpleImmutableEntry](<<painless-api-reference-Map-Entry,Map.Entry>>)++ (link:{java9-javadoc}/java/util/AbstractMap$SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.util.Map$Entry%2D[java 9])
+* ++[[painless-api-reference-AbstractMap-SimpleImmutableEntry-AbstractMap.SimpleImmutableEntry-2]]link:{java8-javadoc}/java/util/AbstractMap$SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.lang.Object%2Djava.lang.Object%2D[AbstractMap.SimpleImmutableEntry](def, def)++ (link:{java9-javadoc}/java/util/AbstractMap$SimpleImmutableEntry.html#AbstractMap.SimpleImmutableEntry%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map-Entry,Map.Entry>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractMap.asciidoc b/docs/reference/painless-api-reference/AbstractMap.asciidoc
new file mode 100644
index 00000000000..ea3be0d125a
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractMap.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractMap]]++AbstractMap++::
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractQueue.asciidoc b/docs/reference/painless-api-reference/AbstractQueue.asciidoc
new file mode 100644
index 00000000000..37fc53bd73d
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractQueue.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractQueue]]++AbstractQueue++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/reference/painless-api-reference/AbstractSequentialList.asciidoc b/docs/reference/painless-api-reference/AbstractSequentialList.asciidoc
new file mode 100644
index 00000000000..822b2038516
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractSequentialList.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractSequentialList]]++AbstractSequentialList++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AbstractSet.asciidoc b/docs/reference/painless-api-reference/AbstractSet.asciidoc
new file mode 100644
index 00000000000..7baa3a58f79
--- /dev/null
+++ b/docs/reference/painless-api-reference/AbstractSet.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AbstractSet]]++AbstractSet++::
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/reference/painless-api-reference/Annotation.asciidoc b/docs/reference/painless-api-reference/Annotation.asciidoc
new file mode 100644
index 00000000000..33252543951
--- /dev/null
+++ b/docs/reference/painless-api-reference/Annotation.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Annotation]]++Annotation++::
+* ++[[painless-api-reference-Annotation-Annotation-1]]link:{java8-javadoc}/java/text/Annotation.html#Annotation%2Djava.lang.Object%2D[Annotation](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Annotation.html#Annotation%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Annotation-getValue-0]]def link:{java8-javadoc}/java/text/Annotation.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/text/Annotation.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Appendable.asciidoc b/docs/reference/painless-api-reference/Appendable.asciidoc
new file mode 100644
index 00000000000..61fcbaf5a60
--- /dev/null
+++ b/docs/reference/painless-api-reference/Appendable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Appendable]]++Appendable++::
+* ++[[painless-api-reference-Appendable-append-3]]<<painless-api-reference-Appendable,Appendable>> link:{java8-javadoc}/java/lang/Appendable.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Appendable.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ArithmeticException.asciidoc b/docs/reference/painless-api-reference/ArithmeticException.asciidoc
new file mode 100644
index 00000000000..f4862893205
--- /dev/null
+++ b/docs/reference/painless-api-reference/ArithmeticException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArithmeticException]]++ArithmeticException++::
+* ++[[painless-api-reference-ArithmeticException-ArithmeticException-0]]link:{java8-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2D%2D[ArithmeticException]()++ (link:{java9-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2D%2D[java 9])
+* ++[[painless-api-reference-ArithmeticException-ArithmeticException-1]]link:{java8-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2Djava.lang.String%2D[ArithmeticException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArithmeticException.html#ArithmeticException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ArrayDeque.asciidoc b/docs/reference/painless-api-reference/ArrayDeque.asciidoc
new file mode 100644
index 00000000000..668a86185e6
--- /dev/null
+++ b/docs/reference/painless-api-reference/ArrayDeque.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayDeque]]++ArrayDeque++::
+* ++[[painless-api-reference-ArrayDeque-ArrayDeque-0]]link:{java8-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2D%2D[ArrayDeque]()++ (link:{java9-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayDeque-ArrayDeque-1]]link:{java8-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2Djava.util.Collection%2D[ArrayDeque](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/ArrayDeque.html#ArrayDeque%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-ArrayDeque-clone-0]]<<painless-api-reference-ArrayDeque,ArrayDeque>> link:{java8-javadoc}/java/util/ArrayDeque.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/ArrayDeque.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Deque,Deque>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/reference/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc b/docs/reference/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc
new file mode 100644
index 00000000000..49b13361163
--- /dev/null
+++ b/docs/reference/painless-api-reference/ArrayIndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayIndexOutOfBoundsException]]++ArrayIndexOutOfBoundsException++::
+* ++[[painless-api-reference-ArrayIndexOutOfBoundsException-ArrayIndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2D%2D[ArrayIndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayIndexOutOfBoundsException-ArrayIndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2Djava.lang.String%2D[ArrayIndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArrayIndexOutOfBoundsException.html#ArrayIndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ArrayList.asciidoc b/docs/reference/painless-api-reference/ArrayList.asciidoc
new file mode 100644
index 00000000000..90386a3742c
--- /dev/null
+++ b/docs/reference/painless-api-reference/ArrayList.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayList]]++ArrayList++::
+* ++[[painless-api-reference-ArrayList-ArrayList-0]]link:{java8-javadoc}/java/util/ArrayList.html#ArrayList%2D%2D[ArrayList]()++ (link:{java9-javadoc}/java/util/ArrayList.html#ArrayList%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayList-ArrayList-1]]link:{java8-javadoc}/java/util/ArrayList.html#ArrayList%2Djava.util.Collection%2D[ArrayList](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/ArrayList.html#ArrayList%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-ArrayList-clone-0]]def link:{java8-javadoc}/java/util/ArrayList.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/ArrayList.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayList-trimToSize-0]]void link:{java8-javadoc}/java/util/ArrayList.html#trimToSize%2D%2D[trimToSize]()++ (link:{java9-javadoc}/java/util/ArrayList.html#trimToSize%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ArrayStoreException.asciidoc b/docs/reference/painless-api-reference/ArrayStoreException.asciidoc
new file mode 100644
index 00000000000..29f7a536ead
--- /dev/null
+++ b/docs/reference/painless-api-reference/ArrayStoreException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ArrayStoreException]]++ArrayStoreException++::
+* ++[[painless-api-reference-ArrayStoreException-ArrayStoreException-0]]link:{java8-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2D%2D[ArrayStoreException]()++ (link:{java9-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2D%2D[java 9])
+* ++[[painless-api-reference-ArrayStoreException-ArrayStoreException-1]]link:{java8-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2Djava.lang.String%2D[ArrayStoreException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ArrayStoreException.html#ArrayStoreException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Arrays.asciidoc b/docs/reference/painless-api-reference/Arrays.asciidoc
new file mode 100644
index 00000000000..97598d64d57
--- /dev/null
+++ b/docs/reference/painless-api-reference/Arrays.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Arrays]]++Arrays++::
+* ++[[painless-api-reference-Arrays-asList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Arrays.html#asList%2Djava.lang.Object:A%2D[asList](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#asList%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepEquals-2]]static boolean link:{java8-javadoc}/java/util/Arrays.html#deepEquals%2Djava.lang.Object:A%2Djava.lang.Object:A%2D[deepEquals](<<painless-api-reference-Object,Object>>[], <<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepEquals%2Djava.lang.Object:A%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepHashCode-1]]static int link:{java8-javadoc}/java/util/Arrays.html#deepHashCode%2Djava.lang.Object:A%2D[deepHashCode](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepHashCode%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Arrays-deepToString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Arrays.html#deepToString%2Djava.lang.Object:A%2D[deepToString](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Arrays.html#deepToString%2Djava.lang.Object:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc b/docs/reference/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc
new file mode 100644
index 00000000000..56ccfbe6aab
--- /dev/null
+++ b/docs/reference/painless-api-reference/AttributedCharacterIterator.Attribute.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedCharacterIterator-Attribute]]++AttributedCharacterIterator.Attribute++::
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-INPUT_METHOD_SEGMENT]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#INPUT_METHOD_SEGMENT[INPUT_METHOD_SEGMENT] (link:{java9-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#INPUT_METHOD_SEGMENT[java 9])
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-LANGUAGE]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#LANGUAGE[LANGUAGE] (link:{java9-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#LANGUAGE[java 9])
+** [[painless-api-reference-AttributedCharacterIterator-Attribute-READING]]static <<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>> link:{java8-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#READING[READING] (link:{java9-javadoc}/java/text/AttributedCharacterIterator$Attribute.html#READING[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AttributedCharacterIterator.asciidoc b/docs/reference/painless-api-reference/AttributedCharacterIterator.asciidoc
new file mode 100644
index 00000000000..a71d3906fa1
--- /dev/null
+++ b/docs/reference/painless-api-reference/AttributedCharacterIterator.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedCharacterIterator]]++AttributedCharacterIterator++::
+* ++[[painless-api-reference-AttributedCharacterIterator-getAllAttributeKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAllAttributeKeys%2D%2D[getAllAttributeKeys]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAllAttributeKeys%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getAttribute-1]]def link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2D[getAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getAttributes-0]]<<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getAttributes%2D%2D[getAttributes]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getAttributes%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunLimit-0]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2D%2D[getRunLimit]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunLimit-1]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2Djava.util.Set%2D[getRunLimit](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunLimit%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunStart-0]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2D%2D[getRunStart]()++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedCharacterIterator-getRunStart-1]]int link:{java8-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2Djava.util.Set%2D[getRunStart](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/text/AttributedCharacterIterator.html#getRunStart%2Djava.util.Set%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharacterIterator,CharacterIterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/AttributedString.asciidoc b/docs/reference/painless-api-reference/AttributedString.asciidoc
new file mode 100644
index 00000000000..0393838a737
--- /dev/null
+++ b/docs/reference/painless-api-reference/AttributedString.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-AttributedString]]++AttributedString++::
+* ++[[painless-api-reference-AttributedString-AttributedString-1]]link:{java8-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2D[AttributedString](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-AttributedString-AttributedString-2]]link:{java8-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2Djava.util.Map%2D[AttributedString](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#AttributedString%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttribute-2]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2D[addAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttribute-4]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2Dint%2Dint%2D[addAttribute](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>, <<painless-api-reference-Object,Object>>, int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttribute%2Djava.text.AttributedCharacterIterator$Attribute%2Djava.lang.Object%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-AttributedString-addAttributes-3]]void link:{java8-javadoc}/java/text/AttributedString.html#addAttributes%2Djava.util.Map%2Dint%2Dint%2D[addAttributes](<<painless-api-reference-Map,Map>>, int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#addAttributes%2Djava.util.Map%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-0]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2D%2D[getIterator]()++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2D%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-1]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2D[getIterator](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>[])++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2D[java 9])
+* ++[[painless-api-reference-AttributedString-getIterator-3]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2Dint%2Dint%2D[getIterator](<<painless-api-reference-AttributedCharacterIterator-Attribute,AttributedCharacterIterator.Attribute>>[], int, int)++ (link:{java9-javadoc}/java/text/AttributedString.html#getIterator%2Djava.text.AttributedCharacterIterator$Attribute:A%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Base64.Decoder.asciidoc b/docs/reference/painless-api-reference/Base64.Decoder.asciidoc
new file mode 100644
index 00000000000..3b3acd16e61
--- /dev/null
+++ b/docs/reference/painless-api-reference/Base64.Decoder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64-Decoder]]++Base64.Decoder++::
+* ++[[painless-api-reference-Base64-Decoder-decode-1]]byte[] link:{java8-javadoc}/java/util/Base64$Decoder.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Base64$Decoder.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Base64-Decoder-decode-2]]int link:{java8-javadoc}/java/util/Base64$Decoder.html#decode%2Dbyte:A%2Dbyte:A%2D[decode](byte[], byte[])++ (link:{java9-javadoc}/java/util/Base64$Decoder.html#decode%2Dbyte:A%2Dbyte:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Base64.Encoder.asciidoc b/docs/reference/painless-api-reference/Base64.Encoder.asciidoc
new file mode 100644
index 00000000000..bc632fc7995
--- /dev/null
+++ b/docs/reference/painless-api-reference/Base64.Encoder.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64-Encoder]]++Base64.Encoder++::
+* ++[[painless-api-reference-Base64-Encoder-encode-2]]int link:{java8-javadoc}/java/util/Base64$Encoder.html#encode%2Dbyte:A%2Dbyte:A%2D[encode](byte[], byte[])++ (link:{java9-javadoc}/java/util/Base64$Encoder.html#encode%2Dbyte:A%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-Encoder-encodeToString-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Base64$Encoder.html#encodeToString%2Dbyte:A%2D[encodeToString](byte[])++ (link:{java9-javadoc}/java/util/Base64$Encoder.html#encodeToString%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-Encoder-withoutPadding-0]]<<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64$Encoder.html#withoutPadding%2D%2D[withoutPadding]()++ (link:{java9-javadoc}/java/util/Base64$Encoder.html#withoutPadding%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Base64.asciidoc b/docs/reference/painless-api-reference/Base64.asciidoc
new file mode 100644
index 00000000000..795f538d7d4
--- /dev/null
+++ b/docs/reference/painless-api-reference/Base64.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Base64]]++Base64++::
+* ++[[painless-api-reference-Base64-getDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getDecoder%2D%2D[getDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getEncoder%2D%2D[getEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getEncoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeDecoder%2D%2D[getMimeDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getMimeDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeEncoder%2D%2D[getMimeEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getMimeEncoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getMimeEncoder-2]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getMimeEncoder%2Dint%2Dbyte:A%2D[getMimeEncoder](int, byte[])++ (link:{java9-javadoc}/java/util/Base64.html#getMimeEncoder%2Dint%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Base64-getUrlDecoder-0]]static <<painless-api-reference-Base64-Decoder,Base64.Decoder>> link:{java8-javadoc}/java/util/Base64.html#getUrlDecoder%2D%2D[getUrlDecoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getUrlDecoder%2D%2D[java 9])
+* ++[[painless-api-reference-Base64-getUrlEncoder-0]]static <<painless-api-reference-Base64-Encoder,Base64.Encoder>> link:{java8-javadoc}/java/util/Base64.html#getUrlEncoder%2D%2D[getUrlEncoder]()++ (link:{java9-javadoc}/java/util/Base64.html#getUrlEncoder%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BaseStream.asciidoc b/docs/reference/painless-api-reference/BaseStream.asciidoc
new file mode 100644
index 00000000000..680e9fff71d
--- /dev/null
+++ b/docs/reference/painless-api-reference/BaseStream.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BaseStream]]++BaseStream++::
+* ++[[painless-api-reference-BaseStream-close-0]]void link:{java8-javadoc}/java/util/stream/BaseStream.html#close%2D%2D[close]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#close%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-isParallel-0]]boolean link:{java8-javadoc}/java/util/stream/BaseStream.html#isParallel%2D%2D[isParallel]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#isParallel%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-iterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/stream/BaseStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-sequential-0]]<<painless-api-reference-BaseStream,BaseStream>> link:{java8-javadoc}/java/util/stream/BaseStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/stream/BaseStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-BaseStream-unordered-0]]<<painless-api-reference-BaseStream,BaseStream>> link:{java8-javadoc}/java/util/stream/BaseStream.html#unordered%2D%2D[unordered]()++ (link:{java9-javadoc}/java/util/stream/BaseStream.html#unordered%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BiConsumer.asciidoc b/docs/reference/painless-api-reference/BiConsumer.asciidoc
new file mode 100644
index 00000000000..a13bbdbd300
--- /dev/null
+++ b/docs/reference/painless-api-reference/BiConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiConsumer]]++BiConsumer++::
+* ++[[painless-api-reference-BiConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/BiConsumer.html#accept%2Djava.lang.Object%2Djava.lang.Object%2D[accept](def, def)++ (link:{java9-javadoc}/java/util/function/BiConsumer.html#accept%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-BiConsumer-andThen-1]]<<painless-api-reference-BiConsumer,BiConsumer>> link:{java8-javadoc}/java/util/function/BiConsumer.html#andThen%2Djava.util.function.BiConsumer%2D[andThen](<<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/function/BiConsumer.html#andThen%2Djava.util.function.BiConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BiFunction.asciidoc b/docs/reference/painless-api-reference/BiFunction.asciidoc
new file mode 100644
index 00000000000..88b31630032
--- /dev/null
+++ b/docs/reference/painless-api-reference/BiFunction.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiFunction]]++BiFunction++::
+* ++[[painless-api-reference-BiFunction-andThen-1]]<<painless-api-reference-BiFunction,BiFunction>> link:{java8-javadoc}/java/util/function/BiFunction.html#andThen%2Djava.util.function.Function%2D[andThen](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/BiFunction.html#andThen%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-BiFunction-apply-2]]def link:{java8-javadoc}/java/util/function/BiFunction.html#apply%2Djava.lang.Object%2Djava.lang.Object%2D[apply](def, def)++ (link:{java9-javadoc}/java/util/function/BiFunction.html#apply%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BiPredicate.asciidoc b/docs/reference/painless-api-reference/BiPredicate.asciidoc
new file mode 100644
index 00000000000..7961800814c
--- /dev/null
+++ b/docs/reference/painless-api-reference/BiPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BiPredicate]]++BiPredicate++::
+* ++[[painless-api-reference-BiPredicate-and-1]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#and%2Djava.util.function.BiPredicate%2D[and](<<painless-api-reference-BiPredicate,BiPredicate>>)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#and%2Djava.util.function.BiPredicate%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-negate-0]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-or-1]]<<painless-api-reference-BiPredicate,BiPredicate>> link:{java8-javadoc}/java/util/function/BiPredicate.html#or%2Djava.util.function.BiPredicate%2D[or](<<painless-api-reference-BiPredicate,BiPredicate>>)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#or%2Djava.util.function.BiPredicate%2D[java 9])
+* ++[[painless-api-reference-BiPredicate-test-2]]boolean link:{java8-javadoc}/java/util/function/BiPredicate.html#test%2Djava.lang.Object%2Djava.lang.Object%2D[test](def, def)++ (link:{java9-javadoc}/java/util/function/BiPredicate.html#test%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Bidi.asciidoc b/docs/reference/painless-api-reference/Bidi.asciidoc
new file mode 100644
index 00000000000..cf6272dd49c
--- /dev/null
+++ b/docs/reference/painless-api-reference/Bidi.asciidoc
@@ -0,0 +1,28 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Bidi]]++Bidi++::
+** [[painless-api-reference-Bidi-DIRECTION_DEFAULT_LEFT_TO_RIGHT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_LEFT_TO_RIGHT[DIRECTION_DEFAULT_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_DEFAULT_RIGHT_TO_LEFT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_RIGHT_TO_LEFT[DIRECTION_DEFAULT_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_DEFAULT_RIGHT_TO_LEFT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_LEFT_TO_RIGHT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_LEFT_TO_RIGHT[DIRECTION_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Bidi-DIRECTION_RIGHT_TO_LEFT]]static int link:{java8-javadoc}/java/text/Bidi.html#DIRECTION_RIGHT_TO_LEFT[DIRECTION_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/text/Bidi.html#DIRECTION_RIGHT_TO_LEFT[java 9])
+* ++[[painless-api-reference-Bidi-reorderVisually-5]]static void link:{java8-javadoc}/java/text/Bidi.html#reorderVisually%2Dbyte:A%2Dint%2Djava.lang.Object:A%2Dint%2Dint%2D[reorderVisually](byte[], int, <<painless-api-reference-Object,Object>>[], int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#reorderVisually%2Dbyte:A%2Dint%2Djava.lang.Object:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-requiresBidi-3]]static boolean link:{java8-javadoc}/java/text/Bidi.html#requiresBidi%2Dchar:A%2Dint%2Dint%2D[requiresBidi](char[], int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#requiresBidi%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-1]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Djava.text.AttributedCharacterIterator%2D[Bidi](<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>>)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Djava.text.AttributedCharacterIterator%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-2]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Djava.lang.String%2Dint%2D[Bidi](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-Bidi-6]]link:{java8-javadoc}/java/text/Bidi.html#Bidi%2Dchar:A%2Dint%2Dbyte:A%2Dint%2Dint%2Dint%2D[Bidi](char[], int, byte[], int, int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#Bidi%2Dchar:A%2Dint%2Dbyte:A%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-baseIsLeftToRight-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#baseIsLeftToRight%2D%2D[baseIsLeftToRight]()++ (link:{java9-javadoc}/java/text/Bidi.html#baseIsLeftToRight%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-createLineBidi-2]]<<painless-api-reference-Bidi,Bidi>> link:{java8-javadoc}/java/text/Bidi.html#createLineBidi%2Dint%2Dint%2D[createLineBidi](int, int)++ (link:{java9-javadoc}/java/text/Bidi.html#createLineBidi%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getBaseLevel-0]]int link:{java8-javadoc}/java/text/Bidi.html#getBaseLevel%2D%2D[getBaseLevel]()++ (link:{java9-javadoc}/java/text/Bidi.html#getBaseLevel%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getLength-0]]int link:{java8-javadoc}/java/text/Bidi.html#getLength%2D%2D[getLength]()++ (link:{java9-javadoc}/java/text/Bidi.html#getLength%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getLevelAt-1]]int link:{java8-javadoc}/java/text/Bidi.html#getLevelAt%2Dint%2D[getLevelAt](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getLevelAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunCount-0]]int link:{java8-javadoc}/java/text/Bidi.html#getRunCount%2D%2D[getRunCount]()++ (link:{java9-javadoc}/java/text/Bidi.html#getRunCount%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunLevel-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunLevel%2Dint%2D[getRunLevel](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunLevel%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunLimit-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunLimit%2Dint%2D[getRunLimit](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunLimit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-getRunStart-1]]int link:{java8-javadoc}/java/text/Bidi.html#getRunStart%2Dint%2D[getRunStart](int)++ (link:{java9-javadoc}/java/text/Bidi.html#getRunStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Bidi-isLeftToRight-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isLeftToRight%2D%2D[isLeftToRight]()++ (link:{java9-javadoc}/java/text/Bidi.html#isLeftToRight%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-isMixed-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isMixed%2D%2D[isMixed]()++ (link:{java9-javadoc}/java/text/Bidi.html#isMixed%2D%2D[java 9])
+* ++[[painless-api-reference-Bidi-isRightToLeft-0]]boolean link:{java8-javadoc}/java/text/Bidi.html#isRightToLeft%2D%2D[isRightToLeft]()++ (link:{java9-javadoc}/java/text/Bidi.html#isRightToLeft%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BigDecimal.asciidoc b/docs/reference/painless-api-reference/BigDecimal.asciidoc
new file mode 100644
index 00000000000..ab8a78f35c1
--- /dev/null
+++ b/docs/reference/painless-api-reference/BigDecimal.asciidoc
@@ -0,0 +1,57 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BigDecimal]]++BigDecimal++::
+** [[painless-api-reference-BigDecimal-ONE]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ONE[ONE] (link:{java9-javadoc}/java/math/BigDecimal.html#ONE[java 9])
+** [[painless-api-reference-BigDecimal-TEN]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#TEN[TEN] (link:{java9-javadoc}/java/math/BigDecimal.html#TEN[java 9])
+** [[painless-api-reference-BigDecimal-ZERO]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ZERO[ZERO] (link:{java9-javadoc}/java/math/BigDecimal.html#ZERO[java 9])
+* ++[[painless-api-reference-BigDecimal-valueOf-1]]static <<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#valueOf%2Ddouble%2D[valueOf](double)++ (link:{java9-javadoc}/java/math/BigDecimal.html#valueOf%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-BigDecimal-1]]link:{java8-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2D[BigDecimal](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-BigDecimal-2]]link:{java8-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2Djava.math.MathContext%2D[BigDecimal](<<painless-api-reference-String,String>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#BigDecimal%2Djava.lang.String%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-abs-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-abs-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#abs%2Djava.math.MathContext%2D[abs](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#abs%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-add-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2D[add](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-add-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[add](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#add%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-byteValueExact-0]]byte link:{java8-javadoc}/java/math/BigDecimal.html#byteValueExact%2D%2D[byteValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#byteValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-compareTo-1]]int link:{java8-javadoc}/java/math/BigDecimal.html#compareTo%2Djava.math.BigDecimal%2D[compareTo](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#compareTo%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divide-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2D[divide](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divide-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divide](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divide%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideAndRemainder-1]]<<painless-api-reference-BigDecimal,BigDecimal>>[] link:{java8-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2D[divideAndRemainder](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideAndRemainder-2]]<<painless-api-reference-BigDecimal,BigDecimal>>[] link:{java8-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divideAndRemainder](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideAndRemainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideToIntegralValue-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2D[divideToIntegralValue](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-divideToIntegralValue-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[divideToIntegralValue](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#divideToIntegralValue%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-intValueExact-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#intValueExact%2D%2D[intValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#intValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-longValueExact-0]]long link:{java8-javadoc}/java/math/BigDecimal.html#longValueExact%2D%2D[longValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#longValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-max-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#max%2Djava.math.BigDecimal%2D[max](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#max%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-min-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#min%2Djava.math.BigDecimal%2D[min](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#min%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-movePointLeft-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#movePointLeft%2Dint%2D[movePointLeft](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#movePointLeft%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-movePointRight-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#movePointRight%2Dint%2D[movePointRight](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#movePointRight%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-multiply-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2D[multiply](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-multiply-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[multiply](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#multiply%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-negate-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-negate-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#negate%2Djava.math.MathContext%2D[negate](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#negate%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-plus-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#plus%2D%2D[plus]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#plus%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-plus-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#plus%2Djava.math.MathContext%2D[plus](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#plus%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-pow-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#pow%2Dint%2D[pow](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#pow%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-pow-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#pow%2Dint%2Djava.math.MathContext%2D[pow](int, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#pow%2Dint%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-precision-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#precision%2D%2D[precision]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#precision%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-remainder-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2D[remainder](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-remainder-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[remainder](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#remainder%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-round-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#round%2Djava.math.MathContext%2D[round](<<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#round%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-scale-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#scale%2D%2D[scale]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#scale%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-scaleByPowerOfTen-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#scaleByPowerOfTen%2Dint%2D[scaleByPowerOfTen](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#scaleByPowerOfTen%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-setScale-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2D[setScale](int)++ (link:{java9-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-setScale-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2Djava.math.RoundingMode%2D[setScale](int, <<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#setScale%2Dint%2Djava.math.RoundingMode%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-shortValueExact-0]]short link:{java8-javadoc}/java/math/BigDecimal.html#shortValueExact%2D%2D[shortValueExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#shortValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-signum-0]]int link:{java8-javadoc}/java/math/BigDecimal.html#signum%2D%2D[signum]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#signum%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-stripTrailingZeros-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#stripTrailingZeros%2D%2D[stripTrailingZeros]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#stripTrailingZeros%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-subtract-1]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2D[subtract](<<painless-api-reference-BigDecimal,BigDecimal>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-subtract-2]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[subtract](<<painless-api-reference-BigDecimal,BigDecimal>>, <<painless-api-reference-MathContext,MathContext>>)++ (link:{java9-javadoc}/java/math/BigDecimal.html#subtract%2Djava.math.BigDecimal%2Djava.math.MathContext%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toBigInteger-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigDecimal.html#toBigInteger%2D%2D[toBigInteger]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toBigInteger%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toBigIntegerExact-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigDecimal.html#toBigIntegerExact%2D%2D[toBigIntegerExact]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toBigIntegerExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toEngineeringString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigDecimal.html#toEngineeringString%2D%2D[toEngineeringString]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toEngineeringString%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-toPlainString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigDecimal.html#toPlainString%2D%2D[toPlainString]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#toPlainString%2D%2D[java 9])
+* ++[[painless-api-reference-BigDecimal-ulp-0]]<<painless-api-reference-BigDecimal,BigDecimal>> link:{java8-javadoc}/java/math/BigDecimal.html#ulp%2D%2D[ulp]()++ (link:{java9-javadoc}/java/math/BigDecimal.html#ulp%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BigInteger.asciidoc b/docs/reference/painless-api-reference/BigInteger.asciidoc
new file mode 100644
index 00000000000..796355cadd3
--- /dev/null
+++ b/docs/reference/painless-api-reference/BigInteger.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BigInteger]]++BigInteger++::
+** [[painless-api-reference-BigInteger-ONE]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#ONE[ONE] (link:{java9-javadoc}/java/math/BigInteger.html#ONE[java 9])
+** [[painless-api-reference-BigInteger-TEN]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#TEN[TEN] (link:{java9-javadoc}/java/math/BigInteger.html#TEN[java 9])
+** [[painless-api-reference-BigInteger-ZERO]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#ZERO[ZERO] (link:{java9-javadoc}/java/math/BigInteger.html#ZERO[java 9])
+* ++[[painless-api-reference-BigInteger-valueOf-1]]static <<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#valueOf%2Dlong%2D[valueOf](long)++ (link:{java9-javadoc}/java/math/BigInteger.html#valueOf%2Dlong%2D[java 9])
+* ++[[painless-api-reference-BigInteger-BigInteger-1]]link:{java8-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2D[BigInteger](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-BigInteger-BigInteger-2]]link:{java8-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2Dint%2D[BigInteger](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/math/BigInteger.html#BigInteger%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-abs-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/math/BigInteger.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-add-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#add%2Djava.math.BigInteger%2D[add](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#add%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-and-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#and%2Djava.math.BigInteger%2D[and](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#and%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-andNot-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#andNot%2Djava.math.BigInteger%2D[andNot](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#andNot%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-bitCount-0]]int link:{java8-javadoc}/java/math/BigInteger.html#bitCount%2D%2D[bitCount]()++ (link:{java9-javadoc}/java/math/BigInteger.html#bitCount%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-bitLength-0]]int link:{java8-javadoc}/java/math/BigInteger.html#bitLength%2D%2D[bitLength]()++ (link:{java9-javadoc}/java/math/BigInteger.html#bitLength%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-byteValueExact-0]]byte link:{java8-javadoc}/java/math/BigInteger.html#byteValueExact%2D%2D[byteValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#byteValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-clearBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#clearBit%2Dint%2D[clearBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#clearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-compareTo-1]]int link:{java8-javadoc}/java/math/BigInteger.html#compareTo%2Djava.math.BigInteger%2D[compareTo](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#compareTo%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-divide-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#divide%2Djava.math.BigInteger%2D[divide](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#divide%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-divideAndRemainder-1]]<<painless-api-reference-BigInteger,BigInteger>>[] link:{java8-javadoc}/java/math/BigInteger.html#divideAndRemainder%2Djava.math.BigInteger%2D[divideAndRemainder](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#divideAndRemainder%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-flipBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#flipBit%2Dint%2D[flipBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#flipBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-gcd-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#gcd%2Djava.math.BigInteger%2D[gcd](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#gcd%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-getLowestSetBit-0]]int link:{java8-javadoc}/java/math/BigInteger.html#getLowestSetBit%2D%2D[getLowestSetBit]()++ (link:{java9-javadoc}/java/math/BigInteger.html#getLowestSetBit%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-intValueExact-0]]int link:{java8-javadoc}/java/math/BigInteger.html#intValueExact%2D%2D[intValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#intValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-longValueExact-0]]long link:{java8-javadoc}/java/math/BigInteger.html#longValueExact%2D%2D[longValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#longValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-max-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#max%2Djava.math.BigInteger%2D[max](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#max%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-min-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#min%2Djava.math.BigInteger%2D[min](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#min%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-mod-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#mod%2Djava.math.BigInteger%2D[mod](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#mod%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-modInverse-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#modInverse%2Djava.math.BigInteger%2D[modInverse](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#modInverse%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-modPow-2]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#modPow%2Djava.math.BigInteger%2Djava.math.BigInteger%2D[modPow](<<painless-api-reference-BigInteger,BigInteger>>, <<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#modPow%2Djava.math.BigInteger%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-multiply-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#multiply%2Djava.math.BigInteger%2D[multiply](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#multiply%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-negate-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/math/BigInteger.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-not-0]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#not%2D%2D[not]()++ (link:{java9-javadoc}/java/math/BigInteger.html#not%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-or-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#or%2Djava.math.BigInteger%2D[or](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#or%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-pow-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#pow%2Dint%2D[pow](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#pow%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-remainder-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#remainder%2Djava.math.BigInteger%2D[remainder](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#remainder%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-setBit-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#setBit%2Dint%2D[setBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#setBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shiftLeft-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#shiftLeft%2Dint%2D[shiftLeft](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#shiftLeft%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shiftRight-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#shiftRight%2Dint%2D[shiftRight](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#shiftRight%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-shortValueExact-0]]short link:{java8-javadoc}/java/math/BigInteger.html#shortValueExact%2D%2D[shortValueExact]()++ (link:{java9-javadoc}/java/math/BigInteger.html#shortValueExact%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-signum-0]]int link:{java8-javadoc}/java/math/BigInteger.html#signum%2D%2D[signum]()++ (link:{java9-javadoc}/java/math/BigInteger.html#signum%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-subtract-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#subtract%2Djava.math.BigInteger%2D[subtract](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#subtract%2Djava.math.BigInteger%2D[java 9])
+* ++[[painless-api-reference-BigInteger-testBit-1]]boolean link:{java8-javadoc}/java/math/BigInteger.html#testBit%2Dint%2D[testBit](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#testBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-toByteArray-0]]byte[] link:{java8-javadoc}/java/math/BigInteger.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/math/BigInteger.html#toByteArray%2D%2D[java 9])
+* ++[[painless-api-reference-BigInteger-toString-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/math/BigInteger.html#toString%2Dint%2D[toString](int)++ (link:{java9-javadoc}/java/math/BigInteger.html#toString%2Dint%2D[java 9])
+* ++[[painless-api-reference-BigInteger-xor-1]]<<painless-api-reference-BigInteger,BigInteger>> link:{java8-javadoc}/java/math/BigInteger.html#xor%2Djava.math.BigInteger%2D[xor](<<painless-api-reference-BigInteger,BigInteger>>)++ (link:{java9-javadoc}/java/math/BigInteger.html#xor%2Djava.math.BigInteger%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BinaryOperator.asciidoc b/docs/reference/painless-api-reference/BinaryOperator.asciidoc
new file mode 100644
index 00000000000..c7e53be47f7
--- /dev/null
+++ b/docs/reference/painless-api-reference/BinaryOperator.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BinaryOperator]]++BinaryOperator++::
+* ++[[painless-api-reference-BinaryOperator-maxBy-1]]static <<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/function/BinaryOperator.html#maxBy%2Djava.util.Comparator%2D[maxBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/function/BinaryOperator.html#maxBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-BinaryOperator-minBy-1]]static <<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/function/BinaryOperator.html#minBy%2Djava.util.Comparator%2D[minBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/function/BinaryOperator.html#minBy%2Djava.util.Comparator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BiFunction,BiFunction>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BitSet.asciidoc b/docs/reference/painless-api-reference/BitSet.asciidoc
new file mode 100644
index 00000000000..158f64675d9
--- /dev/null
+++ b/docs/reference/painless-api-reference/BitSet.asciidoc
@@ -0,0 +1,34 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BitSet]]++BitSet++::
+* ++[[painless-api-reference-BitSet-valueOf-1]]static <<painless-api-reference-BitSet,BitSet>> link:{java8-javadoc}/java/util/BitSet.html#valueOf%2Dlong:A%2D[valueOf](long[])++ (link:{java9-javadoc}/java/util/BitSet.html#valueOf%2Dlong:A%2D[java 9])
+* ++[[painless-api-reference-BitSet-BitSet-0]]link:{java8-javadoc}/java/util/BitSet.html#BitSet%2D%2D[BitSet]()++ (link:{java9-javadoc}/java/util/BitSet.html#BitSet%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-BitSet-1]]link:{java8-javadoc}/java/util/BitSet.html#BitSet%2Dint%2D[BitSet](int)++ (link:{java9-javadoc}/java/util/BitSet.html#BitSet%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-and-1]]void link:{java8-javadoc}/java/util/BitSet.html#and%2Djava.util.BitSet%2D[and](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#and%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-andNot-1]]void link:{java8-javadoc}/java/util/BitSet.html#andNot%2Djava.util.BitSet%2D[andNot](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#andNot%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-cardinality-0]]int link:{java8-javadoc}/java/util/BitSet.html#cardinality%2D%2D[cardinality]()++ (link:{java9-javadoc}/java/util/BitSet.html#cardinality%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-0]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-1]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2Dint%2D[clear](int)++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-clear-2]]void link:{java8-javadoc}/java/util/BitSet.html#clear%2Dint%2Dint%2D[clear](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#clear%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-clone-0]]def link:{java8-javadoc}/java/util/BitSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/BitSet.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-flip-1]]void link:{java8-javadoc}/java/util/BitSet.html#flip%2Dint%2D[flip](int)++ (link:{java9-javadoc}/java/util/BitSet.html#flip%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-flip-2]]void link:{java8-javadoc}/java/util/BitSet.html#flip%2Dint%2Dint%2D[flip](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#flip%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-intersects-1]]boolean link:{java8-javadoc}/java/util/BitSet.html#intersects%2Djava.util.BitSet%2D[intersects](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#intersects%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-isEmpty-0]]boolean link:{java8-javadoc}/java/util/BitSet.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/BitSet.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-length-0]]int link:{java8-javadoc}/java/util/BitSet.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/util/BitSet.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-nextClearBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#nextClearBit%2Dint%2D[nextClearBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#nextClearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-nextSetBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#nextSetBit%2Dint%2D[nextSetBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#nextSetBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-or-1]]void link:{java8-javadoc}/java/util/BitSet.html#or%2Djava.util.BitSet%2D[or](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#or%2Djava.util.BitSet%2D[java 9])
+* ++[[painless-api-reference-BitSet-previousClearBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#previousClearBit%2Dint%2D[previousClearBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#previousClearBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-previousSetBit-1]]int link:{java8-javadoc}/java/util/BitSet.html#previousSetBit%2Dint%2D[previousSetBit](int)++ (link:{java9-javadoc}/java/util/BitSet.html#previousSetBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-1]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2D[set](int)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-2]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-BitSet-set-3]]void link:{java8-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2Dboolean%2D[set](int, int, boolean)++ (link:{java9-javadoc}/java/util/BitSet.html#set%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-BitSet-size-0]]int link:{java8-javadoc}/java/util/BitSet.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/BitSet.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-toByteArray-0]]byte[] link:{java8-javadoc}/java/util/BitSet.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/util/BitSet.html#toByteArray%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-toLongArray-0]]long[] link:{java8-javadoc}/java/util/BitSet.html#toLongArray%2D%2D[toLongArray]()++ (link:{java9-javadoc}/java/util/BitSet.html#toLongArray%2D%2D[java 9])
+* ++[[painless-api-reference-BitSet-xor-1]]void link:{java8-javadoc}/java/util/BitSet.html#xor%2Djava.util.BitSet%2D[xor](<<painless-api-reference-BitSet,BitSet>>)++ (link:{java9-javadoc}/java/util/BitSet.html#xor%2Djava.util.BitSet%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Boolean.asciidoc b/docs/reference/painless-api-reference/Boolean.asciidoc
new file mode 100644
index 00000000000..47000e58fa8
--- /dev/null
+++ b/docs/reference/painless-api-reference/Boolean.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Boolean]]++Boolean++::
+** [[painless-api-reference-Boolean-FALSE]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#FALSE[FALSE] (link:{java9-javadoc}/java/lang/Boolean.html#FALSE[java 9])
+** [[painless-api-reference-Boolean-TRUE]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#TRUE[TRUE] (link:{java9-javadoc}/java/lang/Boolean.html#TRUE[java 9])
+* ++[[painless-api-reference-Boolean-compare-2]]static int link:{java8-javadoc}/java/lang/Boolean.html#compare%2Dboolean%2Dboolean%2D[compare](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#compare%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-hashCode-1]]static int link:{java8-javadoc}/java/lang/Boolean.html#hashCode%2Dboolean%2D[hashCode](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#hashCode%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalAnd-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalAnd%2Dboolean%2Dboolean%2D[logicalAnd](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalAnd%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalOr-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalOr%2Dboolean%2Dboolean%2D[logicalOr](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalOr%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-logicalXor-2]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#logicalXor%2Dboolean%2Dboolean%2D[logicalXor](boolean, boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#logicalXor%2Dboolean%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-parseBoolean-1]]static boolean link:{java8-javadoc}/java/lang/Boolean.html#parseBoolean%2Djava.lang.String%2D[parseBoolean](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Boolean.html#parseBoolean%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Boolean-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Boolean.html#toString%2Dboolean%2D[toString](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#toString%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-valueOf-1]]static <<painless-api-reference-Boolean,Boolean>> link:{java8-javadoc}/java/lang/Boolean.html#valueOf%2Dboolean%2D[valueOf](boolean)++ (link:{java9-javadoc}/java/lang/Boolean.html#valueOf%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Boolean-booleanValue-0]]boolean link:{java8-javadoc}/java/lang/Boolean.html#booleanValue%2D%2D[booleanValue]()++ (link:{java9-javadoc}/java/lang/Boolean.html#booleanValue%2D%2D[java 9])
+* ++[[painless-api-reference-Boolean-compareTo-1]]int link:{java8-javadoc}/java/lang/Boolean.html#compareTo%2Djava.lang.Boolean%2D[compareTo](<<painless-api-reference-Boolean,Boolean>>)++ (link:{java9-javadoc}/java/lang/Boolean.html#compareTo%2Djava.lang.Boolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BooleanSupplier.asciidoc b/docs/reference/painless-api-reference/BooleanSupplier.asciidoc
new file mode 100644
index 00000000000..9a34d1caf2d
--- /dev/null
+++ b/docs/reference/painless-api-reference/BooleanSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BooleanSupplier]]++BooleanSupplier++::
+* ++[[painless-api-reference-BooleanSupplier-getAsBoolean-0]]boolean link:{java8-javadoc}/java/util/function/BooleanSupplier.html#getAsBoolean%2D%2D[getAsBoolean]()++ (link:{java9-javadoc}/java/util/function/BooleanSupplier.html#getAsBoolean%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BreakIterator.asciidoc b/docs/reference/painless-api-reference/BreakIterator.asciidoc
new file mode 100644
index 00000000000..0e830859c45
--- /dev/null
+++ b/docs/reference/painless-api-reference/BreakIterator.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BreakIterator]]++BreakIterator++::
+** [[painless-api-reference-BreakIterator-DONE]]static int link:{java8-javadoc}/java/text/BreakIterator.html#DONE[DONE] (link:{java9-javadoc}/java/text/BreakIterator.html#DONE[java 9])
+* ++[[painless-api-reference-BreakIterator-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/BreakIterator.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getCharacterInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2D%2D[getCharacterInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getCharacterInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2Djava.util.Locale%2D[getCharacterInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getCharacterInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getLineInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getLineInstance%2D%2D[getLineInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getLineInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getLineInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getLineInstance%2Djava.util.Locale%2D[getLineInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getLineInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getSentenceInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2D%2D[getSentenceInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getSentenceInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2Djava.util.Locale%2D[getSentenceInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getSentenceInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getWordInstance-0]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getWordInstance%2D%2D[getWordInstance]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getWordInstance%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getWordInstance-1]]static <<painless-api-reference-BreakIterator,BreakIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getWordInstance%2Djava.util.Locale%2D[getWordInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#getWordInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-clone-0]]def link:{java8-javadoc}/java/text/BreakIterator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-current-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#current%2D%2D[current]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#current%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-first-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-following-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#following%2Dint%2D[following](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#following%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-getText-0]]<<painless-api-reference-CharacterIterator,CharacterIterator>> link:{java8-javadoc}/java/text/BreakIterator.html#getText%2D%2D[getText]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#getText%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-isBoundary-1]]boolean link:{java8-javadoc}/java/text/BreakIterator.html#isBoundary%2Dint%2D[isBoundary](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#isBoundary%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-last-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-next-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-next-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#next%2Dint%2D[next](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#next%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-preceding-1]]int link:{java8-javadoc}/java/text/BreakIterator.html#preceding%2Dint%2D[preceding](int)++ (link:{java9-javadoc}/java/text/BreakIterator.html#preceding%2Dint%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-previous-0]]int link:{java8-javadoc}/java/text/BreakIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/BreakIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-BreakIterator-setText-1]]void link:{java8-javadoc}/java/text/BreakIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/BreakIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Byte.asciidoc b/docs/reference/painless-api-reference/Byte.asciidoc
new file mode 100644
index 00000000000..2fc42b63625
--- /dev/null
+++ b/docs/reference/painless-api-reference/Byte.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Byte]]++Byte++::
+** [[painless-api-reference-Byte-BYTES]]static int link:{java8-javadoc}/java/lang/Byte.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Byte.html#BYTES[java 9])
+** [[painless-api-reference-Byte-MAX_VALUE]]static byte link:{java8-javadoc}/java/lang/Byte.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Byte.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Byte-MIN_VALUE]]static byte link:{java8-javadoc}/java/lang/Byte.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Byte.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Byte-SIZE]]static int link:{java8-javadoc}/java/lang/Byte.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Byte.html#SIZE[java 9])
+* ++[[painless-api-reference-Byte-compare-2]]static int link:{java8-javadoc}/java/lang/Byte.html#compare%2Dbyte%2Dbyte%2D[compare](byte, byte)++ (link:{java9-javadoc}/java/lang/Byte.html#compare%2Dbyte%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-decode-1]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Byte.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Byte-hashCode-1]]static int link:{java8-javadoc}/java/lang/Byte.html#hashCode%2Dbyte%2D[hashCode](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#hashCode%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-parseByte-1]]static byte link:{java8-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2D[parseByte](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Byte-parseByte-2]]static byte link:{java8-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2Dint%2D[parseByte](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Byte.html#parseByte%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Byte-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Byte.html#toString%2Dbyte%2D[toString](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toString%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-toUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Byte.html#toUnsignedInt%2Dbyte%2D[toUnsignedInt](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toUnsignedInt%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Byte.html#toUnsignedLong%2Dbyte%2D[toUnsignedLong](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#toUnsignedLong%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-valueOf-1]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#valueOf%2Dbyte%2D[valueOf](byte)++ (link:{java9-javadoc}/java/lang/Byte.html#valueOf%2Dbyte%2D[java 9])
+* ++[[painless-api-reference-Byte-valueOf-2]]static <<painless-api-reference-Byte,Byte>> link:{java8-javadoc}/java/lang/Byte.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Byte.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Byte-compareTo-1]]int link:{java8-javadoc}/java/lang/Byte.html#compareTo%2Djava.lang.Byte%2D[compareTo](<<painless-api-reference-Byte,Byte>>)++ (link:{java9-javadoc}/java/lang/Byte.html#compareTo%2Djava.lang.Byte%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/BytesRef.asciidoc b/docs/reference/painless-api-reference/BytesRef.asciidoc
new file mode 100644
index 00000000000..6114a5625fc
--- /dev/null
+++ b/docs/reference/painless-api-reference/BytesRef.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-BytesRef]]++BytesRef++::
+** [[painless-api-reference-BytesRef-bytes]]byte[] link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#bytes[bytes]
+** [[painless-api-reference-BytesRef-length]]int link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#length[length]
+** [[painless-api-reference-BytesRef-offset]]int link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#offset[offset]
+* ++[[painless-api-reference-BytesRef-bytesEquals-1]]boolean link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#bytesEquals%2Dorg.apache.lucene.util.BytesRef%2D[bytesEquals](<<painless-api-reference-BytesRef,BytesRef>>)++
+* ++[[painless-api-reference-BytesRef-utf8ToString-0]]<<painless-api-reference-String,String>> link:{lucene-core-javadoc}/org/apache/lucene/util/BytesRef.html#utf8ToString%2D%2D[utf8ToString]()++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Calendar.Builder.asciidoc b/docs/reference/painless-api-reference/Calendar.Builder.asciidoc
new file mode 100644
index 00000000000..2fd4cafa036
--- /dev/null
+++ b/docs/reference/painless-api-reference/Calendar.Builder.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Calendar-Builder]]++Calendar.Builder++::
+* ++[[painless-api-reference-Calendar-Builder-Calendar.Builder-0]]link:{java8-javadoc}/java/util/Calendar$Builder.html#Calendar.Builder%2D%2D[Calendar.Builder]()++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#Calendar.Builder%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-build-0]]<<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#build%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-set-2]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setCalendarType-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setCalendarType%2Djava.lang.String%2D[setCalendarType](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setCalendarType%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setDate-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setDate%2Dint%2Dint%2Dint%2D[setDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setFields-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setFields%2Dint:A%2D[setFields](int[])++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setFields%2Dint:A%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setInstant-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setInstant%2Dlong%2D[setInstant](long)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setInstant%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setLenient-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setLocale-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeOfDay-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2D[setTimeOfDay](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeOfDay-4]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2Dint%2D[setTimeOfDay](int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setTimeOfDay%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setTimeZone-1]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setWeekDate-3]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setWeekDate%2Dint%2Dint%2Dint%2D[setWeekDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setWeekDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-Builder-setWeekDefinition-2]]<<painless-api-reference-Calendar-Builder,Calendar.Builder>> link:{java8-javadoc}/java/util/Calendar$Builder.html#setWeekDefinition%2Dint%2Dint%2D[setWeekDefinition](int, int)++ (link:{java9-javadoc}/java/util/Calendar$Builder.html#setWeekDefinition%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Calendar.asciidoc b/docs/reference/painless-api-reference/Calendar.asciidoc
new file mode 100644
index 00000000000..e1a489d8c87
--- /dev/null
+++ b/docs/reference/painless-api-reference/Calendar.asciidoc
@@ -0,0 +1,102 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Calendar]]++Calendar++::
+** [[painless-api-reference-Calendar-ALL_STYLES]]static int link:{java8-javadoc}/java/util/Calendar.html#ALL_STYLES[ALL_STYLES] (link:{java9-javadoc}/java/util/Calendar.html#ALL_STYLES[java 9])
+** [[painless-api-reference-Calendar-AM]]static int link:{java8-javadoc}/java/util/Calendar.html#AM[AM] (link:{java9-javadoc}/java/util/Calendar.html#AM[java 9])
+** [[painless-api-reference-Calendar-AM_PM]]static int link:{java8-javadoc}/java/util/Calendar.html#AM_PM[AM_PM] (link:{java9-javadoc}/java/util/Calendar.html#AM_PM[java 9])
+** [[painless-api-reference-Calendar-APRIL]]static int link:{java8-javadoc}/java/util/Calendar.html#APRIL[APRIL] (link:{java9-javadoc}/java/util/Calendar.html#APRIL[java 9])
+** [[painless-api-reference-Calendar-AUGUST]]static int link:{java8-javadoc}/java/util/Calendar.html#AUGUST[AUGUST] (link:{java9-javadoc}/java/util/Calendar.html#AUGUST[java 9])
+** [[painless-api-reference-Calendar-DATE]]static int link:{java8-javadoc}/java/util/Calendar.html#DATE[DATE] (link:{java9-javadoc}/java/util/Calendar.html#DATE[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_WEEK]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_WEEK_IN_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_WEEK_IN_MONTH[DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-Calendar-DAY_OF_YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/util/Calendar.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-Calendar-DECEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#DECEMBER[DECEMBER] (link:{java9-javadoc}/java/util/Calendar.html#DECEMBER[java 9])
+** [[painless-api-reference-Calendar-DST_OFFSET]]static int link:{java8-javadoc}/java/util/Calendar.html#DST_OFFSET[DST_OFFSET] (link:{java9-javadoc}/java/util/Calendar.html#DST_OFFSET[java 9])
+** [[painless-api-reference-Calendar-ERA]]static int link:{java8-javadoc}/java/util/Calendar.html#ERA[ERA] (link:{java9-javadoc}/java/util/Calendar.html#ERA[java 9])
+** [[painless-api-reference-Calendar-FEBRUARY]]static int link:{java8-javadoc}/java/util/Calendar.html#FEBRUARY[FEBRUARY] (link:{java9-javadoc}/java/util/Calendar.html#FEBRUARY[java 9])
+** [[painless-api-reference-Calendar-FIELD_COUNT]]static int link:{java8-javadoc}/java/util/Calendar.html#FIELD_COUNT[FIELD_COUNT] (link:{java9-javadoc}/java/util/Calendar.html#FIELD_COUNT[java 9])
+** [[painless-api-reference-Calendar-FRIDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#FRIDAY[FRIDAY] (link:{java9-javadoc}/java/util/Calendar.html#FRIDAY[java 9])
+** [[painless-api-reference-Calendar-HOUR]]static int link:{java8-javadoc}/java/util/Calendar.html#HOUR[HOUR] (link:{java9-javadoc}/java/util/Calendar.html#HOUR[java 9])
+** [[painless-api-reference-Calendar-HOUR_OF_DAY]]static int link:{java8-javadoc}/java/util/Calendar.html#HOUR_OF_DAY[HOUR_OF_DAY] (link:{java9-javadoc}/java/util/Calendar.html#HOUR_OF_DAY[java 9])
+** [[painless-api-reference-Calendar-JANUARY]]static int link:{java8-javadoc}/java/util/Calendar.html#JANUARY[JANUARY] (link:{java9-javadoc}/java/util/Calendar.html#JANUARY[java 9])
+** [[painless-api-reference-Calendar-JULY]]static int link:{java8-javadoc}/java/util/Calendar.html#JULY[JULY] (link:{java9-javadoc}/java/util/Calendar.html#JULY[java 9])
+** [[painless-api-reference-Calendar-JUNE]]static int link:{java8-javadoc}/java/util/Calendar.html#JUNE[JUNE] (link:{java9-javadoc}/java/util/Calendar.html#JUNE[java 9])
+** [[painless-api-reference-Calendar-LONG]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG[LONG] (link:{java9-javadoc}/java/util/Calendar.html#LONG[java 9])
+** [[painless-api-reference-Calendar-LONG_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG_FORMAT[LONG_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#LONG_FORMAT[java 9])
+** [[painless-api-reference-Calendar-LONG_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#LONG_STANDALONE[LONG_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#LONG_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-MARCH]]static int link:{java8-javadoc}/java/util/Calendar.html#MARCH[MARCH] (link:{java9-javadoc}/java/util/Calendar.html#MARCH[java 9])
+** [[painless-api-reference-Calendar-MAY]]static int link:{java8-javadoc}/java/util/Calendar.html#MAY[MAY] (link:{java9-javadoc}/java/util/Calendar.html#MAY[java 9])
+** [[painless-api-reference-Calendar-MILLISECOND]]static int link:{java8-javadoc}/java/util/Calendar.html#MILLISECOND[MILLISECOND] (link:{java9-javadoc}/java/util/Calendar.html#MILLISECOND[java 9])
+** [[painless-api-reference-Calendar-MINUTE]]static int link:{java8-javadoc}/java/util/Calendar.html#MINUTE[MINUTE] (link:{java9-javadoc}/java/util/Calendar.html#MINUTE[java 9])
+** [[painless-api-reference-Calendar-MONDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#MONDAY[MONDAY] (link:{java9-javadoc}/java/util/Calendar.html#MONDAY[java 9])
+** [[painless-api-reference-Calendar-MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#MONTH[MONTH] (link:{java9-javadoc}/java/util/Calendar.html#MONTH[java 9])
+** [[painless-api-reference-Calendar-NARROW_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#NARROW_FORMAT[NARROW_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#NARROW_FORMAT[java 9])
+** [[painless-api-reference-Calendar-NARROW_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#NARROW_STANDALONE[NARROW_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#NARROW_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-NOVEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#NOVEMBER[NOVEMBER] (link:{java9-javadoc}/java/util/Calendar.html#NOVEMBER[java 9])
+** [[painless-api-reference-Calendar-OCTOBER]]static int link:{java8-javadoc}/java/util/Calendar.html#OCTOBER[OCTOBER] (link:{java9-javadoc}/java/util/Calendar.html#OCTOBER[java 9])
+** [[painless-api-reference-Calendar-PM]]static int link:{java8-javadoc}/java/util/Calendar.html#PM[PM] (link:{java9-javadoc}/java/util/Calendar.html#PM[java 9])
+** [[painless-api-reference-Calendar-SATURDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#SATURDAY[SATURDAY] (link:{java9-javadoc}/java/util/Calendar.html#SATURDAY[java 9])
+** [[painless-api-reference-Calendar-SECOND]]static int link:{java8-javadoc}/java/util/Calendar.html#SECOND[SECOND] (link:{java9-javadoc}/java/util/Calendar.html#SECOND[java 9])
+** [[painless-api-reference-Calendar-SEPTEMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#SEPTEMBER[SEPTEMBER] (link:{java9-javadoc}/java/util/Calendar.html#SEPTEMBER[java 9])
+** [[painless-api-reference-Calendar-SHORT]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT[SHORT] (link:{java9-javadoc}/java/util/Calendar.html#SHORT[java 9])
+** [[painless-api-reference-Calendar-SHORT_FORMAT]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT_FORMAT[SHORT_FORMAT] (link:{java9-javadoc}/java/util/Calendar.html#SHORT_FORMAT[java 9])
+** [[painless-api-reference-Calendar-SHORT_STANDALONE]]static int link:{java8-javadoc}/java/util/Calendar.html#SHORT_STANDALONE[SHORT_STANDALONE] (link:{java9-javadoc}/java/util/Calendar.html#SHORT_STANDALONE[java 9])
+** [[painless-api-reference-Calendar-SUNDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#SUNDAY[SUNDAY] (link:{java9-javadoc}/java/util/Calendar.html#SUNDAY[java 9])
+** [[painless-api-reference-Calendar-THURSDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#THURSDAY[THURSDAY] (link:{java9-javadoc}/java/util/Calendar.html#THURSDAY[java 9])
+** [[painless-api-reference-Calendar-TUESDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#TUESDAY[TUESDAY] (link:{java9-javadoc}/java/util/Calendar.html#TUESDAY[java 9])
+** [[painless-api-reference-Calendar-UNDECIMBER]]static int link:{java8-javadoc}/java/util/Calendar.html#UNDECIMBER[UNDECIMBER] (link:{java9-javadoc}/java/util/Calendar.html#UNDECIMBER[java 9])
+** [[painless-api-reference-Calendar-WEDNESDAY]]static int link:{java8-javadoc}/java/util/Calendar.html#WEDNESDAY[WEDNESDAY] (link:{java9-javadoc}/java/util/Calendar.html#WEDNESDAY[java 9])
+** [[painless-api-reference-Calendar-WEEK_OF_MONTH]]static int link:{java8-javadoc}/java/util/Calendar.html#WEEK_OF_MONTH[WEEK_OF_MONTH] (link:{java9-javadoc}/java/util/Calendar.html#WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-Calendar-WEEK_OF_YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#WEEK_OF_YEAR[WEEK_OF_YEAR] (link:{java9-javadoc}/java/util/Calendar.html#WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-Calendar-YEAR]]static int link:{java8-javadoc}/java/util/Calendar.html#YEAR[YEAR] (link:{java9-javadoc}/java/util/Calendar.html#YEAR[java 9])
+** [[painless-api-reference-Calendar-ZONE_OFFSET]]static int link:{java8-javadoc}/java/util/Calendar.html#ZONE_OFFSET[ZONE_OFFSET] (link:{java9-javadoc}/java/util/Calendar.html#ZONE_OFFSET[java 9])
+* ++[[painless-api-reference-Calendar-getAvailableCalendarTypes-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Calendar.html#getAvailableCalendarTypes%2D%2D[getAvailableCalendarTypes]()++ (link:{java9-javadoc}/java/util/Calendar.html#getAvailableCalendarTypes%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/util/Calendar.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/util/Calendar.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-0]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-1]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2D[getInstance](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-getInstance-2]]static <<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-TimeZone,TimeZone>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getInstance%2Djava.util.TimeZone%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-add-2]]void link:{java8-javadoc}/java/util/Calendar.html#add%2Dint%2Dint%2D[add](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#add%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-after-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#after%2Djava.lang.Object%2D[after](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Calendar.html#after%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Calendar-before-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#before%2Djava.lang.Object%2D[before](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Calendar.html#before%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Calendar-clear-0]]void link:{java8-javadoc}/java/util/Calendar.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Calendar.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-clear-1]]void link:{java8-javadoc}/java/util/Calendar.html#clear%2Dint%2D[clear](int)++ (link:{java9-javadoc}/java/util/Calendar.html#clear%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-clone-0]]def link:{java8-javadoc}/java/util/Calendar.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Calendar.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-compareTo-1]]int link:{java8-javadoc}/java/util/Calendar.html#compareTo%2Djava.util.Calendar%2D[compareTo](<<painless-api-reference-Calendar,Calendar>>)++ (link:{java9-javadoc}/java/util/Calendar.html#compareTo%2Djava.util.Calendar%2D[java 9])
+* ++[[painless-api-reference-Calendar-get-1]]int link:{java8-javadoc}/java/util/Calendar.html#get%2Dint%2D[get](int)++ (link:{java9-javadoc}/java/util/Calendar.html#get%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getActualMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getActualMaximum%2Dint%2D[getActualMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getActualMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getActualMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getActualMinimum%2Dint%2D[getActualMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getActualMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getCalendarType-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Calendar.html#getCalendarType%2D%2D[getCalendarType]()++ (link:{java9-javadoc}/java/util/Calendar.html#getCalendarType%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getDisplayName-3]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Calendar.html#getDisplayName%2Dint%2Dint%2Djava.util.Locale%2D[getDisplayName](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getDisplayName%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-getDisplayNames-3]]<<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Calendar.html#getDisplayNames%2Dint%2Dint%2Djava.util.Locale%2D[getDisplayNames](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Calendar.html#getDisplayNames%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Calendar-getFirstDayOfWeek-0]]int link:{java8-javadoc}/java/util/Calendar.html#getFirstDayOfWeek%2D%2D[getFirstDayOfWeek]()++ (link:{java9-javadoc}/java/util/Calendar.html#getFirstDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getGreatestMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getGreatestMinimum%2Dint%2D[getGreatestMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getGreatestMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getLeastMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getLeastMaximum%2Dint%2D[getLeastMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getLeastMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMaximum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getMaximum%2Dint%2D[getMaximum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getMaximum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMinimalDaysInFirstWeek-0]]int link:{java8-javadoc}/java/util/Calendar.html#getMinimalDaysInFirstWeek%2D%2D[getMinimalDaysInFirstWeek]()++ (link:{java9-javadoc}/java/util/Calendar.html#getMinimalDaysInFirstWeek%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getMinimum-1]]int link:{java8-javadoc}/java/util/Calendar.html#getMinimum%2Dint%2D[getMinimum](int)++ (link:{java9-javadoc}/java/util/Calendar.html#getMinimum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTime-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/Calendar.html#getTime%2D%2D[getTime]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTime%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTimeInMillis-0]]long link:{java8-javadoc}/java/util/Calendar.html#getTimeInMillis%2D%2D[getTimeInMillis]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTimeInMillis%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getTimeZone-0]]<<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/Calendar.html#getTimeZone%2D%2D[getTimeZone]()++ (link:{java9-javadoc}/java/util/Calendar.html#getTimeZone%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getWeekYear-0]]int link:{java8-javadoc}/java/util/Calendar.html#getWeekYear%2D%2D[getWeekYear]()++ (link:{java9-javadoc}/java/util/Calendar.html#getWeekYear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-getWeeksInWeekYear-0]]int link:{java8-javadoc}/java/util/Calendar.html#getWeeksInWeekYear%2D%2D[getWeeksInWeekYear]()++ (link:{java9-javadoc}/java/util/Calendar.html#getWeeksInWeekYear%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-isLenient-0]]boolean link:{java8-javadoc}/java/util/Calendar.html#isLenient%2D%2D[isLenient]()++ (link:{java9-javadoc}/java/util/Calendar.html#isLenient%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-isSet-1]]boolean link:{java8-javadoc}/java/util/Calendar.html#isSet%2Dint%2D[isSet](int)++ (link:{java9-javadoc}/java/util/Calendar.html#isSet%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-isWeekDateSupported-0]]boolean link:{java8-javadoc}/java/util/Calendar.html#isWeekDateSupported%2D%2D[isWeekDateSupported]()++ (link:{java9-javadoc}/java/util/Calendar.html#isWeekDateSupported%2D%2D[java 9])
+* ++[[painless-api-reference-Calendar-roll-2]]void link:{java8-javadoc}/java/util/Calendar.html#roll%2Dint%2Dint%2D[roll](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#roll%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-2]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2D[set](int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-3]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2D[set](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-5]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2D[set](int, int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-set-6]]void link:{java8-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[set](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#set%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setFirstDayOfWeek-1]]void link:{java8-javadoc}/java/util/Calendar.html#setFirstDayOfWeek%2Dint%2D[setFirstDayOfWeek](int)++ (link:{java9-javadoc}/java/util/Calendar.html#setFirstDayOfWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setLenient-1]]void link:{java8-javadoc}/java/util/Calendar.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/util/Calendar.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Calendar-setMinimalDaysInFirstWeek-1]]void link:{java8-javadoc}/java/util/Calendar.html#setMinimalDaysInFirstWeek%2Dint%2D[setMinimalDaysInFirstWeek](int)++ (link:{java9-javadoc}/java/util/Calendar.html#setMinimalDaysInFirstWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTime-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTime%2Djava.util.Date%2D[setTime](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Calendar.html#setTime%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTimeInMillis-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTimeInMillis%2Dlong%2D[setTimeInMillis](long)++ (link:{java9-javadoc}/java/util/Calendar.html#setTimeInMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Calendar-setTimeZone-1]]void link:{java8-javadoc}/java/util/Calendar.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/Calendar.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-Calendar-setWeekDate-3]]void link:{java8-javadoc}/java/util/Calendar.html#setWeekDate%2Dint%2Dint%2Dint%2D[setWeekDate](int, int, int)++ (link:{java9-javadoc}/java/util/Calendar.html#setWeekDate%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Calendar-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/util/Calendar.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/util/Calendar.html#toInstant%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/CharSequence.asciidoc b/docs/reference/painless-api-reference/CharSequence.asciidoc
new file mode 100644
index 00000000000..2618a22881e
--- /dev/null
+++ b/docs/reference/painless-api-reference/CharSequence.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CharSequence]]++CharSequence++::
+* ++[[painless-api-reference-CharSequence-charAt-1]]char link:{java8-javadoc}/java/lang/CharSequence.html#charAt%2Dint%2D[charAt](int)++ (link:{java9-javadoc}/java/lang/CharSequence.html#charAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-CharSequence-chars-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/lang/CharSequence.html#chars%2D%2D[chars]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#chars%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-codePoints-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/lang/CharSequence.html#codePoints%2D%2D[codePoints]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#codePoints%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-length-0]]int link:{java8-javadoc}/java/lang/CharSequence.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-CharSequence-replaceAll-2]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#replaceAll%2Djava.lang.CharSequence%2Djava.util.regex.Pattern%2Djava.util.function.Function%2D[replaceAll](<<painless-api-reference-Pattern,Pattern>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-CharSequence-replaceFirst-2]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#replaceFirst%2Djava.lang.CharSequence%2Djava.util.regex.Pattern%2Djava.util.function.Function%2D[replaceFirst](<<painless-api-reference-Pattern,Pattern>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-CharSequence-subSequence-2]]<<painless-api-reference-CharSequence,CharSequence>> link:{java8-javadoc}/java/lang/CharSequence.html#subSequence%2Dint%2Dint%2D[subSequence](int, int)++ (link:{java9-javadoc}/java/lang/CharSequence.html#subSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-CharSequence-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/CharSequence.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/lang/CharSequence.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Character.Subset.asciidoc b/docs/reference/painless-api-reference/Character.Subset.asciidoc
new file mode 100644
index 00000000000..e8dff322e6e
--- /dev/null
+++ b/docs/reference/painless-api-reference/Character.Subset.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-Subset]]++Character.Subset++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Character.UnicodeBlock.asciidoc b/docs/reference/painless-api-reference/Character.UnicodeBlock.asciidoc
new file mode 100644
index 00000000000..b06cc70397a
--- /dev/null
+++ b/docs/reference/painless-api-reference/Character.UnicodeBlock.asciidoc
@@ -0,0 +1,229 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-UnicodeBlock]]++Character.UnicodeBlock++::
+** [[painless-api-reference-Character-UnicodeBlock-AEGEAN_NUMBERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#AEGEAN_NUMBERS[AEGEAN_NUMBERS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#AEGEAN_NUMBERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ALCHEMICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ALCHEMICAL_SYMBOLS[ALCHEMICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ALCHEMICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ALPHABETIC_PRESENTATION_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ALPHABETIC_PRESENTATION_FORMS[ALPHABETIC_PRESENTATION_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ALPHABETIC_PRESENTATION_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_GREEK_MUSICAL_NOTATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_GREEK_MUSICAL_NOTATION[ANCIENT_GREEK_MUSICAL_NOTATION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_GREEK_MUSICAL_NOTATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_GREEK_NUMBERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_GREEK_NUMBERS[ANCIENT_GREEK_NUMBERS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_GREEK_NUMBERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ANCIENT_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_SYMBOLS[ANCIENT_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ANCIENT_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC[ARABIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_EXTENDED_A[ARABIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS[ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_PRESENTATION_FORMS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_A[ARABIC_PRESENTATION_FORMS_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_PRESENTATION_FORMS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_B[ARABIC_PRESENTATION_FORMS_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_PRESENTATION_FORMS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARABIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_SUPPLEMENT[ARABIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARABIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARMENIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARMENIAN[ARMENIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARMENIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ARROWS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ARROWS[ARROWS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ARROWS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-AVESTAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#AVESTAN[AVESTAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#AVESTAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BALINESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BALINESE[BALINESE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BALINESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BAMUM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BAMUM[BAMUM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BAMUM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BAMUM_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BAMUM_SUPPLEMENT[BAMUM_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BAMUM_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BASIC_LATIN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BASIC_LATIN[BASIC_LATIN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BASIC_LATIN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BATAK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BATAK[BATAK] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BATAK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BENGALI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BENGALI[BENGALI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BENGALI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BLOCK_ELEMENTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BLOCK_ELEMENTS[BLOCK_ELEMENTS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BLOCK_ELEMENTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOPOMOFO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BOPOMOFO[BOPOMOFO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BOPOMOFO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOPOMOFO_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BOPOMOFO_EXTENDED[BOPOMOFO_EXTENDED] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BOPOMOFO_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BOX_DRAWING]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BOX_DRAWING[BOX_DRAWING] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BOX_DRAWING[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BRAHMI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BRAHMI[BRAHMI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BRAHMI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BRAILLE_PATTERNS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BRAILLE_PATTERNS[BRAILLE_PATTERNS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BRAILLE_PATTERNS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BUGINESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BUGINESE[BUGINESE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BUGINESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BUHID]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BUHID[BUHID] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BUHID[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-BYZANTINE_MUSICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#BYZANTINE_MUSICAL_SYMBOLS[BYZANTINE_MUSICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#BYZANTINE_MUSICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CARIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CARIAN[CARIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CARIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHAKMA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CHAKMA[CHAKMA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CHAKMA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CHAM[CHAM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CHAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CHEROKEE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CHEROKEE[CHEROKEE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CHEROKEE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY[CJK_COMPATIBILITY] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_FORMS[CJK_COMPATIBILITY_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_IDEOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS[CJK_COMPATIBILITY_IDEOGRAPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT[CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_RADICALS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_RADICALS_SUPPLEMENT[CJK_RADICALS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_RADICALS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_STROKES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_STROKES[CJK_STROKES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_STROKES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_SYMBOLS_AND_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_SYMBOLS_AND_PUNCTUATION[CJK_SYMBOLS_AND_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_SYMBOLS_AND_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS[CJK_UNIFIED_IDEOGRAPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D[CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_DIACRITICAL_MARKS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS[COMBINING_DIACRITICAL_MARKS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_DIACRITICAL_MARKS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS_SUPPLEMENT[COMBINING_DIACRITICAL_MARKS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_DIACRITICAL_MARKS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_HALF_MARKS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_HALF_MARKS[COMBINING_HALF_MARKS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_HALF_MARKS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMBINING_MARKS_FOR_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_MARKS_FOR_SYMBOLS[COMBINING_MARKS_FOR_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COMBINING_MARKS_FOR_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COMMON_INDIC_NUMBER_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COMMON_INDIC_NUMBER_FORMS[COMMON_INDIC_NUMBER_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COMMON_INDIC_NUMBER_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CONTROL_PICTURES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CONTROL_PICTURES[CONTROL_PICTURES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CONTROL_PICTURES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COPTIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COPTIC[COPTIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COPTIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-COUNTING_ROD_NUMERALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#COUNTING_ROD_NUMERALS[COUNTING_ROD_NUMERALS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#COUNTING_ROD_NUMERALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CUNEIFORM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CUNEIFORM[CUNEIFORM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CUNEIFORM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CUNEIFORM_NUMBERS_AND_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CUNEIFORM_NUMBERS_AND_PUNCTUATION[CUNEIFORM_NUMBERS_AND_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CUNEIFORM_NUMBERS_AND_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CURRENCY_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CURRENCY_SYMBOLS[CURRENCY_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CURRENCY_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYPRIOT_SYLLABARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CYPRIOT_SYLLABARY[CYPRIOT_SYLLABARY] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CYPRIOT_SYLLABARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC[CYRILLIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_EXTENDED_A[CYRILLIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_EXTENDED_B[CYRILLIC_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-CYRILLIC_SUPPLEMENTARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_SUPPLEMENTARY[CYRILLIC_SUPPLEMENTARY] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#CYRILLIC_SUPPLEMENTARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DESERET]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#DESERET[DESERET] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#DESERET[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DEVANAGARI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#DEVANAGARI[DEVANAGARI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#DEVANAGARI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DEVANAGARI_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#DEVANAGARI_EXTENDED[DEVANAGARI_EXTENDED] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#DEVANAGARI_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DINGBATS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#DINGBATS[DINGBATS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#DINGBATS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-DOMINO_TILES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#DOMINO_TILES[DOMINO_TILES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#DOMINO_TILES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-EGYPTIAN_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#EGYPTIAN_HIEROGLYPHS[EGYPTIAN_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#EGYPTIAN_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-EMOTICONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#EMOTICONS[EMOTICONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#EMOTICONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_ALPHANUMERICS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_ALPHANUMERICS[ENCLOSED_ALPHANUMERICS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_ALPHANUMERICS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_ALPHANUMERIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_ALPHANUMERIC_SUPPLEMENT[ENCLOSED_ALPHANUMERIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_ALPHANUMERIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_CJK_LETTERS_AND_MONTHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_CJK_LETTERS_AND_MONTHS[ENCLOSED_CJK_LETTERS_AND_MONTHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_CJK_LETTERS_AND_MONTHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ENCLOSED_IDEOGRAPHIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_IDEOGRAPHIC_SUPPLEMENT[ENCLOSED_IDEOGRAPHIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ENCLOSED_IDEOGRAPHIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC[ETHIOPIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_EXTENDED[ETHIOPIC_EXTENDED] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_EXTENDED_A[ETHIOPIC_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ETHIOPIC_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_SUPPLEMENT[ETHIOPIC_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ETHIOPIC_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GENERAL_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GENERAL_PUNCTUATION[GENERAL_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GENERAL_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEOMETRIC_SHAPES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GEOMETRIC_SHAPES[GEOMETRIC_SHAPES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GEOMETRIC_SHAPES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEORGIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GEORGIAN[GEORGIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GEORGIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GEORGIAN_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GEORGIAN_SUPPLEMENT[GEORGIAN_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GEORGIAN_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GLAGOLITIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GLAGOLITIC[GLAGOLITIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GLAGOLITIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GOTHIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GOTHIC[GOTHIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GOTHIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GREEK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GREEK[GREEK] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GREEK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GREEK_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GREEK_EXTENDED[GREEK_EXTENDED] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GREEK_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GUJARATI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GUJARATI[GUJARATI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GUJARATI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-GURMUKHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#GURMUKHI[GURMUKHI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#GURMUKHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HALFWIDTH_AND_FULLWIDTH_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HALFWIDTH_AND_FULLWIDTH_FORMS[HALFWIDTH_AND_FULLWIDTH_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HALFWIDTH_AND_FULLWIDTH_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_COMPATIBILITY_JAMO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_COMPATIBILITY_JAMO[HANGUL_COMPATIBILITY_JAMO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_COMPATIBILITY_JAMO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO[HANGUL_JAMO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO_EXTENDED_A[HANGUL_JAMO_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_JAMO_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO_EXTENDED_B[HANGUL_JAMO_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_JAMO_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANGUL_SYLLABLES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_SYLLABLES[HANGUL_SYLLABLES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANGUL_SYLLABLES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HANUNOO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HANUNOO[HANUNOO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HANUNOO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HEBREW]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HEBREW[HEBREW] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HEBREW[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIGH_PRIVATE_USE_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HIGH_PRIVATE_USE_SURROGATES[HIGH_PRIVATE_USE_SURROGATES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HIGH_PRIVATE_USE_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIGH_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HIGH_SURROGATES[HIGH_SURROGATES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HIGH_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-HIRAGANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#HIRAGANA[HIRAGANA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#HIRAGANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IDEOGRAPHIC_DESCRIPTION_CHARACTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#IDEOGRAPHIC_DESCRIPTION_CHARACTERS[IDEOGRAPHIC_DESCRIPTION_CHARACTERS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#IDEOGRAPHIC_DESCRIPTION_CHARACTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IMPERIAL_ARAMAIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#IMPERIAL_ARAMAIC[IMPERIAL_ARAMAIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#IMPERIAL_ARAMAIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-INSCRIPTIONAL_PAHLAVI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#INSCRIPTIONAL_PAHLAVI[INSCRIPTIONAL_PAHLAVI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#INSCRIPTIONAL_PAHLAVI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-INSCRIPTIONAL_PARTHIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#INSCRIPTIONAL_PARTHIAN[INSCRIPTIONAL_PARTHIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#INSCRIPTIONAL_PARTHIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-IPA_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#IPA_EXTENSIONS[IPA_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#IPA_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-JAVANESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#JAVANESE[JAVANESE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#JAVANESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KAITHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KAITHI[KAITHI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KAITHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANA_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KANA_SUPPLEMENT[KANA_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KANA_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANBUN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KANBUN[KANBUN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KANBUN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANGXI_RADICALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KANGXI_RADICALS[KANGXI_RADICALS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KANGXI_RADICALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KANNADA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KANNADA[KANNADA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KANNADA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KATAKANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KATAKANA[KATAKANA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KATAKANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KATAKANA_PHONETIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KATAKANA_PHONETIC_EXTENSIONS[KATAKANA_PHONETIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KATAKANA_PHONETIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KAYAH_LI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KAYAH_LI[KAYAH_LI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KAYAH_LI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHAROSHTHI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KHAROSHTHI[KHAROSHTHI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KHAROSHTHI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHMER]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KHMER[KHMER] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KHMER[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-KHMER_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#KHMER_SYMBOLS[KHMER_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#KHMER_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LAO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LAO[LAO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LAO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_1_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_1_SUPPLEMENT[LATIN_1_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_1_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_A[LATIN_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_ADDITIONAL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_ADDITIONAL[LATIN_EXTENDED_ADDITIONAL] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_ADDITIONAL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_B[LATIN_EXTENDED_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_C]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_C[LATIN_EXTENDED_C] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_C[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LATIN_EXTENDED_D]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_D[LATIN_EXTENDED_D] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LATIN_EXTENDED_D[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LEPCHA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LEPCHA[LEPCHA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LEPCHA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LETTERLIKE_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LETTERLIKE_SYMBOLS[LETTERLIKE_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LETTERLIKE_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LIMBU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LIMBU[LIMBU] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LIMBU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LINEAR_B_IDEOGRAMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LINEAR_B_IDEOGRAMS[LINEAR_B_IDEOGRAMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LINEAR_B_IDEOGRAMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LINEAR_B_SYLLABARY]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LINEAR_B_SYLLABARY[LINEAR_B_SYLLABARY] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LINEAR_B_SYLLABARY[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LISU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LISU[LISU] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LISU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LOW_SURROGATES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LOW_SURROGATES[LOW_SURROGATES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LOW_SURROGATES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LYCIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LYCIAN[LYCIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LYCIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-LYDIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#LYDIAN[LYDIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#LYDIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MAHJONG_TILES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MAHJONG_TILES[MAHJONG_TILES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MAHJONG_TILES[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MALAYALAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MALAYALAM[MALAYALAM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MALAYALAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MANDAIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MANDAIC[MANDAIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MANDAIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MATHEMATICAL_ALPHANUMERIC_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MATHEMATICAL_ALPHANUMERIC_SYMBOLS[MATHEMATICAL_ALPHANUMERIC_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MATHEMATICAL_ALPHANUMERIC_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MATHEMATICAL_OPERATORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MATHEMATICAL_OPERATORS[MATHEMATICAL_OPERATORS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MATHEMATICAL_OPERATORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEETEI_MAYEK]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MEETEI_MAYEK[MEETEI_MAYEK] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MEETEI_MAYEK[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEETEI_MAYEK_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MEETEI_MAYEK_EXTENSIONS[MEETEI_MAYEK_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MEETEI_MAYEK_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEROITIC_CURSIVE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MEROITIC_CURSIVE[MEROITIC_CURSIVE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MEROITIC_CURSIVE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MEROITIC_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MEROITIC_HIEROGLYPHS[MEROITIC_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MEROITIC_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MIAO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MIAO[MIAO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MIAO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A[MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B[MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS[MISCELLANEOUS_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS_AND_ARROWS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_ARROWS[MISCELLANEOUS_SYMBOLS_AND_ARROWS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_ARROWS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS[MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MISCELLANEOUS_TECHNICAL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_TECHNICAL[MISCELLANEOUS_TECHNICAL] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MISCELLANEOUS_TECHNICAL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MODIFIER_TONE_LETTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MODIFIER_TONE_LETTERS[MODIFIER_TONE_LETTERS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MODIFIER_TONE_LETTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MONGOLIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MONGOLIAN[MONGOLIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MONGOLIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MUSICAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MUSICAL_SYMBOLS[MUSICAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MUSICAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MYANMAR]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MYANMAR[MYANMAR] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MYANMAR[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-MYANMAR_EXTENDED_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#MYANMAR_EXTENDED_A[MYANMAR_EXTENDED_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#MYANMAR_EXTENDED_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NEW_TAI_LUE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#NEW_TAI_LUE[NEW_TAI_LUE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#NEW_TAI_LUE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NKO]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#NKO[NKO] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#NKO[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-NUMBER_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#NUMBER_FORMS[NUMBER_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#NUMBER_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OGHAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OGHAM[OGHAM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OGHAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_ITALIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_ITALIC[OLD_ITALIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_ITALIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_PERSIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_PERSIAN[OLD_PERSIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_PERSIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_SOUTH_ARABIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_SOUTH_ARABIAN[OLD_SOUTH_ARABIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_SOUTH_ARABIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OLD_TURKIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_TURKIC[OLD_TURKIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OLD_TURKIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OL_CHIKI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OL_CHIKI[OL_CHIKI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OL_CHIKI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OPTICAL_CHARACTER_RECOGNITION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OPTICAL_CHARACTER_RECOGNITION[OPTICAL_CHARACTER_RECOGNITION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OPTICAL_CHARACTER_RECOGNITION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-ORIYA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#ORIYA[ORIYA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#ORIYA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-OSMANYA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#OSMANYA[OSMANYA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#OSMANYA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHAGS_PA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PHAGS_PA[PHAGS_PA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PHAGS_PA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHAISTOS_DISC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PHAISTOS_DISC[PHAISTOS_DISC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PHAISTOS_DISC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHOENICIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PHOENICIAN[PHOENICIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PHOENICIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHONETIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PHONETIC_EXTENSIONS[PHONETIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PHONETIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PHONETIC_EXTENSIONS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PHONETIC_EXTENSIONS_SUPPLEMENT[PHONETIC_EXTENSIONS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PHONETIC_EXTENSIONS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PLAYING_CARDS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PLAYING_CARDS[PLAYING_CARDS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PLAYING_CARDS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-PRIVATE_USE_AREA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#PRIVATE_USE_AREA[PRIVATE_USE_AREA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#PRIVATE_USE_AREA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-REJANG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#REJANG[REJANG] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#REJANG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-RUMI_NUMERAL_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#RUMI_NUMERAL_SYMBOLS[RUMI_NUMERAL_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#RUMI_NUMERAL_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-RUNIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#RUNIC[RUNIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#RUNIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SAMARITAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SAMARITAN[SAMARITAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SAMARITAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SAURASHTRA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SAURASHTRA[SAURASHTRA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SAURASHTRA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SHARADA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SHARADA[SHARADA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SHARADA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SHAVIAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SHAVIAN[SHAVIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SHAVIAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SINHALA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SINHALA[SINHALA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SINHALA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SMALL_FORM_VARIANTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SMALL_FORM_VARIANTS[SMALL_FORM_VARIANTS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SMALL_FORM_VARIANTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SORA_SOMPENG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SORA_SOMPENG[SORA_SOMPENG] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SORA_SOMPENG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SPACING_MODIFIER_LETTERS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SPACING_MODIFIER_LETTERS[SPACING_MODIFIER_LETTERS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SPACING_MODIFIER_LETTERS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SPECIALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SPECIALS[SPECIALS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SPECIALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUNDANESE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUNDANESE[SUNDANESE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUNDANESE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUNDANESE_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUNDANESE_SUPPLEMENT[SUNDANESE_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUNDANESE_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPERSCRIPTS_AND_SUBSCRIPTS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPERSCRIPTS_AND_SUBSCRIPTS[SUPERSCRIPTS_AND_SUBSCRIPTS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPERSCRIPTS_AND_SUBSCRIPTS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_ARROWS_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_ARROWS_A[SUPPLEMENTAL_ARROWS_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_ARROWS_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_ARROWS_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_ARROWS_B[SUPPLEMENTAL_ARROWS_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_ARROWS_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_MATHEMATICAL_OPERATORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_MATHEMATICAL_OPERATORS[SUPPLEMENTAL_MATHEMATICAL_OPERATORS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_MATHEMATICAL_OPERATORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTAL_PUNCTUATION]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_PUNCTUATION[SUPPLEMENTAL_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTAL_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTARY_PRIVATE_USE_AREA_A]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_A[SUPPLEMENTARY_PRIVATE_USE_AREA_A] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_A[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SUPPLEMENTARY_PRIVATE_USE_AREA_B]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_B[SUPPLEMENTARY_PRIVATE_USE_AREA_B] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SUPPLEMENTARY_PRIVATE_USE_AREA_B[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SYLOTI_NAGRI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SYLOTI_NAGRI[SYLOTI_NAGRI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SYLOTI_NAGRI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-SYRIAC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#SYRIAC[SYRIAC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#SYRIAC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGALOG]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAGALOG[TAGALOG] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAGALOG[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGBANWA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAGBANWA[TAGBANWA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAGBANWA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAGS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAGS[TAGS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAGS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_LE]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_LE[TAI_LE] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_LE[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_THAM]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_THAM[TAI_THAM] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_THAM[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_VIET]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_VIET[TAI_VIET] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_VIET[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAI_XUAN_JING_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_XUAN_JING_SYMBOLS[TAI_XUAN_JING_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAI_XUAN_JING_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAKRI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAKRI[TAKRI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAKRI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TAMIL]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TAMIL[TAMIL] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TAMIL[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TELUGU]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TELUGU[TELUGU] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TELUGU[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-THAANA]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#THAANA[THAANA] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#THAANA[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-THAI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#THAI[THAI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#THAI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TIBETAN]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TIBETAN[TIBETAN] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TIBETAN[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TIFINAGH]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TIFINAGH[TIFINAGH] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TIFINAGH[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-TRANSPORT_AND_MAP_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#TRANSPORT_AND_MAP_SYMBOLS[TRANSPORT_AND_MAP_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#TRANSPORT_AND_MAP_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UGARITIC]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#UGARITIC[UGARITIC] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#UGARITIC[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS[UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED[UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VAI]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#VAI[VAI] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#VAI[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VARIATION_SELECTORS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#VARIATION_SELECTORS[VARIATION_SELECTORS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#VARIATION_SELECTORS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VARIATION_SELECTORS_SUPPLEMENT]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#VARIATION_SELECTORS_SUPPLEMENT[VARIATION_SELECTORS_SUPPLEMENT] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#VARIATION_SELECTORS_SUPPLEMENT[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VEDIC_EXTENSIONS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#VEDIC_EXTENSIONS[VEDIC_EXTENSIONS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#VEDIC_EXTENSIONS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-VERTICAL_FORMS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#VERTICAL_FORMS[VERTICAL_FORMS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#VERTICAL_FORMS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YIJING_HEXAGRAM_SYMBOLS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#YIJING_HEXAGRAM_SYMBOLS[YIJING_HEXAGRAM_SYMBOLS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#YIJING_HEXAGRAM_SYMBOLS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YI_RADICALS]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#YI_RADICALS[YI_RADICALS] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#YI_RADICALS[java 9])
+** [[painless-api-reference-Character-UnicodeBlock-YI_SYLLABLES]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#YI_SYLLABLES[YI_SYLLABLES] (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#YI_SYLLABLES[java 9])
+* ++[[painless-api-reference-Character-UnicodeBlock-forName-1]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#forName%2Djava.lang.String%2D[forName](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#forName%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeBlock-of-1]]static <<painless-api-reference-Character-UnicodeBlock,Character.UnicodeBlock>> link:{java8-javadoc}/java/lang/Character$UnicodeBlock.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/lang/Character$UnicodeBlock.html#of%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Character.UnicodeScript.asciidoc b/docs/reference/painless-api-reference/Character.UnicodeScript.asciidoc
new file mode 100644
index 00000000000..cc189b1ee1e
--- /dev/null
+++ b/docs/reference/painless-api-reference/Character.UnicodeScript.asciidoc
@@ -0,0 +1,114 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character-UnicodeScript]]++Character.UnicodeScript++::
+** [[painless-api-reference-Character-UnicodeScript-ARABIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#ARABIC[ARABIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#ARABIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ARMENIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#ARMENIAN[ARMENIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#ARMENIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-AVESTAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#AVESTAN[AVESTAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#AVESTAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BALINESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BALINESE[BALINESE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BALINESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BAMUM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BAMUM[BAMUM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BAMUM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BATAK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BATAK[BATAK] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BATAK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BENGALI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BENGALI[BENGALI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BENGALI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BOPOMOFO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BOPOMOFO[BOPOMOFO] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BOPOMOFO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BRAHMI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BRAHMI[BRAHMI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BRAHMI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BRAILLE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BRAILLE[BRAILLE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BRAILLE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BUGINESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BUGINESE[BUGINESE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BUGINESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-BUHID]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#BUHID[BUHID] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#BUHID[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CANADIAN_ABORIGINAL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CANADIAN_ABORIGINAL[CANADIAN_ABORIGINAL] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CANADIAN_ABORIGINAL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CARIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CARIAN[CARIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CARIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHAKMA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CHAKMA[CHAKMA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CHAKMA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CHAM[CHAM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CHAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CHEROKEE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CHEROKEE[CHEROKEE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CHEROKEE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-COMMON]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#COMMON[COMMON] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#COMMON[java 9])
+** [[painless-api-reference-Character-UnicodeScript-COPTIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#COPTIC[COPTIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#COPTIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CUNEIFORM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CUNEIFORM[CUNEIFORM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CUNEIFORM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CYPRIOT]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CYPRIOT[CYPRIOT] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CYPRIOT[java 9])
+** [[painless-api-reference-Character-UnicodeScript-CYRILLIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#CYRILLIC[CYRILLIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#CYRILLIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-DESERET]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#DESERET[DESERET] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#DESERET[java 9])
+** [[painless-api-reference-Character-UnicodeScript-DEVANAGARI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#DEVANAGARI[DEVANAGARI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#DEVANAGARI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-EGYPTIAN_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#EGYPTIAN_HIEROGLYPHS[EGYPTIAN_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#EGYPTIAN_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ETHIOPIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#ETHIOPIC[ETHIOPIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#ETHIOPIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GEORGIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GEORGIAN[GEORGIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GEORGIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GLAGOLITIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GLAGOLITIC[GLAGOLITIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GLAGOLITIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GOTHIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GOTHIC[GOTHIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GOTHIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GREEK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GREEK[GREEK] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GREEK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GUJARATI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GUJARATI[GUJARATI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GUJARATI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-GURMUKHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#GURMUKHI[GURMUKHI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#GURMUKHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#HAN[HAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#HAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HANGUL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#HANGUL[HANGUL] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#HANGUL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HANUNOO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#HANUNOO[HANUNOO] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#HANUNOO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HEBREW]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#HEBREW[HEBREW] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#HEBREW[java 9])
+** [[painless-api-reference-Character-UnicodeScript-HIRAGANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#HIRAGANA[HIRAGANA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#HIRAGANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-IMPERIAL_ARAMAIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#IMPERIAL_ARAMAIC[IMPERIAL_ARAMAIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#IMPERIAL_ARAMAIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INHERITED]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#INHERITED[INHERITED] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#INHERITED[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INSCRIPTIONAL_PAHLAVI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#INSCRIPTIONAL_PAHLAVI[INSCRIPTIONAL_PAHLAVI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#INSCRIPTIONAL_PAHLAVI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-INSCRIPTIONAL_PARTHIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#INSCRIPTIONAL_PARTHIAN[INSCRIPTIONAL_PARTHIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#INSCRIPTIONAL_PARTHIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-JAVANESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#JAVANESE[JAVANESE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#JAVANESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KAITHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KAITHI[KAITHI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KAITHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KANNADA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KANNADA[KANNADA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KANNADA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KATAKANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KATAKANA[KATAKANA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KATAKANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KAYAH_LI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KAYAH_LI[KAYAH_LI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KAYAH_LI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KHAROSHTHI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KHAROSHTHI[KHAROSHTHI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KHAROSHTHI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-KHMER]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#KHMER[KHMER] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#KHMER[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LAO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LAO[LAO] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LAO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LATIN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LATIN[LATIN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LATIN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LEPCHA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LEPCHA[LEPCHA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LEPCHA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LIMBU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LIMBU[LIMBU] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LIMBU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LINEAR_B]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LINEAR_B[LINEAR_B] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LINEAR_B[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LISU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LISU[LISU] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LISU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LYCIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LYCIAN[LYCIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LYCIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-LYDIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#LYDIAN[LYDIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#LYDIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MALAYALAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MALAYALAM[MALAYALAM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MALAYALAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MANDAIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MANDAIC[MANDAIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MANDAIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEETEI_MAYEK]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MEETEI_MAYEK[MEETEI_MAYEK] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MEETEI_MAYEK[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEROITIC_CURSIVE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MEROITIC_CURSIVE[MEROITIC_CURSIVE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MEROITIC_CURSIVE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MEROITIC_HIEROGLYPHS]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MEROITIC_HIEROGLYPHS[MEROITIC_HIEROGLYPHS] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MEROITIC_HIEROGLYPHS[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MIAO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MIAO[MIAO] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MIAO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MONGOLIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MONGOLIAN[MONGOLIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MONGOLIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-MYANMAR]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#MYANMAR[MYANMAR] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#MYANMAR[java 9])
+** [[painless-api-reference-Character-UnicodeScript-NEW_TAI_LUE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#NEW_TAI_LUE[NEW_TAI_LUE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#NEW_TAI_LUE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-NKO]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#NKO[NKO] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#NKO[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OGHAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OGHAM[OGHAM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OGHAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_ITALIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OLD_ITALIC[OLD_ITALIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OLD_ITALIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_PERSIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OLD_PERSIAN[OLD_PERSIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OLD_PERSIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_SOUTH_ARABIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OLD_SOUTH_ARABIAN[OLD_SOUTH_ARABIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OLD_SOUTH_ARABIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OLD_TURKIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OLD_TURKIC[OLD_TURKIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OLD_TURKIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OL_CHIKI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OL_CHIKI[OL_CHIKI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OL_CHIKI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-ORIYA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#ORIYA[ORIYA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#ORIYA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-OSMANYA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#OSMANYA[OSMANYA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#OSMANYA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-PHAGS_PA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#PHAGS_PA[PHAGS_PA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#PHAGS_PA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-PHOENICIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#PHOENICIAN[PHOENICIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#PHOENICIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-REJANG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#REJANG[REJANG] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#REJANG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-RUNIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#RUNIC[RUNIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#RUNIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SAMARITAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SAMARITAN[SAMARITAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SAMARITAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SAURASHTRA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SAURASHTRA[SAURASHTRA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SAURASHTRA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SHARADA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SHARADA[SHARADA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SHARADA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SHAVIAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SHAVIAN[SHAVIAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SHAVIAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SINHALA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SINHALA[SINHALA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SINHALA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SORA_SOMPENG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SORA_SOMPENG[SORA_SOMPENG] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SORA_SOMPENG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SUNDANESE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SUNDANESE[SUNDANESE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SUNDANESE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SYLOTI_NAGRI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SYLOTI_NAGRI[SYLOTI_NAGRI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SYLOTI_NAGRI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-SYRIAC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#SYRIAC[SYRIAC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#SYRIAC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAGALOG]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAGALOG[TAGALOG] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAGALOG[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAGBANWA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAGBANWA[TAGBANWA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAGBANWA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_LE]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAI_LE[TAI_LE] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAI_LE[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_THAM]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAI_THAM[TAI_THAM] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAI_THAM[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAI_VIET]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAI_VIET[TAI_VIET] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAI_VIET[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAKRI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAKRI[TAKRI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAKRI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TAMIL]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TAMIL[TAMIL] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TAMIL[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TELUGU]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TELUGU[TELUGU] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TELUGU[java 9])
+** [[painless-api-reference-Character-UnicodeScript-THAANA]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#THAANA[THAANA] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#THAANA[java 9])
+** [[painless-api-reference-Character-UnicodeScript-THAI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#THAI[THAI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#THAI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TIBETAN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TIBETAN[TIBETAN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TIBETAN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-TIFINAGH]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#TIFINAGH[TIFINAGH] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#TIFINAGH[java 9])
+** [[painless-api-reference-Character-UnicodeScript-UGARITIC]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#UGARITIC[UGARITIC] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#UGARITIC[java 9])
+** [[painless-api-reference-Character-UnicodeScript-UNKNOWN]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#UNKNOWN[UNKNOWN] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#UNKNOWN[java 9])
+** [[painless-api-reference-Character-UnicodeScript-VAI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#VAI[VAI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#VAI[java 9])
+** [[painless-api-reference-Character-UnicodeScript-YI]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#YI[YI] (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#YI[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-forName-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#forName%2Djava.lang.String%2D[forName](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#forName%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-of-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-valueOf-1]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>> link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Character-UnicodeScript-values-0]]static <<painless-api-reference-Character-UnicodeScript,Character.UnicodeScript>>[] link:{java8-javadoc}/java/lang/Character$UnicodeScript.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/lang/Character$UnicodeScript.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Character.asciidoc b/docs/reference/painless-api-reference/Character.asciidoc
new file mode 100644
index 00000000000..929a4ea3c5e
--- /dev/null
+++ b/docs/reference/painless-api-reference/Character.asciidoc
@@ -0,0 +1,125 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Character]]++Character++::
+** [[painless-api-reference-Character-BYTES]]static int link:{java8-javadoc}/java/lang/Character.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Character.html#BYTES[java 9])
+** [[painless-api-reference-Character-COMBINING_SPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#COMBINING_SPACING_MARK[COMBINING_SPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#COMBINING_SPACING_MARK[java 9])
+** [[painless-api-reference-Character-CONNECTOR_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#CONNECTOR_PUNCTUATION[CONNECTOR_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#CONNECTOR_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-CONTROL]]static byte link:{java8-javadoc}/java/lang/Character.html#CONTROL[CONTROL] (link:{java9-javadoc}/java/lang/Character.html#CONTROL[java 9])
+** [[painless-api-reference-Character-CURRENCY_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#CURRENCY_SYMBOL[CURRENCY_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#CURRENCY_SYMBOL[java 9])
+** [[painless-api-reference-Character-DASH_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#DASH_PUNCTUATION[DASH_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#DASH_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-DECIMAL_DIGIT_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DECIMAL_DIGIT_NUMBER[DECIMAL_DIGIT_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DECIMAL_DIGIT_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_ARABIC_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_ARABIC_NUMBER[DIRECTIONALITY_ARABIC_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_ARABIC_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_BOUNDARY_NEUTRAL]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_BOUNDARY_NEUTRAL[DIRECTIONALITY_BOUNDARY_NEUTRAL] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_BOUNDARY_NEUTRAL[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_COMMON_NUMBER_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_COMMON_NUMBER_SEPARATOR[DIRECTIONALITY_COMMON_NUMBER_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_COMMON_NUMBER_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER[DIRECTIONALITY_EUROPEAN_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR[DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR[DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT[DIRECTIONALITY_LEFT_TO_RIGHT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING[DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE[DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_NONSPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_NONSPACING_MARK[DIRECTIONALITY_NONSPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_NONSPACING_MARK[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_OTHER_NEUTRALS]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_OTHER_NEUTRALS[DIRECTIONALITY_OTHER_NEUTRALS] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_OTHER_NEUTRALS[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_PARAGRAPH_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_PARAGRAPH_SEPARATOR[DIRECTIONALITY_PARAGRAPH_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_PARAGRAPH_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_POP_DIRECTIONAL_FORMAT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_POP_DIRECTIONAL_FORMAT[DIRECTIONALITY_POP_DIRECTIONAL_FORMAT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_POP_DIRECTIONAL_FORMAT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT[DIRECTIONALITY_RIGHT_TO_LEFT] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC[DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING[DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE[DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_SEGMENT_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_SEGMENT_SEPARATOR[DIRECTIONALITY_SEGMENT_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_SEGMENT_SEPARATOR[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_UNDEFINED]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_UNDEFINED[DIRECTIONALITY_UNDEFINED] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_UNDEFINED[java 9])
+** [[painless-api-reference-Character-DIRECTIONALITY_WHITESPACE]]static byte link:{java8-javadoc}/java/lang/Character.html#DIRECTIONALITY_WHITESPACE[DIRECTIONALITY_WHITESPACE] (link:{java9-javadoc}/java/lang/Character.html#DIRECTIONALITY_WHITESPACE[java 9])
+** [[painless-api-reference-Character-ENCLOSING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#ENCLOSING_MARK[ENCLOSING_MARK] (link:{java9-javadoc}/java/lang/Character.html#ENCLOSING_MARK[java 9])
+** [[painless-api-reference-Character-END_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#END_PUNCTUATION[END_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#END_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-FINAL_QUOTE_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#FINAL_QUOTE_PUNCTUATION[FINAL_QUOTE_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#FINAL_QUOTE_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-FORMAT]]static byte link:{java8-javadoc}/java/lang/Character.html#FORMAT[FORMAT] (link:{java9-javadoc}/java/lang/Character.html#FORMAT[java 9])
+** [[painless-api-reference-Character-INITIAL_QUOTE_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#INITIAL_QUOTE_PUNCTUATION[INITIAL_QUOTE_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#INITIAL_QUOTE_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-LETTER_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#LETTER_NUMBER[LETTER_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#LETTER_NUMBER[java 9])
+** [[painless-api-reference-Character-LINE_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#LINE_SEPARATOR[LINE_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#LINE_SEPARATOR[java 9])
+** [[painless-api-reference-Character-LOWERCASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#LOWERCASE_LETTER[LOWERCASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#LOWERCASE_LETTER[java 9])
+** [[painless-api-reference-Character-MATH_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#MATH_SYMBOL[MATH_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#MATH_SYMBOL[java 9])
+** [[painless-api-reference-Character-MAX_CODE_POINT]]static int link:{java8-javadoc}/java/lang/Character.html#MAX_CODE_POINT[MAX_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MAX_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MAX_HIGH_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_HIGH_SURROGATE[MAX_HIGH_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_HIGH_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_LOW_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_LOW_SURROGATE[MAX_LOW_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_LOW_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_RADIX]]static int link:{java8-javadoc}/java/lang/Character.html#MAX_RADIX[MAX_RADIX] (link:{java9-javadoc}/java/lang/Character.html#MAX_RADIX[java 9])
+** [[painless-api-reference-Character-MAX_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_SURROGATE[MAX_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MAX_SURROGATE[java 9])
+** [[painless-api-reference-Character-MAX_VALUE]]static char link:{java8-javadoc}/java/lang/Character.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Character.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Character-MIN_CODE_POINT]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_CODE_POINT[MIN_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MIN_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MIN_HIGH_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_HIGH_SURROGATE[MIN_HIGH_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_HIGH_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_LOW_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_LOW_SURROGATE[MIN_LOW_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_LOW_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_RADIX]]static int link:{java8-javadoc}/java/lang/Character.html#MIN_RADIX[MIN_RADIX] (link:{java9-javadoc}/java/lang/Character.html#MIN_RADIX[java 9])
+** [[painless-api-reference-Character-MIN_SUPPLEMENTARY_CODE_POINT]]static int link:{java8-javadoc}/java/lang/Character.html#MIN_SUPPLEMENTARY_CODE_POINT[MIN_SUPPLEMENTARY_CODE_POINT] (link:{java9-javadoc}/java/lang/Character.html#MIN_SUPPLEMENTARY_CODE_POINT[java 9])
+** [[painless-api-reference-Character-MIN_SURROGATE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_SURROGATE[MIN_SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#MIN_SURROGATE[java 9])
+** [[painless-api-reference-Character-MIN_VALUE]]static char link:{java8-javadoc}/java/lang/Character.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Character.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Character-MODIFIER_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#MODIFIER_LETTER[MODIFIER_LETTER] (link:{java9-javadoc}/java/lang/Character.html#MODIFIER_LETTER[java 9])
+** [[painless-api-reference-Character-MODIFIER_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#MODIFIER_SYMBOL[MODIFIER_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#MODIFIER_SYMBOL[java 9])
+** [[painless-api-reference-Character-NON_SPACING_MARK]]static byte link:{java8-javadoc}/java/lang/Character.html#NON_SPACING_MARK[NON_SPACING_MARK] (link:{java9-javadoc}/java/lang/Character.html#NON_SPACING_MARK[java 9])
+** [[painless-api-reference-Character-OTHER_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_LETTER[OTHER_LETTER] (link:{java9-javadoc}/java/lang/Character.html#OTHER_LETTER[java 9])
+** [[painless-api-reference-Character-OTHER_NUMBER]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_NUMBER[OTHER_NUMBER] (link:{java9-javadoc}/java/lang/Character.html#OTHER_NUMBER[java 9])
+** [[painless-api-reference-Character-OTHER_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_PUNCTUATION[OTHER_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#OTHER_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-OTHER_SYMBOL]]static byte link:{java8-javadoc}/java/lang/Character.html#OTHER_SYMBOL[OTHER_SYMBOL] (link:{java9-javadoc}/java/lang/Character.html#OTHER_SYMBOL[java 9])
+** [[painless-api-reference-Character-PARAGRAPH_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#PARAGRAPH_SEPARATOR[PARAGRAPH_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#PARAGRAPH_SEPARATOR[java 9])
+** [[painless-api-reference-Character-PRIVATE_USE]]static byte link:{java8-javadoc}/java/lang/Character.html#PRIVATE_USE[PRIVATE_USE] (link:{java9-javadoc}/java/lang/Character.html#PRIVATE_USE[java 9])
+** [[painless-api-reference-Character-SIZE]]static int link:{java8-javadoc}/java/lang/Character.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Character.html#SIZE[java 9])
+** [[painless-api-reference-Character-SPACE_SEPARATOR]]static byte link:{java8-javadoc}/java/lang/Character.html#SPACE_SEPARATOR[SPACE_SEPARATOR] (link:{java9-javadoc}/java/lang/Character.html#SPACE_SEPARATOR[java 9])
+** [[painless-api-reference-Character-START_PUNCTUATION]]static byte link:{java8-javadoc}/java/lang/Character.html#START_PUNCTUATION[START_PUNCTUATION] (link:{java9-javadoc}/java/lang/Character.html#START_PUNCTUATION[java 9])
+** [[painless-api-reference-Character-SURROGATE]]static byte link:{java8-javadoc}/java/lang/Character.html#SURROGATE[SURROGATE] (link:{java9-javadoc}/java/lang/Character.html#SURROGATE[java 9])
+** [[painless-api-reference-Character-TITLECASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#TITLECASE_LETTER[TITLECASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#TITLECASE_LETTER[java 9])
+** [[painless-api-reference-Character-UNASSIGNED]]static byte link:{java8-javadoc}/java/lang/Character.html#UNASSIGNED[UNASSIGNED] (link:{java9-javadoc}/java/lang/Character.html#UNASSIGNED[java 9])
+** [[painless-api-reference-Character-UPPERCASE_LETTER]]static byte link:{java8-javadoc}/java/lang/Character.html#UPPERCASE_LETTER[UPPERCASE_LETTER] (link:{java9-javadoc}/java/lang/Character.html#UPPERCASE_LETTER[java 9])
+* ++[[painless-api-reference-Character-charCount-1]]static int link:{java8-javadoc}/java/lang/Character.html#charCount%2Dint%2D[charCount](int)++ (link:{java9-javadoc}/java/lang/Character.html#charCount%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointAt-2]]static int link:{java8-javadoc}/java/lang/Character.html#codePointAt%2Djava.lang.CharSequence%2Dint%2D[codePointAt](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointAt%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointAt-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointAt%2Dchar:A%2Dint%2Dint%2D[codePointAt](char[], int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointAt%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointBefore-2]]static int link:{java8-javadoc}/java/lang/Character.html#codePointBefore%2Djava.lang.CharSequence%2Dint%2D[codePointBefore](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointBefore%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointBefore-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointBefore%2Dchar:A%2Dint%2Dint%2D[codePointBefore](char[], int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointBefore%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-codePointCount-3]]static int link:{java8-javadoc}/java/lang/Character.html#codePointCount%2Djava.lang.CharSequence%2Dint%2Dint%2D[codePointCount](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#codePointCount%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-compare-2]]static int link:{java8-javadoc}/java/lang/Character.html#compare%2Dchar%2Dchar%2D[compare](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#compare%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-digit-2]]static int link:{java8-javadoc}/java/lang/Character.html#digit%2Dint%2Dint%2D[digit](int, int)++ (link:{java9-javadoc}/java/lang/Character.html#digit%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-forDigit-2]]static char link:{java8-javadoc}/java/lang/Character.html#forDigit%2Dint%2Dint%2D[forDigit](int, int)++ (link:{java9-javadoc}/java/lang/Character.html#forDigit%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getDirectionality-1]]static byte link:{java8-javadoc}/java/lang/Character.html#getDirectionality%2Dint%2D[getDirectionality](int)++ (link:{java9-javadoc}/java/lang/Character.html#getDirectionality%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getName-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Character.html#getName%2Dint%2D[getName](int)++ (link:{java9-javadoc}/java/lang/Character.html#getName%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getNumericValue-1]]static int link:{java8-javadoc}/java/lang/Character.html#getNumericValue%2Dint%2D[getNumericValue](int)++ (link:{java9-javadoc}/java/lang/Character.html#getNumericValue%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-getType-1]]static int link:{java8-javadoc}/java/lang/Character.html#getType%2Dint%2D[getType](int)++ (link:{java9-javadoc}/java/lang/Character.html#getType%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-hashCode-1]]static int link:{java8-javadoc}/java/lang/Character.html#hashCode%2Dchar%2D[hashCode](char)++ (link:{java9-javadoc}/java/lang/Character.html#hashCode%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-highSurrogate-1]]static char link:{java8-javadoc}/java/lang/Character.html#highSurrogate%2Dint%2D[highSurrogate](int)++ (link:{java9-javadoc}/java/lang/Character.html#highSurrogate%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isAlphabetic-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isAlphabetic%2Dint%2D[isAlphabetic](int)++ (link:{java9-javadoc}/java/lang/Character.html#isAlphabetic%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isBmpCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isBmpCodePoint%2Dint%2D[isBmpCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isBmpCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isDefined-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isDefined%2Dint%2D[isDefined](int)++ (link:{java9-javadoc}/java/lang/Character.html#isDefined%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isDigit-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isDigit%2Dint%2D[isDigit](int)++ (link:{java9-javadoc}/java/lang/Character.html#isDigit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isHighSurrogate-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isHighSurrogate%2Dchar%2D[isHighSurrogate](char)++ (link:{java9-javadoc}/java/lang/Character.html#isHighSurrogate%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isISOControl-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isISOControl%2Dint%2D[isISOControl](int)++ (link:{java9-javadoc}/java/lang/Character.html#isISOControl%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isIdentifierIgnorable-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isIdentifierIgnorable%2Dint%2D[isIdentifierIgnorable](int)++ (link:{java9-javadoc}/java/lang/Character.html#isIdentifierIgnorable%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isIdeographic-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isIdeographic%2Dint%2D[isIdeographic](int)++ (link:{java9-javadoc}/java/lang/Character.html#isIdeographic%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isJavaIdentifierPart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isJavaIdentifierPart%2Dint%2D[isJavaIdentifierPart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isJavaIdentifierPart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isJavaIdentifierStart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isJavaIdentifierStart%2Dint%2D[isJavaIdentifierStart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isJavaIdentifierStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLetter-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLetter%2Dint%2D[isLetter](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLetter%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLetterOrDigit-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLetterOrDigit%2Dint%2D[isLetterOrDigit](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLetterOrDigit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isLowerCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isLowerCase%2Dint%2D[isLowerCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isLowerCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isMirrored-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isMirrored%2Dint%2D[isMirrored](int)++ (link:{java9-javadoc}/java/lang/Character.html#isMirrored%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSpaceChar-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSpaceChar%2Dint%2D[isSpaceChar](int)++ (link:{java9-javadoc}/java/lang/Character.html#isSpaceChar%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSupplementaryCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSupplementaryCodePoint%2Dint%2D[isSupplementaryCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isSupplementaryCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isSurrogate-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSurrogate%2Dchar%2D[isSurrogate](char)++ (link:{java9-javadoc}/java/lang/Character.html#isSurrogate%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isSurrogatePair-2]]static boolean link:{java8-javadoc}/java/lang/Character.html#isSurrogatePair%2Dchar%2Dchar%2D[isSurrogatePair](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#isSurrogatePair%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-isTitleCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isTitleCase%2Dint%2D[isTitleCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isTitleCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUnicodeIdentifierPart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUnicodeIdentifierPart%2Dint%2D[isUnicodeIdentifierPart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUnicodeIdentifierPart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUnicodeIdentifierStart-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUnicodeIdentifierStart%2Dint%2D[isUnicodeIdentifierStart](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUnicodeIdentifierStart%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isUpperCase-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isUpperCase%2Dint%2D[isUpperCase](int)++ (link:{java9-javadoc}/java/lang/Character.html#isUpperCase%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isValidCodePoint-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isValidCodePoint%2Dint%2D[isValidCodePoint](int)++ (link:{java9-javadoc}/java/lang/Character.html#isValidCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-isWhitespace-1]]static boolean link:{java8-javadoc}/java/lang/Character.html#isWhitespace%2Dint%2D[isWhitespace](int)++ (link:{java9-javadoc}/java/lang/Character.html#isWhitespace%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-lowSurrogate-1]]static char link:{java8-javadoc}/java/lang/Character.html#lowSurrogate%2Dint%2D[lowSurrogate](int)++ (link:{java9-javadoc}/java/lang/Character.html#lowSurrogate%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-offsetByCodePoints-3]]static int link:{java8-javadoc}/java/lang/Character.html#offsetByCodePoints%2Djava.lang.CharSequence%2Dint%2Dint%2D[offsetByCodePoints](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#offsetByCodePoints%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-offsetByCodePoints-5]]static int link:{java8-javadoc}/java/lang/Character.html#offsetByCodePoints%2Dchar:A%2Dint%2Dint%2Dint%2Dint%2D[offsetByCodePoints](char[], int, int, int, int)++ (link:{java9-javadoc}/java/lang/Character.html#offsetByCodePoints%2Dchar:A%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-reverseBytes-1]]static char link:{java8-javadoc}/java/lang/Character.html#reverseBytes%2Dchar%2D[reverseBytes](char)++ (link:{java9-javadoc}/java/lang/Character.html#reverseBytes%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toChars-1]]static char[] link:{java8-javadoc}/java/lang/Character.html#toChars%2Dint%2D[toChars](int)++ (link:{java9-javadoc}/java/lang/Character.html#toChars%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-toChars-3]]static int link:{java8-javadoc}/java/lang/Character.html#toChars%2Dint%2Dchar:A%2Dint%2D[toChars](int, char[], int)++ (link:{java9-javadoc}/java/lang/Character.html#toChars%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-Character-toCodePoint-2]]static int link:{java8-javadoc}/java/lang/Character.html#toCodePoint%2Dchar%2Dchar%2D[toCodePoint](char, char)++ (link:{java9-javadoc}/java/lang/Character.html#toCodePoint%2Dchar%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toLowerCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toLowerCase%2Dchar%2D[toLowerCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toLowerCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Character.html#toString%2Dchar%2D[toString](char)++ (link:{java9-javadoc}/java/lang/Character.html#toString%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toTitleCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toTitleCase%2Dchar%2D[toTitleCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toTitleCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-toUpperCase-1]]static char link:{java8-javadoc}/java/lang/Character.html#toUpperCase%2Dchar%2D[toUpperCase](char)++ (link:{java9-javadoc}/java/lang/Character.html#toUpperCase%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-valueOf-1]]static <<painless-api-reference-Character,Character>> link:{java8-javadoc}/java/lang/Character.html#valueOf%2Dchar%2D[valueOf](char)++ (link:{java9-javadoc}/java/lang/Character.html#valueOf%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Character-charValue-0]]char link:{java8-javadoc}/java/lang/Character.html#charValue%2D%2D[charValue]()++ (link:{java9-javadoc}/java/lang/Character.html#charValue%2D%2D[java 9])
+* ++[[painless-api-reference-Character-compareTo-1]]int link:{java8-javadoc}/java/lang/Character.html#compareTo%2Djava.lang.Character%2D[compareTo](<<painless-api-reference-Character,Character>>)++ (link:{java9-javadoc}/java/lang/Character.html#compareTo%2Djava.lang.Character%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/CharacterIterator.asciidoc b/docs/reference/painless-api-reference/CharacterIterator.asciidoc
new file mode 100644
index 00000000000..1303e2d4739
--- /dev/null
+++ b/docs/reference/painless-api-reference/CharacterIterator.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CharacterIterator]]++CharacterIterator++::
+** [[painless-api-reference-CharacterIterator-DONE]]static char link:{java8-javadoc}/java/text/CharacterIterator.html#DONE[DONE] (link:{java9-javadoc}/java/text/CharacterIterator.html#DONE[java 9])
+* ++[[painless-api-reference-CharacterIterator-clone-0]]def link:{java8-javadoc}/java/text/CharacterIterator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-current-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#current%2D%2D[current]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#current%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-first-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getBeginIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getBeginIndex%2D%2D[getBeginIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getBeginIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getEndIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getEndIndex%2D%2D[getEndIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getEndIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-getIndex-0]]int link:{java8-javadoc}/java/text/CharacterIterator.html#getIndex%2D%2D[getIndex]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#getIndex%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-last-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-next-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-previous-0]]char link:{java8-javadoc}/java/text/CharacterIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/CharacterIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-CharacterIterator-setIndex-1]]char link:{java8-javadoc}/java/text/CharacterIterator.html#setIndex%2Dint%2D[setIndex](int)++ (link:{java9-javadoc}/java/text/CharacterIterator.html#setIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ChoiceFormat.asciidoc b/docs/reference/painless-api-reference/ChoiceFormat.asciidoc
new file mode 100644
index 00000000000..426120a194f
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChoiceFormat.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChoiceFormat]]++ChoiceFormat++::
+* ++[[painless-api-reference-ChoiceFormat-nextDouble-1]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2D[nextDouble](double)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-nextDouble-2]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2Dboolean%2D[nextDouble](double, boolean)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#nextDouble%2Ddouble%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-previousDouble-1]]static double link:{java8-javadoc}/java/text/ChoiceFormat.html#previousDouble%2Ddouble%2D[previousDouble](double)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#previousDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-ChoiceFormat-1]]link:{java8-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Djava.lang.String%2D[ChoiceFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-ChoiceFormat-2]]link:{java8-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Ddouble:A%2Djava.lang.String:A%2D[ChoiceFormat](double[], <<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#ChoiceFormat%2Ddouble:A%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/ChoiceFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-getFormats-0]]def[] link:{java8-javadoc}/java/text/ChoiceFormat.html#getFormats%2D%2D[getFormats]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#getFormats%2D%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-getLimits-0]]double[] link:{java8-javadoc}/java/text/ChoiceFormat.html#getLimits%2D%2D[getLimits]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#getLimits%2D%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-setChoices-2]]void link:{java8-javadoc}/java/text/ChoiceFormat.html#setChoices%2Ddouble:A%2Djava.lang.String:A%2D[setChoices](double[], <<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#setChoices%2Ddouble:A%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-ChoiceFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/ChoiceFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/ChoiceFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-NumberFormat,NumberFormat>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ChronoField.asciidoc b/docs/reference/painless-api-reference/ChronoField.asciidoc
new file mode 100644
index 00000000000..33143a52eaf
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoField.asciidoc
@@ -0,0 +1,41 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoField]]++ChronoField++::
+** [[painless-api-reference-ChronoField-ALIGNED_DAY_OF_WEEK_IN_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_MONTH[ALIGNED_DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_DAY_OF_WEEK_IN_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_YEAR[ALIGNED_DAY_OF_WEEK_IN_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_DAY_OF_WEEK_IN_YEAR[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_WEEK_OF_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_MONTH[ALIGNED_WEEK_OF_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-ChronoField-ALIGNED_WEEK_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_YEAR[ALIGNED_WEEK_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ALIGNED_WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-AMPM_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#AMPM_OF_DAY[AMPM_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#AMPM_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-CLOCK_HOUR_OF_AMPM]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_AMPM[CLOCK_HOUR_OF_AMPM] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_AMPM[java 9])
+** [[painless-api-reference-ChronoField-CLOCK_HOUR_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_DAY[CLOCK_HOUR_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#CLOCK_HOUR_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_WEEK]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-ChronoField-DAY_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-EPOCH_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#EPOCH_DAY[EPOCH_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#EPOCH_DAY[java 9])
+** [[painless-api-reference-ChronoField-ERA]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#ERA[ERA] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#ERA[java 9])
+** [[painless-api-reference-ChronoField-HOUR_OF_AMPM]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_AMPM[HOUR_OF_AMPM] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_AMPM[java 9])
+** [[painless-api-reference-ChronoField-HOUR_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_DAY[HOUR_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#HOUR_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-INSTANT_SECONDS]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#INSTANT_SECONDS[INSTANT_SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#INSTANT_SECONDS[java 9])
+** [[painless-api-reference-ChronoField-MICRO_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_DAY[MICRO_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MICRO_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_SECOND[MICRO_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MICRO_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-MILLI_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_DAY[MILLI_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MILLI_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_SECOND[MILLI_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MILLI_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-MINUTE_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_DAY[MINUTE_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-MINUTE_OF_HOUR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_HOUR[MINUTE_OF_HOUR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MINUTE_OF_HOUR[java 9])
+** [[painless-api-reference-ChronoField-MONTH_OF_YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#MONTH_OF_YEAR[MONTH_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#MONTH_OF_YEAR[java 9])
+** [[painless-api-reference-ChronoField-NANO_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_DAY[NANO_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-NANO_OF_SECOND]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_SECOND[NANO_OF_SECOND] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#NANO_OF_SECOND[java 9])
+** [[painless-api-reference-ChronoField-OFFSET_SECONDS]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#OFFSET_SECONDS[OFFSET_SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#OFFSET_SECONDS[java 9])
+** [[painless-api-reference-ChronoField-PROLEPTIC_MONTH]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#PROLEPTIC_MONTH[PROLEPTIC_MONTH] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#PROLEPTIC_MONTH[java 9])
+** [[painless-api-reference-ChronoField-SECOND_OF_DAY]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_DAY[SECOND_OF_DAY] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_DAY[java 9])
+** [[painless-api-reference-ChronoField-SECOND_OF_MINUTE]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_MINUTE[SECOND_OF_MINUTE] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#SECOND_OF_MINUTE[java 9])
+** [[painless-api-reference-ChronoField-YEAR]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#YEAR[YEAR] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#YEAR[java 9])
+** [[painless-api-reference-ChronoField-YEAR_OF_ERA]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#YEAR_OF_ERA[YEAR_OF_ERA] (link:{java9-javadoc}/java/time/temporal/ChronoField.html#YEAR_OF_ERA[java 9])
+* ++[[painless-api-reference-ChronoField-valueOf-1]]static <<painless-api-reference-ChronoField,ChronoField>> link:{java8-javadoc}/java/time/temporal/ChronoField.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChronoField-values-0]]static <<painless-api-reference-ChronoField,ChronoField>>[] link:{java8-javadoc}/java/time/temporal/ChronoField.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoField-checkValidIntValue-1]]int link:{java8-javadoc}/java/time/temporal/ChronoField.html#checkValidIntValue%2Dlong%2D[checkValidIntValue](long)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#checkValidIntValue%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ChronoField-checkValidValue-1]]long link:{java8-javadoc}/java/time/temporal/ChronoField.html#checkValidValue%2Dlong%2D[checkValidValue](long)++ (link:{java9-javadoc}/java/time/temporal/ChronoField.html#checkValidValue%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalField,TemporalField>>++
diff --git a/docs/reference/painless-api-reference/ChronoLocalDate.asciidoc b/docs/reference/painless-api-reference/ChronoLocalDate.asciidoc
new file mode 100644
index 00000000000..2c483c27bd8
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoLocalDate.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoLocalDate]]++ChronoLocalDate++::
+* ++[[painless-api-reference-ChronoLocalDate-from-1]]static <<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-atTime-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#atTime%2Djava.time.LocalTime%2D[atTime](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#atTime%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#compareTo%2Djava.time.chrono.ChronoLocalDate%2D[compareTo](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#compareTo%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-getEra-0]]<<painless-api-reference-Era,Era>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isAfter%2Djava.time.chrono.ChronoLocalDate%2D[isAfter](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isAfter%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isBefore%2Djava.time.chrono.ChronoLocalDate%2D[isBefore](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isBefore%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isEqual%2Djava.time.chrono.ChronoLocalDate%2D[isEqual](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isEqual%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-isLeapYear-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#isLeapYear%2D%2D[isLeapYear]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#isLeapYear%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-lengthOfMonth-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfMonth%2D%2D[lengthOfMonth]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-lengthOfYear-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfYear%2D%2D[lengthOfYear]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#lengthOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-minus-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-minus-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-plus-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-plus-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-toEpochDay-0]]long link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#toEpochDay%2D%2D[toEpochDay]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#toEpochDay%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-until-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[until](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-with-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDate-with-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/ChronoLocalDateTime.asciidoc b/docs/reference/painless-api-reference/ChronoLocalDateTime.asciidoc
new file mode 100644
index 00000000000..7b49712ee09
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoLocalDateTime.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoLocalDateTime]]++ChronoLocalDateTime++::
+* ++[[painless-api-reference-ChronoLocalDateTime-from-1]]static <<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-atZone-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#compareTo%2Djava.time.chrono.ChronoLocalDateTime%2D[compareTo](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#compareTo%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isAfter%2Djava.time.chrono.ChronoLocalDateTime%2D[isAfter](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isAfter%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isBefore%2Djava.time.chrono.ChronoLocalDateTime%2D[isBefore](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isBefore%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isEqual%2Djava.time.chrono.ChronoLocalDateTime%2D[isEqual](<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#isEqual%2Djava.time.chrono.ChronoLocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-minus-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-minus-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-plus-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-plus-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toEpochSecond-1]]long link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toEpochSecond%2Djava.time.ZoneOffset%2D[toEpochSecond](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toEpochSecond%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toInstant-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toInstant%2Djava.time.ZoneOffset%2D[toInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toLocalDate-0]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-with-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoLocalDateTime-with-2]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoLocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/ChronoPeriod.asciidoc b/docs/reference/painless-api-reference/ChronoPeriod.asciidoc
new file mode 100644
index 00000000000..1b15df31c76
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoPeriod.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoPeriod]]++ChronoPeriod++::
+* ++[[painless-api-reference-ChronoPeriod-between-2]]static <<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#between%2Djava.time.chrono.ChronoLocalDate%2Djava.time.chrono.ChronoLocalDate%2D[between](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>, <<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#between%2Djava.time.chrono.ChronoLocalDate%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-getUnits-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#getUnits%2D%2D[getUnits]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#getUnits%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-isNegative-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#isNegative%2D%2D[isNegative]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#isNegative%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-isZero-0]]boolean link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#isZero%2D%2D[isZero]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#isZero%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-minus-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-multipliedBy-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#multipliedBy%2Dint%2D[multipliedBy](int)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#multipliedBy%2Dint%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-negated-0]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-normalized-0]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#normalized%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-plus-1]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoPeriod-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoPeriod.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoPeriod.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/reference/painless-api-reference/ChronoUnit.asciidoc b/docs/reference/painless-api-reference/ChronoUnit.asciidoc
new file mode 100644
index 00000000000..297fbf61835
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoUnit.asciidoc
@@ -0,0 +1,25 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoUnit]]++ChronoUnit++::
+** [[painless-api-reference-ChronoUnit-CENTURIES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#CENTURIES[CENTURIES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#CENTURIES[java 9])
+** [[painless-api-reference-ChronoUnit-DAYS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#DAYS[DAYS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#DAYS[java 9])
+** [[painless-api-reference-ChronoUnit-DECADES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#DECADES[DECADES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#DECADES[java 9])
+** [[painless-api-reference-ChronoUnit-ERAS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#ERAS[ERAS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#ERAS[java 9])
+** [[painless-api-reference-ChronoUnit-FOREVER]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#FOREVER[FOREVER] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#FOREVER[java 9])
+** [[painless-api-reference-ChronoUnit-HALF_DAYS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#HALF_DAYS[HALF_DAYS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#HALF_DAYS[java 9])
+** [[painless-api-reference-ChronoUnit-HOURS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#HOURS[HOURS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#HOURS[java 9])
+** [[painless-api-reference-ChronoUnit-MICROS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MICROS[MICROS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MICROS[java 9])
+** [[painless-api-reference-ChronoUnit-MILLENNIA]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MILLENNIA[MILLENNIA] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MILLENNIA[java 9])
+** [[painless-api-reference-ChronoUnit-MILLIS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MILLIS[MILLIS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MILLIS[java 9])
+** [[painless-api-reference-ChronoUnit-MINUTES]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MINUTES[MINUTES] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MINUTES[java 9])
+** [[painless-api-reference-ChronoUnit-MONTHS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#MONTHS[MONTHS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#MONTHS[java 9])
+** [[painless-api-reference-ChronoUnit-NANOS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#NANOS[NANOS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#NANOS[java 9])
+** [[painless-api-reference-ChronoUnit-SECONDS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#SECONDS[SECONDS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#SECONDS[java 9])
+** [[painless-api-reference-ChronoUnit-WEEKS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#WEEKS[WEEKS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#WEEKS[java 9])
+** [[painless-api-reference-ChronoUnit-YEARS]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#YEARS[YEARS] (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#YEARS[java 9])
+* ++[[painless-api-reference-ChronoUnit-valueOf-1]]static <<painless-api-reference-ChronoUnit,ChronoUnit>> link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ChronoUnit-values-0]]static <<painless-api-reference-ChronoUnit,ChronoUnit>>[] link:{java8-javadoc}/java/time/temporal/ChronoUnit.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/temporal/ChronoUnit.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalUnit,TemporalUnit>>++
diff --git a/docs/reference/painless-api-reference/ChronoZonedDateTime.asciidoc b/docs/reference/painless-api-reference/ChronoZonedDateTime.asciidoc
new file mode 100644
index 00000000000..0684b8b541e
--- /dev/null
+++ b/docs/reference/painless-api-reference/ChronoZonedDateTime.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ChronoZonedDateTime]]++ChronoZonedDateTime++::
+* ++[[painless-api-reference-ChronoZonedDateTime-from-1]]static <<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#compareTo%2Djava.time.chrono.ChronoZonedDateTime%2D[compareTo](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#compareTo%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isAfter%2Djava.time.chrono.ChronoZonedDateTime%2D[isAfter](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isAfter%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isBefore%2Djava.time.chrono.ChronoZonedDateTime%2D[isBefore](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isBefore%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isEqual%2Djava.time.chrono.ChronoZonedDateTime%2D[isEqual](<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#isEqual%2Djava.time.chrono.ChronoZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-minus-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-minus-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-plus-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-plus-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toEpochSecond-0]]long link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toInstant%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalDate-0]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalDateTime-0]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-with-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-with-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withEarlierOffsetAtOverlap-0]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[withEarlierOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withLaterOffsetAtOverlap-0]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[withLaterOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withZoneSameInstant-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[withZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ChronoZonedDateTime-withZoneSameLocal-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[withZoneSameLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/ChronoZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/reference/painless-api-reference/Chronology.asciidoc b/docs/reference/painless-api-reference/Chronology.asciidoc
new file mode 100644
index 00000000000..42b395d930a
--- /dev/null
+++ b/docs/reference/painless-api-reference/Chronology.asciidoc
@@ -0,0 +1,33 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Chronology]]++Chronology++::
+* ++[[painless-api-reference-Chronology-from-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-getAvailableChronologies-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getAvailableChronologies%2D%2D[getAvailableChronologies]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getAvailableChronologies%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-of-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Chronology-ofLocale-1]]static <<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/chrono/Chronology.html#ofLocale%2Djava.util.Locale%2D[ofLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#ofLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Chronology-compareTo-1]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#compareTo%2Djava.time.chrono.Chronology%2D[compareTo](<<painless-api-reference-Chronology,Chronology>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#compareTo%2Djava.time.chrono.Chronology%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-3]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-date-4]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateEpochDay-1]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateYearDay-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-dateYearDay-3]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-equals-1]]boolean link:{java8-javadoc}/java/time/chrono/Chronology.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Chronology-eraOf-1]]<<painless-api-reference-Era,Era>> link:{java8-javadoc}/java/time/chrono/Chronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-eras-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/chrono/Chronology.html#eras%2D%2D[eras]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#eras%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-getCalendarType-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getCalendarType%2D%2D[getCalendarType]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getCalendarType%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Chronology-getId-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#getId%2D%2D[getId]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#getId%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-hashCode-0]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-isLeapYear-1]]boolean link:{java8-javadoc}/java/time/chrono/Chronology.html#isLeapYear%2Dlong%2D[isLeapYear](long)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#isLeapYear%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Chronology-localDateTime-1]]<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[localDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-period-3]]<<painless-api-reference-ChronoPeriod,ChronoPeriod>> link:{java8-javadoc}/java/time/chrono/Chronology.html#period%2Dint%2Dint%2Dint%2D[period](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#period%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-prolepticYear-2]]int link:{java8-javadoc}/java/time/chrono/Chronology.html#prolepticYear%2Djava.time.chrono.Era%2Dint%2D[prolepticYear](<<painless-api-reference-Era,Era>>, int)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#prolepticYear%2Djava.time.chrono.Era%2Dint%2D[java 9])
+* ++[[painless-api-reference-Chronology-range-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/chrono/Chronology.html#range%2Djava.time.temporal.ChronoField%2D[range](<<painless-api-reference-ChronoField,ChronoField>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#range%2Djava.time.temporal.ChronoField%2D[java 9])
+* ++[[painless-api-reference-Chronology-resolveDate-2]]<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>> link:{java8-javadoc}/java/time/chrono/Chronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-Chronology-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Chronology.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#toString%2D%2D[java 9])
+* ++[[painless-api-reference-Chronology-zonedDateTime-1]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[zonedDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Chronology-zonedDateTime-2]]<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>> link:{java8-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[zonedDateTime](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/Chronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
diff --git a/docs/reference/painless-api-reference/ClassCastException.asciidoc b/docs/reference/painless-api-reference/ClassCastException.asciidoc
new file mode 100644
index 00000000000..f837c66f9db
--- /dev/null
+++ b/docs/reference/painless-api-reference/ClassCastException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ClassCastException]]++ClassCastException++::
+* ++[[painless-api-reference-ClassCastException-ClassCastException-0]]link:{java8-javadoc}/java/lang/ClassCastException.html#ClassCastException%2D%2D[ClassCastException]()++ (link:{java9-javadoc}/java/lang/ClassCastException.html#ClassCastException%2D%2D[java 9])
+* ++[[painless-api-reference-ClassCastException-ClassCastException-1]]link:{java8-javadoc}/java/lang/ClassCastException.html#ClassCastException%2Djava.lang.String%2D[ClassCastException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ClassCastException.html#ClassCastException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ClassNotFoundException.asciidoc b/docs/reference/painless-api-reference/ClassNotFoundException.asciidoc
new file mode 100644
index 00000000000..f4d0d63aa06
--- /dev/null
+++ b/docs/reference/painless-api-reference/ClassNotFoundException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ClassNotFoundException]]++ClassNotFoundException++::
+* ++[[painless-api-reference-ClassNotFoundException-ClassNotFoundException-0]]link:{java8-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2D%2D[ClassNotFoundException]()++ (link:{java9-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2D%2D[java 9])
+* ++[[painless-api-reference-ClassNotFoundException-ClassNotFoundException-1]]link:{java8-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2Djava.lang.String%2D[ClassNotFoundException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ClassNotFoundException.html#ClassNotFoundException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Clock.asciidoc b/docs/reference/painless-api-reference/Clock.asciidoc
new file mode 100644
index 00000000000..c40a09a2fe9
--- /dev/null
+++ b/docs/reference/painless-api-reference/Clock.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Clock]]++Clock++::
+* ++[[painless-api-reference-Clock-fixed-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#fixed%2Djava.time.Instant%2Djava.time.ZoneId%2D[fixed](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/Clock.html#fixed%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-Clock-offset-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#offset%2Djava.time.Clock%2Djava.time.Duration%2D[offset](<<painless-api-reference-Clock,Clock>>, <<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Clock.html#offset%2Djava.time.Clock%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Clock-tick-2]]static <<painless-api-reference-Clock,Clock>> link:{java8-javadoc}/java/time/Clock.html#tick%2Djava.time.Clock%2Djava.time.Duration%2D[tick](<<painless-api-reference-Clock,Clock>>, <<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Clock.html#tick%2Djava.time.Clock%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Clock-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/Clock.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/Clock.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-Clock-instant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Clock.html#instant%2D%2D[instant]()++ (link:{java9-javadoc}/java/time/Clock.html#instant%2D%2D[java 9])
+* ++[[painless-api-reference-Clock-millis-0]]long link:{java8-javadoc}/java/time/Clock.html#millis%2D%2D[millis]()++ (link:{java9-javadoc}/java/time/Clock.html#millis%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/CloneNotSupportedException.asciidoc b/docs/reference/painless-api-reference/CloneNotSupportedException.asciidoc
new file mode 100644
index 00000000000..deeb87d599b
--- /dev/null
+++ b/docs/reference/painless-api-reference/CloneNotSupportedException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CloneNotSupportedException]]++CloneNotSupportedException++::
+* ++[[painless-api-reference-CloneNotSupportedException-CloneNotSupportedException-0]]link:{java8-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2D%2D[CloneNotSupportedException]()++ (link:{java9-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2D%2D[java 9])
+* ++[[painless-api-reference-CloneNotSupportedException-CloneNotSupportedException-1]]link:{java8-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2Djava.lang.String%2D[CloneNotSupportedException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/CloneNotSupportedException.html#CloneNotSupportedException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/CollationElementIterator.asciidoc b/docs/reference/painless-api-reference/CollationElementIterator.asciidoc
new file mode 100644
index 00000000000..f277709123d
--- /dev/null
+++ b/docs/reference/painless-api-reference/CollationElementIterator.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CollationElementIterator]]++CollationElementIterator++::
+** [[painless-api-reference-CollationElementIterator-NULLORDER]]static int link:{java8-javadoc}/java/text/CollationElementIterator.html#NULLORDER[NULLORDER] (link:{java9-javadoc}/java/text/CollationElementIterator.html#NULLORDER[java 9])
+* ++[[painless-api-reference-CollationElementIterator-primaryOrder-1]]static int link:{java8-javadoc}/java/text/CollationElementIterator.html#primaryOrder%2Dint%2D[primaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#primaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-secondaryOrder-1]]static short link:{java8-javadoc}/java/text/CollationElementIterator.html#secondaryOrder%2Dint%2D[secondaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#secondaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-tertiaryOrder-1]]static short link:{java8-javadoc}/java/text/CollationElementIterator.html#tertiaryOrder%2Dint%2D[tertiaryOrder](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#tertiaryOrder%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-getMaxExpansion-1]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#getMaxExpansion%2Dint%2D[getMaxExpansion](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#getMaxExpansion%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-getOffset-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-next-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-previous-0]]int link:{java8-javadoc}/java/text/CollationElementIterator.html#previous%2D%2D[previous]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#previous%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-reset-0]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#reset%2D%2D[reset]()++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#reset%2D%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-setOffset-1]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#setOffset%2Dint%2D[setOffset](int)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#setOffset%2Dint%2D[java 9])
+* ++[[painless-api-reference-CollationElementIterator-setText-1]]void link:{java8-javadoc}/java/text/CollationElementIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/CollationElementIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/CollationKey.asciidoc b/docs/reference/painless-api-reference/CollationKey.asciidoc
new file mode 100644
index 00000000000..adb6d29cf81
--- /dev/null
+++ b/docs/reference/painless-api-reference/CollationKey.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-CollationKey]]++CollationKey++::
+* ++[[painless-api-reference-CollationKey-compareTo-1]]int link:{java8-javadoc}/java/text/CollationKey.html#compareTo%2Djava.text.CollationKey%2D[compareTo](<<painless-api-reference-CollationKey,CollationKey>>)++ (link:{java9-javadoc}/java/text/CollationKey.html#compareTo%2Djava.text.CollationKey%2D[java 9])
+* ++[[painless-api-reference-CollationKey-getSourceString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/CollationKey.html#getSourceString%2D%2D[getSourceString]()++ (link:{java9-javadoc}/java/text/CollationKey.html#getSourceString%2D%2D[java 9])
+* ++[[painless-api-reference-CollationKey-toByteArray-0]]byte[] link:{java8-javadoc}/java/text/CollationKey.html#toByteArray%2D%2D[toByteArray]()++ (link:{java9-javadoc}/java/text/CollationKey.html#toByteArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collator.asciidoc b/docs/reference/painless-api-reference/Collator.asciidoc
new file mode 100644
index 00000000000..a14f29d9868
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collator.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collator]]++Collator++::
+** [[painless-api-reference-Collator-CANONICAL_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#CANONICAL_DECOMPOSITION[CANONICAL_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#CANONICAL_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-FULL_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#FULL_DECOMPOSITION[FULL_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#FULL_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-IDENTICAL]]static int link:{java8-javadoc}/java/text/Collator.html#IDENTICAL[IDENTICAL] (link:{java9-javadoc}/java/text/Collator.html#IDENTICAL[java 9])
+** [[painless-api-reference-Collator-NO_DECOMPOSITION]]static int link:{java8-javadoc}/java/text/Collator.html#NO_DECOMPOSITION[NO_DECOMPOSITION] (link:{java9-javadoc}/java/text/Collator.html#NO_DECOMPOSITION[java 9])
+** [[painless-api-reference-Collator-PRIMARY]]static int link:{java8-javadoc}/java/text/Collator.html#PRIMARY[PRIMARY] (link:{java9-javadoc}/java/text/Collator.html#PRIMARY[java 9])
+** [[painless-api-reference-Collator-SECONDARY]]static int link:{java8-javadoc}/java/text/Collator.html#SECONDARY[SECONDARY] (link:{java9-javadoc}/java/text/Collator.html#SECONDARY[java 9])
+** [[painless-api-reference-Collator-TERTIARY]]static int link:{java8-javadoc}/java/text/Collator.html#TERTIARY[TERTIARY] (link:{java9-javadoc}/java/text/Collator.html#TERTIARY[java 9])
+* ++[[painless-api-reference-Collator-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/Collator.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/Collator.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getInstance-0]]static <<painless-api-reference-Collator,Collator>> link:{java8-javadoc}/java/text/Collator.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/Collator.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getInstance-1]]static <<painless-api-reference-Collator,Collator>> link:{java8-javadoc}/java/text/Collator.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/Collator.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Collator-clone-0]]def link:{java8-javadoc}/java/text/Collator.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/Collator.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-equals-2]]boolean link:{java8-javadoc}/java/text/Collator.html#equals%2Djava.lang.String%2Djava.lang.String%2D[equals](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Collator.html#equals%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collator-getCollationKey-1]]<<painless-api-reference-CollationKey,CollationKey>> link:{java8-javadoc}/java/text/Collator.html#getCollationKey%2Djava.lang.String%2D[getCollationKey](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Collator.html#getCollationKey%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collator-getDecomposition-0]]int link:{java8-javadoc}/java/text/Collator.html#getDecomposition%2D%2D[getDecomposition]()++ (link:{java9-javadoc}/java/text/Collator.html#getDecomposition%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-getStrength-0]]int link:{java8-javadoc}/java/text/Collator.html#getStrength%2D%2D[getStrength]()++ (link:{java9-javadoc}/java/text/Collator.html#getStrength%2D%2D[java 9])
+* ++[[painless-api-reference-Collator-setDecomposition-1]]void link:{java8-javadoc}/java/text/Collator.html#setDecomposition%2Dint%2D[setDecomposition](int)++ (link:{java9-javadoc}/java/text/Collator.html#setDecomposition%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collator-setStrength-1]]void link:{java8-javadoc}/java/text/Collator.html#setStrength%2Dint%2D[setStrength](int)++ (link:{java9-javadoc}/java/text/Collator.html#setStrength%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Comparator,Comparator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collection.asciidoc b/docs/reference/painless-api-reference/Collection.asciidoc
new file mode 100644
index 00000000000..e8faba23a61
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collection.asciidoc
@@ -0,0 +1,28 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collection]]++Collection++::
+* ++[[painless-api-reference-Collection-add-1]]boolean link:{java8-javadoc}/java/util/Collection.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/Collection.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collection-addAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#addAll%2Djava.util.Collection%2D[addAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#addAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-clear-0]]void link:{java8-javadoc}/java/util/Collection.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Collection.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-collect-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Collection%2Djava.util.function.Function%2D[collect](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-collect-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Collection%2Djava.util.Collection%2Djava.util.function.Function%2D[collect](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-contains-1]]boolean link:{java8-javadoc}/java/util/Collection.html#contains%2Djava.lang.Object%2D[contains](def)++ (link:{java9-javadoc}/java/util/Collection.html#contains%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collection-containsAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#containsAll%2Djava.util.Collection%2D[containsAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#containsAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-find-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#find%2Djava.util.Collection%2Djava.util.function.Predicate%2D[find](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-findAll-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findAll%2Djava.util.Collection%2Djava.util.function.Predicate%2D[findAll](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-findResult-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Collection%2Djava.util.function.Function%2D[findResult](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-findResult-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Collection%2Djava.lang.Object%2Djava.util.function.Function%2D[findResult](def, <<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Collection-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Collection.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Collection.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-removeAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#removeAll%2Djava.util.Collection%2D[removeAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#removeAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-removeIf-1]]boolean link:{java8-javadoc}/java/util/Collection.html#removeIf%2Djava.util.function.Predicate%2D[removeIf](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/Collection.html#removeIf%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Collection-retainAll-1]]boolean link:{java8-javadoc}/java/util/Collection.html#retainAll%2Djava.util.Collection%2D[retainAll](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collection.html#retainAll%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collection-size-0]]int link:{java8-javadoc}/java/util/Collection.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Collection.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-split-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#split%2Djava.util.Collection%2Djava.util.function.Predicate%2D[split](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Collection-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Collection.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/Collection.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-stream-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/Collection.html#stream%2D%2D[stream]()++ (link:{java9-javadoc}/java/util/Collection.html#stream%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-toArray-0]]def[] link:{java8-javadoc}/java/util/Collection.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/Collection.html#toArray%2D%2D[java 9])
+* ++[[painless-api-reference-Collection-toArray-1]]def[] link:{java8-javadoc}/java/util/Collection.html#toArray%2Djava.lang.Object:A%2D[toArray](def[])++ (link:{java9-javadoc}/java/util/Collection.html#toArray%2Djava.lang.Object:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collections.asciidoc b/docs/reference/painless-api-reference/Collections.asciidoc
new file mode 100644
index 00000000000..db6d87e8521
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collections.asciidoc
@@ -0,0 +1,59 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collections]]++Collections++::
+** [[painless-api-reference-Collections-EMPTY_LIST]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_LIST[EMPTY_LIST] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_LIST[java 9])
+** [[painless-api-reference-Collections-EMPTY_MAP]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_MAP[EMPTY_MAP] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_MAP[java 9])
+** [[painless-api-reference-Collections-EMPTY_SET]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#EMPTY_SET[EMPTY_SET] (link:{java9-javadoc}/java/util/Collections.html#EMPTY_SET[java 9])
+* ++[[painless-api-reference-Collections-addAll-2]]static boolean link:{java8-javadoc}/java/util/Collections.html#addAll%2Djava.util.Collection%2Djava.lang.Object:A%2D[addAll](<<painless-api-reference-Collection,Collection>>, def[])++ (link:{java9-javadoc}/java/util/Collections.html#addAll%2Djava.util.Collection%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Collections-asLifoQueue-1]]static <<painless-api-reference-Queue,Queue>> link:{java8-javadoc}/java/util/Collections.html#asLifoQueue%2Djava.util.Deque%2D[asLifoQueue](<<painless-api-reference-Deque,Deque>>)++ (link:{java9-javadoc}/java/util/Collections.html#asLifoQueue%2Djava.util.Deque%2D[java 9])
+* ++[[painless-api-reference-Collections-binarySearch-2]]static int link:{java8-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2D[binarySearch](<<painless-api-reference-List,List>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-binarySearch-3]]static int link:{java8-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2Djava.util.Comparator%2D[binarySearch](<<painless-api-reference-List,List>>, def, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#binarySearch%2Djava.util.List%2Djava.lang.Object%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-copy-2]]static void link:{java8-javadoc}/java/util/Collections.html#copy%2Djava.util.List%2Djava.util.List%2D[copy](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#copy%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-disjoint-2]]static boolean link:{java8-javadoc}/java/util/Collections.html#disjoint%2Djava.util.Collection%2Djava.util.Collection%2D[disjoint](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#disjoint%2Djava.util.Collection%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyEnumeration-0]]static <<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Collections.html#emptyEnumeration%2D%2D[emptyEnumeration]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyEnumeration%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyIterator-0]]static <<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Collections.html#emptyIterator%2D%2D[emptyIterator]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyList-0]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#emptyList%2D%2D[emptyList]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyList%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyListIterator-0]]static <<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/Collections.html#emptyListIterator%2D%2D[emptyListIterator]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyListIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyMap-0]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#emptyMap%2D%2D[emptyMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyNavigableMap-0]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/Collections.html#emptyNavigableMap%2D%2D[emptyNavigableMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyNavigableMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptyNavigableSet-0]]static <<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/Collections.html#emptyNavigableSet%2D%2D[emptyNavigableSet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptyNavigableSet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySet-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#emptySet%2D%2D[emptySet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySortedMap-0]]static <<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/Collections.html#emptySortedMap%2D%2D[emptySortedMap]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySortedMap%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-emptySortedSet-0]]static <<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/Collections.html#emptySortedSet%2D%2D[emptySortedSet]()++ (link:{java9-javadoc}/java/util/Collections.html#emptySortedSet%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-enumeration-1]]static <<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Collections.html#enumeration%2Djava.util.Collection%2D[enumeration](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#enumeration%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-fill-2]]static void link:{java8-javadoc}/java/util/Collections.html#fill%2Djava.util.List%2Djava.lang.Object%2D[fill](<<painless-api-reference-List,List>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#fill%2Djava.util.List%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-frequency-2]]static int link:{java8-javadoc}/java/util/Collections.html#frequency%2Djava.util.Collection%2Djava.lang.Object%2D[frequency](<<painless-api-reference-Collection,Collection>>, def)++ (link:{java9-javadoc}/java/util/Collections.html#frequency%2Djava.util.Collection%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-indexOfSubList-2]]static int link:{java8-javadoc}/java/util/Collections.html#indexOfSubList%2Djava.util.List%2Djava.util.List%2D[indexOfSubList](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#indexOfSubList%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-lastIndexOfSubList-2]]static int link:{java8-javadoc}/java/util/Collections.html#lastIndexOfSubList%2Djava.util.List%2Djava.util.List%2D[lastIndexOfSubList](<<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#lastIndexOfSubList%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-list-1]]static <<painless-api-reference-ArrayList,ArrayList>> link:{java8-javadoc}/java/util/Collections.html#list%2Djava.util.Enumeration%2D[list](<<painless-api-reference-Enumeration,Enumeration>>)++ (link:{java9-javadoc}/java/util/Collections.html#list%2Djava.util.Enumeration%2D[java 9])
+* ++[[painless-api-reference-Collections-max-1]]static def link:{java8-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2D[max](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-max-2]]static def link:{java8-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2Djava.util.Comparator%2D[max](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#max%2Djava.util.Collection%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-min-1]]static def link:{java8-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2D[min](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-min-2]]static def link:{java8-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2Djava.util.Comparator%2D[min](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#min%2Djava.util.Collection%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-nCopies-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#nCopies%2Dint%2Djava.lang.Object%2D[nCopies](int, def)++ (link:{java9-javadoc}/java/util/Collections.html#nCopies%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-newSetFromMap-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#newSetFromMap%2Djava.util.Map%2D[newSetFromMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Collections.html#newSetFromMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Collections-replaceAll-3]]static boolean link:{java8-javadoc}/java/util/Collections.html#replaceAll%2Djava.util.List%2Djava.lang.Object%2Djava.lang.Object%2D[replaceAll](<<painless-api-reference-List,List>>, def, def)++ (link:{java9-javadoc}/java/util/Collections.html#replaceAll%2Djava.util.List%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-reverse-1]]static void link:{java8-javadoc}/java/util/Collections.html#reverse%2Djava.util.List%2D[reverse](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#reverse%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-reverseOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Collections.html#reverseOrder%2D%2D[reverseOrder]()++ (link:{java9-javadoc}/java/util/Collections.html#reverseOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Collections-reverseOrder-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Collections.html#reverseOrder%2Djava.util.Comparator%2D[reverseOrder](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#reverseOrder%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-rotate-2]]static void link:{java8-javadoc}/java/util/Collections.html#rotate%2Djava.util.List%2Dint%2D[rotate](<<painless-api-reference-List,List>>, int)++ (link:{java9-javadoc}/java/util/Collections.html#rotate%2Djava.util.List%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collections-shuffle-1]]static void link:{java8-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2D[shuffle](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-shuffle-2]]static void link:{java8-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2Djava.util.Random%2D[shuffle](<<painless-api-reference-List,List>>, <<painless-api-reference-Random,Random>>)++ (link:{java9-javadoc}/java/util/Collections.html#shuffle%2Djava.util.List%2Djava.util.Random%2D[java 9])
+* ++[[painless-api-reference-Collections-singleton-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#singleton%2Djava.lang.Object%2D[singleton](def)++ (link:{java9-javadoc}/java/util/Collections.html#singleton%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-singletonList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#singletonList%2Djava.lang.Object%2D[singletonList](def)++ (link:{java9-javadoc}/java/util/Collections.html#singletonList%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-singletonMap-2]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#singletonMap%2Djava.lang.Object%2Djava.lang.Object%2D[singletonMap](def, def)++ (link:{java9-javadoc}/java/util/Collections.html#singletonMap%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Collections-sort-1]]static void link:{java8-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2D[sort](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-sort-2]]static void link:{java8-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2Djava.util.Comparator%2D[sort](<<painless-api-reference-List,List>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Collections.html#sort%2Djava.util.List%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collections-swap-3]]static void link:{java8-javadoc}/java/util/Collections.html#swap%2Djava.util.List%2Dint%2Dint%2D[swap](<<painless-api-reference-List,List>>, int, int)++ (link:{java9-javadoc}/java/util/Collections.html#swap%2Djava.util.List%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableCollection-1]]static <<painless-api-reference-Collection,Collection>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableCollection%2Djava.util.Collection%2D[unmodifiableCollection](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableCollection%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableList-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableList%2Djava.util.List%2D[unmodifiableList](<<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableList%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableMap-1]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableMap%2Djava.util.Map%2D[unmodifiableMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableNavigableMap-1]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableNavigableMap%2Djava.util.NavigableMap%2D[unmodifiableNavigableMap](<<painless-api-reference-NavigableMap,NavigableMap>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableNavigableMap%2Djava.util.NavigableMap%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableNavigableSet-1]]static <<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableNavigableSet%2Djava.util.NavigableSet%2D[unmodifiableNavigableSet](<<painless-api-reference-NavigableSet,NavigableSet>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableNavigableSet%2Djava.util.NavigableSet%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSet-1]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSet%2Djava.util.Set%2D[unmodifiableSet](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSet%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSortedMap-1]]static <<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSortedMap%2Djava.util.SortedMap%2D[unmodifiableSortedMap](<<painless-api-reference-SortedMap,SortedMap>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSortedMap%2Djava.util.SortedMap%2D[java 9])
+* ++[[painless-api-reference-Collections-unmodifiableSortedSet-1]]static <<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/Collections.html#unmodifiableSortedSet%2Djava.util.SortedSet%2D[unmodifiableSortedSet](<<painless-api-reference-SortedSet,SortedSet>>)++ (link:{java9-javadoc}/java/util/Collections.html#unmodifiableSortedSet%2Djava.util.SortedSet%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collector.Characteristics.asciidoc b/docs/reference/painless-api-reference/Collector.Characteristics.asciidoc
new file mode 100644
index 00000000000..17c6bec9475
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collector.Characteristics.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collector-Characteristics]]++Collector.Characteristics++::
+** [[painless-api-reference-Collector-Characteristics-CONCURRENT]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector$Characteristics.html#CONCURRENT[CONCURRENT] (link:{java9-javadoc}/java/util/stream/Collector$Characteristics.html#CONCURRENT[java 9])
+** [[painless-api-reference-Collector-Characteristics-IDENTITY_FINISH]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector$Characteristics.html#IDENTITY_FINISH[IDENTITY_FINISH] (link:{java9-javadoc}/java/util/stream/Collector$Characteristics.html#IDENTITY_FINISH[java 9])
+** [[painless-api-reference-Collector-Characteristics-UNORDERED]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector$Characteristics.html#UNORDERED[UNORDERED] (link:{java9-javadoc}/java/util/stream/Collector$Characteristics.html#UNORDERED[java 9])
+* ++[[painless-api-reference-Collector-Characteristics-valueOf-1]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>> link:{java8-javadoc}/java/util/stream/Collector$Characteristics.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/stream/Collector$Characteristics.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Collector-Characteristics-values-0]]static <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[] link:{java8-javadoc}/java/util/stream/Collector$Characteristics.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/stream/Collector$Characteristics.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collector.asciidoc b/docs/reference/painless-api-reference/Collector.asciidoc
new file mode 100644
index 00000000000..1d1b92f1e09
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collector.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collector]]++Collector++::
+* ++[[painless-api-reference-Collector-of-4]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.stream.Collector$Characteristics:A%2D[of](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[])++ (link:{java9-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.stream.Collector$Characteristics:A%2D[java 9])
+* ++[[painless-api-reference-Collector-of-5]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.function.Function%2Djava.util.stream.Collector$Characteristics:A%2D[of](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector-Characteristics,Collector.Characteristics>>[])++ (link:{java9-javadoc}/java/util/stream/Collector.html#of%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BinaryOperator%2Djava.util.function.Function%2Djava.util.stream.Collector$Characteristics:A%2D[java 9])
+* ++[[painless-api-reference-Collector-accumulator-0]]<<painless-api-reference-BiConsumer,BiConsumer>> link:{java8-javadoc}/java/util/stream/Collector.html#accumulator%2D%2D[accumulator]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#accumulator%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-characteristics-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/stream/Collector.html#characteristics%2D%2D[characteristics]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#characteristics%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-combiner-0]]<<painless-api-reference-BinaryOperator,BinaryOperator>> link:{java8-javadoc}/java/util/stream/Collector.html#combiner%2D%2D[combiner]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#combiner%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-finisher-0]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/stream/Collector.html#finisher%2D%2D[finisher]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#finisher%2D%2D[java 9])
+* ++[[painless-api-reference-Collector-supplier-0]]<<painless-api-reference-Supplier,Supplier>> link:{java8-javadoc}/java/util/stream/Collector.html#supplier%2D%2D[supplier]()++ (link:{java9-javadoc}/java/util/stream/Collector.html#supplier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Collectors.asciidoc b/docs/reference/painless-api-reference/Collectors.asciidoc
new file mode 100644
index 00000000000..ef4d0199386
--- /dev/null
+++ b/docs/reference/painless-api-reference/Collectors.asciidoc
@@ -0,0 +1,38 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Collectors]]++Collectors++::
+* ++[[painless-api-reference-Collectors-averagingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingDouble%2Djava.util.function.ToDoubleFunction%2D[averagingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-averagingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingInt%2Djava.util.function.ToIntFunction%2D[averagingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-averagingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#averagingLong%2Djava.util.function.ToLongFunction%2D[averagingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#averagingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-collectingAndThen-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#collectingAndThen%2Djava.util.stream.Collector%2Djava.util.function.Function%2D[collectingAndThen](<<painless-api-reference-Collector,Collector>>, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#collectingAndThen%2Djava.util.stream.Collector%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-counting-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#counting%2D%2D[counting]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#counting%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2D[groupingBy](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[groupingBy](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-groupingBy-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.function.Supplier%2Djava.util.stream.Collector%2D[groupingBy](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#groupingBy%2Djava.util.function.Function%2Djava.util.function.Supplier%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2D%2D[joining]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2D[joining](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Collectors-joining-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[joining](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#joining%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Collectors-mapping-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#mapping%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[mapping](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#mapping%2Djava.util.function.Function%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-maxBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#maxBy%2Djava.util.Comparator%2D[maxBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#maxBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collectors-minBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#minBy%2Djava.util.Comparator%2D[minBy](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#minBy%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Collectors-partitioningBy-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2D[partitioningBy](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Collectors-partitioningBy-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2Djava.util.stream.Collector%2D[partitioningBy](<<painless-api-reference-Predicate,Predicate>>, <<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#partitioningBy%2Djava.util.function.Predicate%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.util.function.BinaryOperator%2D[reducing](<<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[reducing](def, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-reducing-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[reducing](def, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#reducing%2Djava.lang.Object%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingDouble%2Djava.util.function.ToDoubleFunction%2D[summarizingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingInt%2Djava.util.function.ToIntFunction%2D[summarizingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summarizingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summarizingLong%2Djava.util.function.ToLongFunction%2D[summarizingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summarizingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingDouble-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingDouble%2Djava.util.function.ToDoubleFunction%2D[summingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingInt-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingInt%2Djava.util.function.ToIntFunction%2D[summingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-summingLong-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#summingLong%2Djava.util.function.ToLongFunction%2D[summingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#summingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Collectors-toCollection-1]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toCollection%2Djava.util.function.Supplier%2D[toCollection](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toCollection%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Collectors-toList-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toList%2D%2D[toList]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toList%2D%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-2]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-3]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Collectors-toMap-4]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2Djava.util.function.Supplier%2D[toMap](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>, <<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toMap%2Djava.util.function.Function%2Djava.util.function.Function%2Djava.util.function.BinaryOperator%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Collectors-toSet-0]]static <<painless-api-reference-Collector,Collector>> link:{java8-javadoc}/java/util/stream/Collectors.html#toSet%2D%2D[toSet]()++ (link:{java9-javadoc}/java/util/stream/Collectors.html#toSet%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Comparable.asciidoc b/docs/reference/painless-api-reference/Comparable.asciidoc
new file mode 100644
index 00000000000..1550d93b511
--- /dev/null
+++ b/docs/reference/painless-api-reference/Comparable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Comparable]]++Comparable++::
+* ++[[painless-api-reference-Comparable-compareTo-1]]int link:{java8-javadoc}/java/lang/Comparable.html#compareTo%2Djava.lang.Object%2D[compareTo](def)++ (link:{java9-javadoc}/java/lang/Comparable.html#compareTo%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Comparator.asciidoc b/docs/reference/painless-api-reference/Comparator.asciidoc
new file mode 100644
index 00000000000..25545376942
--- /dev/null
+++ b/docs/reference/painless-api-reference/Comparator.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Comparator]]++Comparator++::
+* ++[[painless-api-reference-Comparator-comparing-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2D[comparing](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparing-2]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[comparing](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingDouble-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingDouble%2Djava.util.function.ToDoubleFunction%2D[comparingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingInt-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingInt%2Djava.util.function.ToIntFunction%2D[comparingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-comparingLong-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#comparingLong%2Djava.util.function.ToLongFunction%2D[comparingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#comparingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-naturalOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#naturalOrder%2D%2D[naturalOrder]()++ (link:{java9-javadoc}/java/util/Comparator.html#naturalOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-nullsFirst-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#nullsFirst%2Djava.util.Comparator%2D[nullsFirst](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#nullsFirst%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-nullsLast-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#nullsLast%2Djava.util.Comparator%2D[nullsLast](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#nullsLast%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-reverseOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#reverseOrder%2D%2D[reverseOrder]()++ (link:{java9-javadoc}/java/util/Comparator.html#reverseOrder%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-compare-2]]int link:{java8-javadoc}/java/util/Comparator.html#compare%2Djava.lang.Object%2Djava.lang.Object%2D[compare](def, def)++ (link:{java9-javadoc}/java/util/Comparator.html#compare%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Comparator-equals-1]]boolean link:{java8-javadoc}/java/util/Comparator.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Comparator.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Comparator-reversed-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#reversed%2D%2D[reversed]()++ (link:{java9-javadoc}/java/util/Comparator.html#reversed%2D%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparing-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.Comparator%2D[thenComparing](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparing-2]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[thenComparing](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparing%2Djava.util.function.Function%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingDouble-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingDouble%2Djava.util.function.ToDoubleFunction%2D[thenComparingDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingInt-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingInt%2Djava.util.function.ToIntFunction%2D[thenComparingInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Comparator-thenComparingLong-1]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Comparator.html#thenComparingLong%2Djava.util.function.ToLongFunction%2D[thenComparingLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/Comparator.html#thenComparingLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ConcurrentModificationException.asciidoc b/docs/reference/painless-api-reference/ConcurrentModificationException.asciidoc
new file mode 100644
index 00000000000..b33eefee1e5
--- /dev/null
+++ b/docs/reference/painless-api-reference/ConcurrentModificationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ConcurrentModificationException]]++ConcurrentModificationException++::
+* ++[[painless-api-reference-ConcurrentModificationException-ConcurrentModificationException-0]]link:{java8-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2D%2D[ConcurrentModificationException]()++ (link:{java9-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2D%2D[java 9])
+* ++[[painless-api-reference-ConcurrentModificationException-ConcurrentModificationException-1]]link:{java8-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2Djava.lang.String%2D[ConcurrentModificationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/ConcurrentModificationException.html#ConcurrentModificationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Consumer.asciidoc b/docs/reference/painless-api-reference/Consumer.asciidoc
new file mode 100644
index 00000000000..a93f47e1a1a
--- /dev/null
+++ b/docs/reference/painless-api-reference/Consumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Consumer]]++Consumer++::
+* ++[[painless-api-reference-Consumer-accept-1]]void link:{java8-javadoc}/java/util/function/Consumer.html#accept%2Djava.lang.Object%2D[accept](def)++ (link:{java9-javadoc}/java/util/function/Consumer.html#accept%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Consumer-andThen-1]]<<painless-api-reference-Consumer,Consumer>> link:{java8-javadoc}/java/util/function/Consumer.html#andThen%2Djava.util.function.Consumer%2D[andThen](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/function/Consumer.html#andThen%2Djava.util.function.Consumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Currency.asciidoc b/docs/reference/painless-api-reference/Currency.asciidoc
new file mode 100644
index 00000000000..12e172e01ee
--- /dev/null
+++ b/docs/reference/painless-api-reference/Currency.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Currency]]++Currency++::
+* ++[[painless-api-reference-Currency-getAvailableCurrencies-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Currency.html#getAvailableCurrencies%2D%2D[getAvailableCurrencies]()++ (link:{java9-javadoc}/java/util/Currency.html#getAvailableCurrencies%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getInstance-1]]static <<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/util/Currency.html#getInstance%2Djava.lang.String%2D[getInstance](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Currency.html#getInstance%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Currency-getCurrencyCode-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getCurrencyCode%2D%2D[getCurrencyCode]()++ (link:{java9-javadoc}/java/util/Currency.html#getCurrencyCode%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDefaultFractionDigits-0]]int link:{java8-javadoc}/java/util/Currency.html#getDefaultFractionDigits%2D%2D[getDefaultFractionDigits]()++ (link:{java9-javadoc}/java/util/Currency.html#getDefaultFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/Currency.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Currency.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Currency-getNumericCode-0]]int link:{java8-javadoc}/java/util/Currency.html#getNumericCode%2D%2D[getNumericCode]()++ (link:{java9-javadoc}/java/util/Currency.html#getNumericCode%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getSymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getSymbol%2D%2D[getSymbol]()++ (link:{java9-javadoc}/java/util/Currency.html#getSymbol%2D%2D[java 9])
+* ++[[painless-api-reference-Currency-getSymbol-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Currency.html#getSymbol%2Djava.util.Locale%2D[getSymbol](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Currency.html#getSymbol%2Djava.util.Locale%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Date.asciidoc b/docs/reference/painless-api-reference/Date.asciidoc
new file mode 100644
index 00000000000..5b83f70a0ea
--- /dev/null
+++ b/docs/reference/painless-api-reference/Date.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Date]]++Date++::
+* ++[[painless-api-reference-Date-from-1]]static <<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/Date.html#from%2Djava.time.Instant%2D[from](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/util/Date.html#from%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Date-Date-0]]link:{java8-javadoc}/java/util/Date.html#Date%2D%2D[Date]()++ (link:{java9-javadoc}/java/util/Date.html#Date%2D%2D[java 9])
+* ++[[painless-api-reference-Date-Date-1]]link:{java8-javadoc}/java/util/Date.html#Date%2Dlong%2D[Date](long)++ (link:{java9-javadoc}/java/util/Date.html#Date%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Date-after-1]]boolean link:{java8-javadoc}/java/util/Date.html#after%2Djava.util.Date%2D[after](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#after%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-before-1]]boolean link:{java8-javadoc}/java/util/Date.html#before%2Djava.util.Date%2D[before](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#before%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-clone-0]]def link:{java8-javadoc}/java/util/Date.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Date.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Date-compareTo-1]]int link:{java8-javadoc}/java/util/Date.html#compareTo%2Djava.util.Date%2D[compareTo](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/Date.html#compareTo%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-Date-getTime-0]]long link:{java8-javadoc}/java/util/Date.html#getTime%2D%2D[getTime]()++ (link:{java9-javadoc}/java/util/Date.html#getTime%2D%2D[java 9])
+* ++[[painless-api-reference-Date-setTime-1]]void link:{java8-javadoc}/java/util/Date.html#setTime%2Dlong%2D[setTime](long)++ (link:{java9-javadoc}/java/util/Date.html#setTime%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateFormat.Field.asciidoc b/docs/reference/painless-api-reference/DateFormat.Field.asciidoc
new file mode 100644
index 00000000000..b9ad2cda292
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateFormat.Field.asciidoc
@@ -0,0 +1,27 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormat-Field]]++DateFormat.Field++::
+** [[painless-api-reference-DateFormat-Field-AM_PM]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#AM_PM[AM_PM] (link:{java9-javadoc}/java/text/DateFormat$Field.html#AM_PM[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#DAY_OF_MONTH[DAY_OF_MONTH] (link:{java9-javadoc}/java/text/DateFormat$Field.html#DAY_OF_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_WEEK]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#DAY_OF_WEEK[DAY_OF_WEEK] (link:{java9-javadoc}/java/text/DateFormat$Field.html#DAY_OF_WEEK[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_WEEK_IN_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#DAY_OF_WEEK_IN_MONTH[DAY_OF_WEEK_IN_MONTH] (link:{java9-javadoc}/java/text/DateFormat$Field.html#DAY_OF_WEEK_IN_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-DAY_OF_YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#DAY_OF_YEAR[DAY_OF_YEAR] (link:{java9-javadoc}/java/text/DateFormat$Field.html#DAY_OF_YEAR[java 9])
+** [[painless-api-reference-DateFormat-Field-ERA]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#ERA[ERA] (link:{java9-javadoc}/java/text/DateFormat$Field.html#ERA[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR0]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#HOUR0[HOUR0] (link:{java9-javadoc}/java/text/DateFormat$Field.html#HOUR0[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#HOUR1[HOUR1] (link:{java9-javadoc}/java/text/DateFormat$Field.html#HOUR1[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR_OF_DAY0]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#HOUR_OF_DAY0[HOUR_OF_DAY0] (link:{java9-javadoc}/java/text/DateFormat$Field.html#HOUR_OF_DAY0[java 9])
+** [[painless-api-reference-DateFormat-Field-HOUR_OF_DAY1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#HOUR_OF_DAY1[HOUR_OF_DAY1] (link:{java9-javadoc}/java/text/DateFormat$Field.html#HOUR_OF_DAY1[java 9])
+** [[painless-api-reference-DateFormat-Field-MILLISECOND]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#MILLISECOND[MILLISECOND] (link:{java9-javadoc}/java/text/DateFormat$Field.html#MILLISECOND[java 9])
+** [[painless-api-reference-DateFormat-Field-MINUTE]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#MINUTE[MINUTE] (link:{java9-javadoc}/java/text/DateFormat$Field.html#MINUTE[java 9])
+** [[painless-api-reference-DateFormat-Field-MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#MONTH[MONTH] (link:{java9-javadoc}/java/text/DateFormat$Field.html#MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-SECOND]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#SECOND[SECOND] (link:{java9-javadoc}/java/text/DateFormat$Field.html#SECOND[java 9])
+** [[painless-api-reference-DateFormat-Field-TIME_ZONE]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#TIME_ZONE[TIME_ZONE] (link:{java9-javadoc}/java/text/DateFormat$Field.html#TIME_ZONE[java 9])
+** [[painless-api-reference-DateFormat-Field-WEEK_OF_MONTH]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#WEEK_OF_MONTH[WEEK_OF_MONTH] (link:{java9-javadoc}/java/text/DateFormat$Field.html#WEEK_OF_MONTH[java 9])
+** [[painless-api-reference-DateFormat-Field-WEEK_OF_YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#WEEK_OF_YEAR[WEEK_OF_YEAR] (link:{java9-javadoc}/java/text/DateFormat$Field.html#WEEK_OF_YEAR[java 9])
+** [[painless-api-reference-DateFormat-Field-YEAR]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#YEAR[YEAR] (link:{java9-javadoc}/java/text/DateFormat$Field.html#YEAR[java 9])
+* ++[[painless-api-reference-DateFormat-Field-ofCalendarField-1]]static <<painless-api-reference-DateFormat-Field,DateFormat.Field>> link:{java8-javadoc}/java/text/DateFormat$Field.html#ofCalendarField%2Dint%2D[ofCalendarField](int)++ (link:{java9-javadoc}/java/text/DateFormat$Field.html#ofCalendarField%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-Field-getCalendarField-0]]int link:{java8-javadoc}/java/text/DateFormat$Field.html#getCalendarField%2D%2D[getCalendarField]()++ (link:{java9-javadoc}/java/text/DateFormat$Field.html#getCalendarField%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateFormat.asciidoc b/docs/reference/painless-api-reference/DateFormat.asciidoc
new file mode 100644
index 00000000000..8bb4fdf2eff
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateFormat.asciidoc
@@ -0,0 +1,51 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormat]]++DateFormat++::
+** [[painless-api-reference-DateFormat-AM_PM_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#AM_PM_FIELD[AM_PM_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#AM_PM_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DATE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DATE_FIELD[DATE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DATE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_WEEK_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_FIELD[DAY_OF_WEEK_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_WEEK_IN_MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_IN_MONTH_FIELD[DAY_OF_WEEK_IN_MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_WEEK_IN_MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DAY_OF_YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#DAY_OF_YEAR_FIELD[DAY_OF_YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#DAY_OF_YEAR_FIELD[java 9])
+** [[painless-api-reference-DateFormat-DEFAULT]]static int link:{java8-javadoc}/java/text/DateFormat.html#DEFAULT[DEFAULT] (link:{java9-javadoc}/java/text/DateFormat.html#DEFAULT[java 9])
+** [[painless-api-reference-DateFormat-ERA_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#ERA_FIELD[ERA_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#ERA_FIELD[java 9])
+** [[painless-api-reference-DateFormat-FULL]]static int link:{java8-javadoc}/java/text/DateFormat.html#FULL[FULL] (link:{java9-javadoc}/java/text/DateFormat.html#FULL[java 9])
+** [[painless-api-reference-DateFormat-HOUR0_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR0_FIELD[HOUR0_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR0_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR1_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR1_FIELD[HOUR1_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR1_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR_OF_DAY0_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY0_FIELD[HOUR_OF_DAY0_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY0_FIELD[java 9])
+** [[painless-api-reference-DateFormat-HOUR_OF_DAY1_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY1_FIELD[HOUR_OF_DAY1_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#HOUR_OF_DAY1_FIELD[java 9])
+** [[painless-api-reference-DateFormat-LONG]]static int link:{java8-javadoc}/java/text/DateFormat.html#LONG[LONG] (link:{java9-javadoc}/java/text/DateFormat.html#LONG[java 9])
+** [[painless-api-reference-DateFormat-MEDIUM]]static int link:{java8-javadoc}/java/text/DateFormat.html#MEDIUM[MEDIUM] (link:{java9-javadoc}/java/text/DateFormat.html#MEDIUM[java 9])
+** [[painless-api-reference-DateFormat-MILLISECOND_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MILLISECOND_FIELD[MILLISECOND_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MILLISECOND_FIELD[java 9])
+** [[painless-api-reference-DateFormat-MINUTE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MINUTE_FIELD[MINUTE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MINUTE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#MONTH_FIELD[MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-SECOND_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#SECOND_FIELD[SECOND_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#SECOND_FIELD[java 9])
+** [[painless-api-reference-DateFormat-SHORT]]static int link:{java8-javadoc}/java/text/DateFormat.html#SHORT[SHORT] (link:{java9-javadoc}/java/text/DateFormat.html#SHORT[java 9])
+** [[painless-api-reference-DateFormat-TIMEZONE_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#TIMEZONE_FIELD[TIMEZONE_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#TIMEZONE_FIELD[java 9])
+** [[painless-api-reference-DateFormat-WEEK_OF_MONTH_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#WEEK_OF_MONTH_FIELD[WEEK_OF_MONTH_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#WEEK_OF_MONTH_FIELD[java 9])
+** [[painless-api-reference-DateFormat-WEEK_OF_YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#WEEK_OF_YEAR_FIELD[WEEK_OF_YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#WEEK_OF_YEAR_FIELD[java 9])
+** [[painless-api-reference-DateFormat-YEAR_FIELD]]static int link:{java8-javadoc}/java/text/DateFormat.html#YEAR_FIELD[YEAR_FIELD] (link:{java9-javadoc}/java/text/DateFormat.html#YEAR_FIELD[java 9])
+* ++[[painless-api-reference-DateFormat-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DateFormat.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2D%2D[getDateInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-1]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2D[getDateInstance](int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2Djava.util.Locale%2D[getDateInstance](int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateInstance%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2D%2D[getDateTimeInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2D[getDateTimeInstance](int, int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getDateTimeInstance-3]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2Djava.util.Locale%2D[getDateTimeInstance](int, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getDateTimeInstance%2Dint%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-0]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2D%2D[getTimeInstance]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-1]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2D[getTimeInstance](int)++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeInstance-2]]static <<painless-api-reference-DateFormat,DateFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2Djava.util.Locale%2D[getTimeInstance](int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeInstance%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getCalendar-0]]<<painless-api-reference-Calendar,Calendar>> link:{java8-javadoc}/java/text/DateFormat.html#getCalendar%2D%2D[getCalendar]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getCalendar%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getNumberFormat-0]]<<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/DateFormat.html#getNumberFormat%2D%2D[getNumberFormat]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getNumberFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-getTimeZone-0]]<<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/text/DateFormat.html#getTimeZone%2D%2D[getTimeZone]()++ (link:{java9-javadoc}/java/text/DateFormat.html#getTimeZone%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-isLenient-0]]boolean link:{java8-javadoc}/java/text/DateFormat.html#isLenient%2D%2D[isLenient]()++ (link:{java9-javadoc}/java/text/DateFormat.html#isLenient%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormat-parse-1]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateFormat-parse-2]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setCalendar-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setCalendar%2Djava.util.Calendar%2D[setCalendar](<<painless-api-reference-Calendar,Calendar>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setCalendar%2Djava.util.Calendar%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setLenient-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setLenient%2Dboolean%2D[setLenient](boolean)++ (link:{java9-javadoc}/java/text/DateFormat.html#setLenient%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setNumberFormat-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setNumberFormat%2Djava.text.NumberFormat%2D[setNumberFormat](<<painless-api-reference-NumberFormat,NumberFormat>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setNumberFormat%2Djava.text.NumberFormat%2D[java 9])
+* ++[[painless-api-reference-DateFormat-setTimeZone-1]]void link:{java8-javadoc}/java/text/DateFormat.html#setTimeZone%2Djava.util.TimeZone%2D[setTimeZone](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/text/DateFormat.html#setTimeZone%2Djava.util.TimeZone%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateFormatSymbols.asciidoc b/docs/reference/painless-api-reference/DateFormatSymbols.asciidoc
new file mode 100644
index 00000000000..31cf39bbc45
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateFormatSymbols.asciidoc
@@ -0,0 +1,30 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateFormatSymbols]]++DateFormatSymbols++::
+* ++[[painless-api-reference-DateFormatSymbols-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getInstance-0]]static <<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getInstance-1]]static <<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-DateFormatSymbols-0]]link:{java8-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2D%2D[DateFormatSymbols]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-DateFormatSymbols-1]]link:{java8-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2Djava.util.Locale%2D[DateFormatSymbols](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#DateFormatSymbols%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-clone-0]]def link:{java8-javadoc}/java/text/DateFormatSymbols.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getAmPmStrings-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getAmPmStrings%2D%2D[getAmPmStrings]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getAmPmStrings%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getEras-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getEras%2D%2D[getEras]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getEras%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getLocalPatternChars-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DateFormatSymbols.html#getLocalPatternChars%2D%2D[getLocalPatternChars]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getLocalPatternChars%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getMonths-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getMonths%2D%2D[getMonths]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getMonths%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getShortMonths-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getShortMonths%2D%2D[getShortMonths]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getShortMonths%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getShortWeekdays-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getShortWeekdays%2D%2D[getShortWeekdays]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getShortWeekdays%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getWeekdays-0]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getWeekdays%2D%2D[getWeekdays]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getWeekdays%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-getZoneStrings-0]]<<painless-api-reference-String,String>>[][] link:{java8-javadoc}/java/text/DateFormatSymbols.html#getZoneStrings%2D%2D[getZoneStrings]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#getZoneStrings%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-hashCode-0]]int link:{java8-javadoc}/java/text/DateFormatSymbols.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setAmPmStrings-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setAmPmStrings%2Djava.lang.String:A%2D[setAmPmStrings](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setAmPmStrings%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setEras-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setEras%2Djava.lang.String:A%2D[setEras](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setEras%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setLocalPatternChars-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setLocalPatternChars%2Djava.lang.String%2D[setLocalPatternChars](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setLocalPatternChars%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setMonths-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setMonths%2Djava.lang.String:A%2D[setMonths](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setMonths%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setShortMonths-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setShortMonths%2Djava.lang.String:A%2D[setShortMonths](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setShortMonths%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setShortWeekdays-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setShortWeekdays%2Djava.lang.String:A%2D[setShortWeekdays](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setShortWeekdays%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setWeekdays-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setWeekdays%2Djava.lang.String:A%2D[setWeekdays](<<painless-api-reference-String,String>>[])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setWeekdays%2Djava.lang.String:A%2D[java 9])
+* ++[[painless-api-reference-DateFormatSymbols-setZoneStrings-1]]void link:{java8-javadoc}/java/text/DateFormatSymbols.html#setZoneStrings%2Djava.lang.String:A%2D[setZoneStrings](<<painless-api-reference-String,String>>[][])++ (link:{java9-javadoc}/java/text/DateFormatSymbols.html#setZoneStrings%2Djava.lang.String:A%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateTimeException.asciidoc b/docs/reference/painless-api-reference/DateTimeException.asciidoc
new file mode 100644
index 00000000000..5a9953c5bd5
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateTimeException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeException]]++DateTimeException++::
+* ++[[painless-api-reference-DateTimeException-DateTimeException-1]]link:{java8-javadoc}/java/time/DateTimeException.html#DateTimeException%2Djava.lang.String%2D[DateTimeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/DateTimeException.html#DateTimeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateTimeFormatter.asciidoc b/docs/reference/painless-api-reference/DateTimeFormatter.asciidoc
new file mode 100644
index 00000000000..ad104dfe5d6
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateTimeFormatter.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeFormatter]]++DateTimeFormatter++::
+** [[painless-api-reference-DateTimeFormatter-BASIC_ISO_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#BASIC_ISO_DATE[BASIC_ISO_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#BASIC_ISO_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE[ISO_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE_TIME[ISO_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_INSTANT]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_INSTANT[ISO_INSTANT] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_INSTANT[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE[ISO_LOCAL_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE_TIME[ISO_LOCAL_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_LOCAL_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_TIME[ISO_LOCAL_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_LOCAL_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE[ISO_OFFSET_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE_TIME[ISO_OFFSET_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_OFFSET_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_TIME[ISO_OFFSET_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_OFFSET_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_ORDINAL_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ORDINAL_DATE[ISO_ORDINAL_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ORDINAL_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_TIME[ISO_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_WEEK_DATE]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_WEEK_DATE[ISO_WEEK_DATE] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_WEEK_DATE[java 9])
+** [[painless-api-reference-DateTimeFormatter-ISO_ZONED_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ZONED_DATE_TIME[ISO_ZONED_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ISO_ZONED_DATE_TIME[java 9])
+** [[painless-api-reference-DateTimeFormatter-RFC_1123_DATE_TIME]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#RFC_1123_DATE_TIME[RFC_1123_DATE_TIME] (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#RFC_1123_DATE_TIME[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDate-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDate%2Djava.time.format.FormatStyle%2D[ofLocalizedDate](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDate%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDateTime-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2D[ofLocalizedDateTime](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedDateTime-2]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[ofLocalizedDateTime](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedDateTime%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofLocalizedTime-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedTime%2Djava.time.format.FormatStyle%2D[ofLocalizedTime](<<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofLocalizedTime%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofPattern-1]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2D[ofPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-ofPattern-2]]static <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2Djava.util.Locale%2D[ofPattern](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#ofPattern%2Djava.lang.String%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parsedExcessDays-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parsedExcessDays%2D%2D[parsedExcessDays]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parsedExcessDays%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parsedLeapSecond-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parsedLeapSecond%2D%2D[parsedLeapSecond]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parsedLeapSecond%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#format%2Djava.time.temporal.TemporalAccessor%2D[format](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#format%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-formatTo-2]]void link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#formatTo%2Djava.time.temporal.TemporalAccessor%2Djava.lang.Appendable%2D[formatTo](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>, <<painless-api-reference-Appendable,Appendable>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#formatTo%2Djava.time.temporal.TemporalAccessor%2Djava.lang.Appendable%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getChronology-0]]<<painless-api-reference-Chronology,Chronology>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getDecimalStyle-0]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getDecimalStyle%2D%2D[getDecimalStyle]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getDecimalStyle%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getLocale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getLocale%2D%2D[getLocale]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getLocale%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getResolverFields-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getResolverFields%2D%2D[getResolverFields]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getResolverFields%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getResolverStyle-0]]<<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getResolverStyle%2D%2D[getResolverStyle]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getResolverStyle%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-getZone-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#getZone%2D%2D[getZone]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#getZone%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parse-1]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parse-2]]def link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parse%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parseBest-2]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parseBest%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery:A%2D[parseBest](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-TemporalQuery,TemporalQuery>>[])++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parseBest%2Djava.lang.CharSequence%2Djava.time.temporal.TemporalQuery:A%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-parseUnresolved-2]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#parseUnresolved%2Djava.lang.CharSequence%2Djava.text.ParsePosition%2D[parseUnresolved](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#parseUnresolved%2Djava.lang.CharSequence%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-toFormat-0]]<<painless-api-reference-Format,Format>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2D%2D[toFormat]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-toFormat-1]]<<painless-api-reference-Format,Format>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2Djava.time.temporal.TemporalQuery%2D[toFormat](<<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#toFormat%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withChronology-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withChronology%2Djava.time.chrono.Chronology%2D[withChronology](<<painless-api-reference-Chronology,Chronology>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withChronology%2Djava.time.chrono.Chronology%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withDecimalStyle-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withDecimalStyle%2Djava.time.format.DecimalStyle%2D[withDecimalStyle](<<painless-api-reference-DecimalStyle,DecimalStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withDecimalStyle%2Djava.time.format.DecimalStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withLocale-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withLocale%2Djava.util.Locale%2D[withLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withResolverFields-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withResolverFields%2Djava.util.Set%2D[withResolverFields](<<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withResolverFields%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withResolverStyle-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withResolverStyle%2Djava.time.format.ResolverStyle%2D[withResolverStyle](<<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withResolverStyle%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatter-withZone-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatter.html#withZone%2Djava.time.ZoneId%2D[withZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatter.html#withZone%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateTimeFormatterBuilder.asciidoc b/docs/reference/painless-api-reference/DateTimeFormatterBuilder.asciidoc
new file mode 100644
index 00000000000..cc352560f41
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateTimeFormatterBuilder.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeFormatterBuilder]]++DateTimeFormatterBuilder++::
+* ++[[painless-api-reference-DateTimeFormatterBuilder-getLocalizedDateTimePattern-4]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#getLocalizedDateTimePattern%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2Djava.time.chrono.Chronology%2Djava.util.Locale%2D[getLocalizedDateTimePattern](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-Chronology,Chronology>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#getLocalizedDateTimePattern%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2Djava.time.chrono.Chronology%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-DateTimeFormatterBuilder-0]]link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#DateTimeFormatterBuilder%2D%2D[DateTimeFormatterBuilder]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#DateTimeFormatterBuilder%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-append-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#append%2Djava.time.format.DateTimeFormatter%2D[append](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#append%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendChronologyId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyId%2D%2D[appendChronologyId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendChronologyText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyText%2Djava.time.format.TextStyle%2D[appendChronologyText](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendChronologyText%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendFraction-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendFraction%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dboolean%2D[appendFraction](<<painless-api-reference-TemporalField,TemporalField>>, int, int, boolean)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendFraction%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendInstant-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2D%2D[appendInstant]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendInstant-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2Dint%2D[appendInstant](int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendInstant%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLiteral-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLiteral%2Djava.lang.String%2D[appendLiteral](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLiteral%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLocalized-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalized%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[appendLocalized](<<painless-api-reference-FormatStyle,FormatStyle>>, <<painless-api-reference-FormatStyle,FormatStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalized%2Djava.time.format.FormatStyle%2Djava.time.format.FormatStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendLocalizedOffset-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalizedOffset%2Djava.time.format.TextStyle%2D[appendLocalizedOffset](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendLocalizedOffset%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOffset-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffset%2Djava.lang.String%2Djava.lang.String%2D[appendOffset](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffset%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOffsetId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffsetId%2D%2D[appendOffsetId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOffsetId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendOptional-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOptional%2Djava.time.format.DateTimeFormatter%2D[appendOptional](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendOptional%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendPattern-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendPattern%2Djava.lang.String%2D[appendPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2D[appendText](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendText-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2Djava.time.format.TextStyle%2D[appendText](<<painless-api-reference-TemporalField,TemporalField>>, <<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendText%2Djava.time.temporal.TemporalField%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>, int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValue-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Djava.time.format.SignStyle%2D[appendValue](<<painless-api-reference-TemporalField,TemporalField>>, int, int, <<painless-api-reference-SignStyle,SignStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValue%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Djava.time.format.SignStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendValueReduced-4]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValueReduced%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dint%2D[appendValueReduced](<<painless-api-reference-TemporalField,TemporalField>>, int, int, int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendValueReduced%2Djava.time.temporal.TemporalField%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneId%2D%2D[appendZoneId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneOrOffsetId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneOrOffsetId%2D%2D[appendZoneOrOffsetId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneOrOffsetId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneRegionId-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneRegionId%2D%2D[appendZoneRegionId]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneRegionId%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneText-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2D[appendZoneText](<<painless-api-reference-TextStyle,TextStyle>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-appendZoneText-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2Djava.util.Set%2D[appendZoneText](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Set,Set>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#appendZoneText%2Djava.time.format.TextStyle%2Djava.util.Set%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-optionalEnd-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalEnd%2D%2D[optionalEnd]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalEnd%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-optionalStart-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalStart%2D%2D[optionalStart]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#optionalStart%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-padNext-1]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2D[padNext](int)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-padNext-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2Dchar%2D[padNext](int, char)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#padNext%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseCaseInsensitive-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseInsensitive%2D%2D[parseCaseInsensitive]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseInsensitive%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseCaseSensitive-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseSensitive%2D%2D[parseCaseSensitive]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseCaseSensitive%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseDefaulting-2]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseDefaulting%2Djava.time.temporal.TemporalField%2Dlong%2D[parseDefaulting](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseDefaulting%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseLenient-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseLenient%2D%2D[parseLenient]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseLenient%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-parseStrict-0]]<<painless-api-reference-DateTimeFormatterBuilder,DateTimeFormatterBuilder>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseStrict%2D%2D[parseStrict]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#parseStrict%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-toFormatter-0]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2D%2D[toFormatter]()++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeFormatterBuilder-toFormatter-1]]<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>> link:{java8-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2Djava.util.Locale%2D[toFormatter](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DateTimeFormatterBuilder.html#toFormatter%2Djava.util.Locale%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DateTimeParseException.asciidoc b/docs/reference/painless-api-reference/DateTimeParseException.asciidoc
new file mode 100644
index 00000000000..8a9626b6405
--- /dev/null
+++ b/docs/reference/painless-api-reference/DateTimeParseException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DateTimeParseException]]++DateTimeParseException++::
+* ++[[painless-api-reference-DateTimeParseException-DateTimeParseException-3]]link:{java8-javadoc}/java/time/format/DateTimeParseException.html#DateTimeParseException%2Djava.lang.String%2Djava.lang.CharSequence%2Dint%2D[DateTimeParseException](<<painless-api-reference-String,String>>, <<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#DateTimeParseException%2Djava.lang.String%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-DateTimeParseException-getErrorIndex-0]]int link:{java8-javadoc}/java/time/format/DateTimeParseException.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#getErrorIndex%2D%2D[java 9])
+* ++[[painless-api-reference-DateTimeParseException-getParsedString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/format/DateTimeParseException.html#getParsedString%2D%2D[getParsedString]()++ (link:{java9-javadoc}/java/time/format/DateTimeParseException.html#getParsedString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DayOfWeek.asciidoc b/docs/reference/painless-api-reference/DayOfWeek.asciidoc
new file mode 100644
index 00000000000..fd847f3d019
--- /dev/null
+++ b/docs/reference/painless-api-reference/DayOfWeek.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DayOfWeek]]++DayOfWeek++::
+** [[painless-api-reference-DayOfWeek-FRIDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#FRIDAY[FRIDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#FRIDAY[java 9])
+** [[painless-api-reference-DayOfWeek-MONDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#MONDAY[MONDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#MONDAY[java 9])
+** [[painless-api-reference-DayOfWeek-SATURDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#SATURDAY[SATURDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#SATURDAY[java 9])
+** [[painless-api-reference-DayOfWeek-SUNDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#SUNDAY[SUNDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#SUNDAY[java 9])
+** [[painless-api-reference-DayOfWeek-THURSDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#THURSDAY[THURSDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#THURSDAY[java 9])
+** [[painless-api-reference-DayOfWeek-TUESDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#TUESDAY[TUESDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#TUESDAY[java 9])
+** [[painless-api-reference-DayOfWeek-WEDNESDAY]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#WEDNESDAY[WEDNESDAY] (link:{java9-javadoc}/java/time/DayOfWeek.html#WEDNESDAY[java 9])
+* ++[[painless-api-reference-DayOfWeek-from-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-of-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-valueOf-1]]static <<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-values-0]]static <<painless-api-reference-DayOfWeek,DayOfWeek>>[] link:{java8-javadoc}/java/time/DayOfWeek.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/DayOfWeek.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/DayOfWeek.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-getValue-0]]int link:{java8-javadoc}/java/time/DayOfWeek.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/DayOfWeek.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-minus-1]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#minus%2Dlong%2D[minus](long)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#minus%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DayOfWeek-plus-1]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/DayOfWeek.html#plus%2Dlong%2D[plus](long)++ (link:{java9-javadoc}/java/time/DayOfWeek.html#plus%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/Debug.asciidoc b/docs/reference/painless-api-reference/Debug.asciidoc
new file mode 100644
index 00000000000..8c64ed6bcc4
--- /dev/null
+++ b/docs/reference/painless-api-reference/Debug.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Debug]]++Debug++::
+* ++[[painless-api-reference-Debug-explain-1]]static void link:{painless-javadoc}/org/elasticsearch/painless/api/Debug.html#explain%2Djava.lang.Object%2D[explain](<<painless-api-reference-Object,Object>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DecimalFormat.asciidoc b/docs/reference/painless-api-reference/DecimalFormat.asciidoc
new file mode 100644
index 00000000000..96768b77273
--- /dev/null
+++ b/docs/reference/painless-api-reference/DecimalFormat.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalFormat]]++DecimalFormat++::
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-0]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2D%2D[DecimalFormat]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-1]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2D[DecimalFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-DecimalFormat-2]]link:{java8-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2Djava.text.DecimalFormatSymbols%2D[DecimalFormat](<<painless-api-reference-String,String>>, <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#DecimalFormat%2Djava.lang.String%2Djava.text.DecimalFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-applyLocalizedPattern-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[applyLocalizedPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getDecimalFormatSymbols-0]]<<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormat.html#getDecimalFormatSymbols%2D%2D[getDecimalFormatSymbols]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getDecimalFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getGroupingSize-0]]int link:{java8-javadoc}/java/text/DecimalFormat.html#getGroupingSize%2D%2D[getGroupingSize]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getGroupingSize%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getMultiplier-0]]int link:{java8-javadoc}/java/text/DecimalFormat.html#getMultiplier%2D%2D[getMultiplier]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getMultiplier%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getNegativePrefix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getNegativePrefix%2D%2D[getNegativePrefix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getNegativePrefix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getNegativeSuffix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getNegativeSuffix%2D%2D[getNegativeSuffix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getNegativeSuffix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getPositivePrefix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getPositivePrefix%2D%2D[getPositivePrefix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getPositivePrefix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-getPositiveSuffix-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#getPositiveSuffix%2D%2D[getPositiveSuffix]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#getPositiveSuffix%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-isDecimalSeparatorAlwaysShown-0]]boolean link:{java8-javadoc}/java/text/DecimalFormat.html#isDecimalSeparatorAlwaysShown%2D%2D[isDecimalSeparatorAlwaysShown]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#isDecimalSeparatorAlwaysShown%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-isParseBigDecimal-0]]boolean link:{java8-javadoc}/java/text/DecimalFormat.html#isParseBigDecimal%2D%2D[isParseBigDecimal]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#isParseBigDecimal%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setDecimalFormatSymbols-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setDecimalFormatSymbols%2Djava.text.DecimalFormatSymbols%2D[setDecimalFormatSymbols](<<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setDecimalFormatSymbols%2Djava.text.DecimalFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setDecimalSeparatorAlwaysShown-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setDecimalSeparatorAlwaysShown%2Dboolean%2D[setDecimalSeparatorAlwaysShown](boolean)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setDecimalSeparatorAlwaysShown%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setGroupingSize-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setGroupingSize%2Dint%2D[setGroupingSize](int)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setGroupingSize%2Dint%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setMultiplier-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setMultiplier%2Dint%2D[setMultiplier](int)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setMultiplier%2Dint%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setNegativePrefix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setNegativePrefix%2Djava.lang.String%2D[setNegativePrefix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setNegativePrefix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setNegativeSuffix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setNegativeSuffix%2Djava.lang.String%2D[setNegativeSuffix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setNegativeSuffix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setParseBigDecimal-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setParseBigDecimal%2Dboolean%2D[setParseBigDecimal](boolean)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setParseBigDecimal%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setPositivePrefix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setPositivePrefix%2Djava.lang.String%2D[setPositivePrefix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setPositivePrefix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-setPositiveSuffix-1]]void link:{java8-javadoc}/java/text/DecimalFormat.html#setPositiveSuffix%2Djava.lang.String%2D[setPositiveSuffix](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormat.html#setPositiveSuffix%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-toLocalizedPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#toLocalizedPattern%2D%2D[toLocalizedPattern]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#toLocalizedPattern%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/DecimalFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-NumberFormat,NumberFormat>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DecimalFormatSymbols.asciidoc b/docs/reference/painless-api-reference/DecimalFormatSymbols.asciidoc
new file mode 100644
index 00000000000..37f6fc090d1
--- /dev/null
+++ b/docs/reference/painless-api-reference/DecimalFormatSymbols.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalFormatSymbols]]++DecimalFormatSymbols++::
+* ++[[painless-api-reference-DecimalFormatSymbols-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInstance-0]]static <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInstance-1]]static <<painless-api-reference-DecimalFormatSymbols,DecimalFormatSymbols>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-DecimalFormatSymbols-0]]link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2D%2D[DecimalFormatSymbols]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-DecimalFormatSymbols-1]]link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2Djava.util.Locale%2D[DecimalFormatSymbols](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#DecimalFormatSymbols%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-clone-0]]def link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getCurrency-0]]<<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getCurrency%2D%2D[getCurrency]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getCurrency%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getCurrencySymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getCurrencySymbol%2D%2D[getCurrencySymbol]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getCurrencySymbol%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getDecimalSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getDecimalSeparator%2D%2D[getDecimalSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getDigit-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getDigit%2D%2D[getDigit]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getExponentSeparator-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getExponentSeparator%2D%2D[getExponentSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getExponentSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getGroupingSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getGroupingSeparator%2D%2D[getGroupingSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getGroupingSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInfinity-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInfinity%2D%2D[getInfinity]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInfinity%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getInternationalCurrencySymbol-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getInternationalCurrencySymbol%2D%2D[getInternationalCurrencySymbol]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getInternationalCurrencySymbol%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getMinusSign-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getMinusSign%2D%2D[getMinusSign]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getMinusSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getMonetaryDecimalSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getMonetaryDecimalSeparator%2D%2D[getMonetaryDecimalSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getMonetaryDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getNaN-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getNaN%2D%2D[getNaN]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getNaN%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPatternSeparator-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPatternSeparator%2D%2D[getPatternSeparator]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPatternSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPerMill-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPerMill%2D%2D[getPerMill]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPerMill%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getPercent-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getPercent%2D%2D[getPercent]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getPercent%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-getZeroDigit-0]]char link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#getZeroDigit%2D%2D[getZeroDigit]()++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#getZeroDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setCurrency-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setCurrency%2Djava.util.Currency%2D[setCurrency](<<painless-api-reference-Currency,Currency>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setCurrency%2Djava.util.Currency%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setCurrencySymbol-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setCurrencySymbol%2Djava.lang.String%2D[setCurrencySymbol](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setCurrencySymbol%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setDecimalSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setDecimalSeparator%2Dchar%2D[setDecimalSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setDigit-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setDigit%2Dchar%2D[setDigit](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setDigit%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setExponentSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setExponentSeparator%2Djava.lang.String%2D[setExponentSeparator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setExponentSeparator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setGroupingSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setGroupingSeparator%2Dchar%2D[setGroupingSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setGroupingSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setInfinity-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setInfinity%2Djava.lang.String%2D[setInfinity](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setInfinity%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setInternationalCurrencySymbol-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setInternationalCurrencySymbol%2Djava.lang.String%2D[setInternationalCurrencySymbol](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setInternationalCurrencySymbol%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setMinusSign-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setMinusSign%2Dchar%2D[setMinusSign](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setMinusSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setMonetaryDecimalSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setMonetaryDecimalSeparator%2Dchar%2D[setMonetaryDecimalSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setMonetaryDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setNaN-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setNaN%2Djava.lang.String%2D[setNaN](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setNaN%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPatternSeparator-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPatternSeparator%2Dchar%2D[setPatternSeparator](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPatternSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPerMill-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPerMill%2Dchar%2D[setPerMill](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPerMill%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setPercent-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setPercent%2Dchar%2D[setPercent](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setPercent%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalFormatSymbols-setZeroDigit-1]]void link:{java8-javadoc}/java/text/DecimalFormatSymbols.html#setZeroDigit%2Dchar%2D[setZeroDigit](char)++ (link:{java9-javadoc}/java/text/DecimalFormatSymbols.html#setZeroDigit%2Dchar%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DecimalStyle.asciidoc b/docs/reference/painless-api-reference/DecimalStyle.asciidoc
new file mode 100644
index 00000000000..8dac3a7ae24
--- /dev/null
+++ b/docs/reference/painless-api-reference/DecimalStyle.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DecimalStyle]]++DecimalStyle++::
+** [[painless-api-reference-DecimalStyle-STANDARD]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#STANDARD[STANDARD] (link:{java9-javadoc}/java/time/format/DecimalStyle.html#STANDARD[java 9])
+* ++[[painless-api-reference-DecimalStyle-getAvailableLocales-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-of-1]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#of%2Djava.util.Locale%2D[of](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#of%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-ofDefaultLocale-0]]static <<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#ofDefaultLocale%2D%2D[ofDefaultLocale]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#ofDefaultLocale%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getDecimalSeparator-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getDecimalSeparator%2D%2D[getDecimalSeparator]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getDecimalSeparator%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getNegativeSign-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getNegativeSign%2D%2D[getNegativeSign]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getNegativeSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getPositiveSign-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getPositiveSign%2D%2D[getPositiveSign]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getPositiveSign%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-getZeroDigit-0]]char link:{java8-javadoc}/java/time/format/DecimalStyle.html#getZeroDigit%2D%2D[getZeroDigit]()++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#getZeroDigit%2D%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withDecimalSeparator-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withDecimalSeparator%2Dchar%2D[withDecimalSeparator](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withDecimalSeparator%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withNegativeSign-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withNegativeSign%2Dchar%2D[withNegativeSign](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withNegativeSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withPositiveSign-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withPositiveSign%2Dchar%2D[withPositiveSign](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withPositiveSign%2Dchar%2D[java 9])
+* ++[[painless-api-reference-DecimalStyle-withZeroDigit-1]]<<painless-api-reference-DecimalStyle,DecimalStyle>> link:{java8-javadoc}/java/time/format/DecimalStyle.html#withZeroDigit%2Dchar%2D[withZeroDigit](char)++ (link:{java9-javadoc}/java/time/format/DecimalStyle.html#withZeroDigit%2Dchar%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Deque.asciidoc b/docs/reference/painless-api-reference/Deque.asciidoc
new file mode 100644
index 00000000000..4dc6b5c06fc
--- /dev/null
+++ b/docs/reference/painless-api-reference/Deque.asciidoc
@@ -0,0 +1,25 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Deque]]++Deque++::
+* ++[[painless-api-reference-Deque-addFirst-1]]void link:{java8-javadoc}/java/util/Deque.html#addFirst%2Djava.lang.Object%2D[addFirst](def)++ (link:{java9-javadoc}/java/util/Deque.html#addFirst%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-addLast-1]]void link:{java8-javadoc}/java/util/Deque.html#addLast%2Djava.lang.Object%2D[addLast](def)++ (link:{java9-javadoc}/java/util/Deque.html#addLast%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-descendingIterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Deque.html#descendingIterator%2D%2D[descendingIterator]()++ (link:{java9-javadoc}/java/util/Deque.html#descendingIterator%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-getFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#getFirst%2D%2D[getFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#getFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-getLast-0]]def link:{java8-javadoc}/java/util/Deque.html#getLast%2D%2D[getLast]()++ (link:{java9-javadoc}/java/util/Deque.html#getLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-offerFirst-1]]boolean link:{java8-javadoc}/java/util/Deque.html#offerFirst%2Djava.lang.Object%2D[offerFirst](def)++ (link:{java9-javadoc}/java/util/Deque.html#offerFirst%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-offerLast-1]]boolean link:{java8-javadoc}/java/util/Deque.html#offerLast%2Djava.lang.Object%2D[offerLast](def)++ (link:{java9-javadoc}/java/util/Deque.html#offerLast%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-peekFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#peekFirst%2D%2D[peekFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#peekFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-peekLast-0]]def link:{java8-javadoc}/java/util/Deque.html#peekLast%2D%2D[peekLast]()++ (link:{java9-javadoc}/java/util/Deque.html#peekLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pollFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#pollFirst%2D%2D[pollFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#pollFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pollLast-0]]def link:{java8-javadoc}/java/util/Deque.html#pollLast%2D%2D[pollLast]()++ (link:{java9-javadoc}/java/util/Deque.html#pollLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-pop-0]]def link:{java8-javadoc}/java/util/Deque.html#pop%2D%2D[pop]()++ (link:{java9-javadoc}/java/util/Deque.html#pop%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-push-1]]void link:{java8-javadoc}/java/util/Deque.html#push%2Djava.lang.Object%2D[push](def)++ (link:{java9-javadoc}/java/util/Deque.html#push%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-remove-1]]boolean link:{java8-javadoc}/java/util/Deque.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Deque.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-removeFirst-0]]def link:{java8-javadoc}/java/util/Deque.html#removeFirst%2D%2D[removeFirst]()++ (link:{java9-javadoc}/java/util/Deque.html#removeFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-removeFirstOccurrence-1]]boolean link:{java8-javadoc}/java/util/Deque.html#removeFirstOccurrence%2Djava.lang.Object%2D[removeFirstOccurrence](def)++ (link:{java9-javadoc}/java/util/Deque.html#removeFirstOccurrence%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Deque-removeLast-0]]def link:{java8-javadoc}/java/util/Deque.html#removeLast%2D%2D[removeLast]()++ (link:{java9-javadoc}/java/util/Deque.html#removeLast%2D%2D[java 9])
+* ++[[painless-api-reference-Deque-removeLastOccurrence-1]]boolean link:{java8-javadoc}/java/util/Deque.html#removeLastOccurrence%2Djava.lang.Object%2D[removeLastOccurrence](def)++ (link:{java9-javadoc}/java/util/Deque.html#removeLastOccurrence%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/reference/painless-api-reference/Dictionary.asciidoc b/docs/reference/painless-api-reference/Dictionary.asciidoc
new file mode 100644
index 00000000000..bfa8f2c495e
--- /dev/null
+++ b/docs/reference/painless-api-reference/Dictionary.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Dictionary]]++Dictionary++::
+* ++[[painless-api-reference-Dictionary-elements-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Dictionary.html#elements%2D%2D[elements]()++ (link:{java9-javadoc}/java/util/Dictionary.html#elements%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-get-1]]def link:{java8-javadoc}/java/util/Dictionary.html#get%2Djava.lang.Object%2D[get](def)++ (link:{java9-javadoc}/java/util/Dictionary.html#get%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Dictionary.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Dictionary.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-keys-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Dictionary.html#keys%2D%2D[keys]()++ (link:{java9-javadoc}/java/util/Dictionary.html#keys%2D%2D[java 9])
+* ++[[painless-api-reference-Dictionary-put-2]]def link:{java8-javadoc}/java/util/Dictionary.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[put](def, def)++ (link:{java9-javadoc}/java/util/Dictionary.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-remove-1]]def link:{java8-javadoc}/java/util/Dictionary.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Dictionary.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Dictionary-size-0]]int link:{java8-javadoc}/java/util/Dictionary.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Dictionary.html#size%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Double.asciidoc b/docs/reference/painless-api-reference/Double.asciidoc
new file mode 100644
index 00000000000..50d1d8c217c
--- /dev/null
+++ b/docs/reference/painless-api-reference/Double.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Double]]++Double++::
+** [[painless-api-reference-Double-BYTES]]static int link:{java8-javadoc}/java/lang/Double.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Double.html#BYTES[java 9])
+** [[painless-api-reference-Double-MAX_EXPONENT]]static int link:{java8-javadoc}/java/lang/Double.html#MAX_EXPONENT[MAX_EXPONENT] (link:{java9-javadoc}/java/lang/Double.html#MAX_EXPONENT[java 9])
+** [[painless-api-reference-Double-MAX_VALUE]]static double link:{java8-javadoc}/java/lang/Double.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Double.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Double-MIN_EXPONENT]]static int link:{java8-javadoc}/java/lang/Double.html#MIN_EXPONENT[MIN_EXPONENT] (link:{java9-javadoc}/java/lang/Double.html#MIN_EXPONENT[java 9])
+** [[painless-api-reference-Double-MIN_NORMAL]]static double link:{java8-javadoc}/java/lang/Double.html#MIN_NORMAL[MIN_NORMAL] (link:{java9-javadoc}/java/lang/Double.html#MIN_NORMAL[java 9])
+** [[painless-api-reference-Double-MIN_VALUE]]static double link:{java8-javadoc}/java/lang/Double.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Double.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Double-NEGATIVE_INFINITY]]static double link:{java8-javadoc}/java/lang/Double.html#NEGATIVE_INFINITY[NEGATIVE_INFINITY] (link:{java9-javadoc}/java/lang/Double.html#NEGATIVE_INFINITY[java 9])
+** [[painless-api-reference-Double-NaN]]static double link:{java8-javadoc}/java/lang/Double.html#NaN[NaN] (link:{java9-javadoc}/java/lang/Double.html#NaN[java 9])
+** [[painless-api-reference-Double-POSITIVE_INFINITY]]static double link:{java8-javadoc}/java/lang/Double.html#POSITIVE_INFINITY[POSITIVE_INFINITY] (link:{java9-javadoc}/java/lang/Double.html#POSITIVE_INFINITY[java 9])
+** [[painless-api-reference-Double-SIZE]]static int link:{java8-javadoc}/java/lang/Double.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Double.html#SIZE[java 9])
+* ++[[painless-api-reference-Double-compare-2]]static int link:{java8-javadoc}/java/lang/Double.html#compare%2Ddouble%2Ddouble%2D[compare](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#compare%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-doubleToLongBits-1]]static long link:{java8-javadoc}/java/lang/Double.html#doubleToLongBits%2Ddouble%2D[doubleToLongBits](double)++ (link:{java9-javadoc}/java/lang/Double.html#doubleToLongBits%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-doubleToRawLongBits-1]]static long link:{java8-javadoc}/java/lang/Double.html#doubleToRawLongBits%2Ddouble%2D[doubleToRawLongBits](double)++ (link:{java9-javadoc}/java/lang/Double.html#doubleToRawLongBits%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-hashCode-1]]static int link:{java8-javadoc}/java/lang/Double.html#hashCode%2Ddouble%2D[hashCode](double)++ (link:{java9-javadoc}/java/lang/Double.html#hashCode%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isFinite-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isFinite%2Ddouble%2D[isFinite](double)++ (link:{java9-javadoc}/java/lang/Double.html#isFinite%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isInfinite-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isInfinite%2Ddouble%2D[isInfinite](double)++ (link:{java9-javadoc}/java/lang/Double.html#isInfinite%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-isNaN-1]]static boolean link:{java8-javadoc}/java/lang/Double.html#isNaN%2Ddouble%2D[isNaN](double)++ (link:{java9-javadoc}/java/lang/Double.html#isNaN%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-longBitsToDouble-1]]static double link:{java8-javadoc}/java/lang/Double.html#longBitsToDouble%2Dlong%2D[longBitsToDouble](long)++ (link:{java9-javadoc}/java/lang/Double.html#longBitsToDouble%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Double-max-2]]static double link:{java8-javadoc}/java/lang/Double.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-min-2]]static double link:{java8-javadoc}/java/lang/Double.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-parseDouble-1]]static double link:{java8-javadoc}/java/lang/Double.html#parseDouble%2Djava.lang.String%2D[parseDouble](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Double.html#parseDouble%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Double-sum-2]]static double link:{java8-javadoc}/java/lang/Double.html#sum%2Ddouble%2Ddouble%2D[sum](double, double)++ (link:{java9-javadoc}/java/lang/Double.html#sum%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Double.html#toHexString%2Ddouble%2D[toHexString](double)++ (link:{java9-javadoc}/java/lang/Double.html#toHexString%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Double.html#toString%2Ddouble%2D[toString](double)++ (link:{java9-javadoc}/java/lang/Double.html#toString%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-valueOf-1]]static <<painless-api-reference-Double,Double>> link:{java8-javadoc}/java/lang/Double.html#valueOf%2Ddouble%2D[valueOf](double)++ (link:{java9-javadoc}/java/lang/Double.html#valueOf%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Double-compareTo-1]]int link:{java8-javadoc}/java/lang/Double.html#compareTo%2Djava.lang.Double%2D[compareTo](<<painless-api-reference-Double,Double>>)++ (link:{java9-javadoc}/java/lang/Double.html#compareTo%2Djava.lang.Double%2D[java 9])
+* ++[[painless-api-reference-Double-isInfinite-0]]boolean link:{java8-javadoc}/java/lang/Double.html#isInfinite%2D%2D[isInfinite]()++ (link:{java9-javadoc}/java/lang/Double.html#isInfinite%2D%2D[java 9])
+* ++[[painless-api-reference-Double-isNaN-0]]boolean link:{java8-javadoc}/java/lang/Double.html#isNaN%2D%2D[isNaN]()++ (link:{java9-javadoc}/java/lang/Double.html#isNaN%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleBinaryOperator.asciidoc b/docs/reference/painless-api-reference/DoubleBinaryOperator.asciidoc
new file mode 100644
index 00000000000..deec8c5337c
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleBinaryOperator]]++DoubleBinaryOperator++::
+* ++[[painless-api-reference-DoubleBinaryOperator-applyAsDouble-2]]double link:{java8-javadoc}/java/util/function/DoubleBinaryOperator.html#applyAsDouble%2Ddouble%2Ddouble%2D[applyAsDouble](double, double)++ (link:{java9-javadoc}/java/util/function/DoubleBinaryOperator.html#applyAsDouble%2Ddouble%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleConsumer.asciidoc b/docs/reference/painless-api-reference/DoubleConsumer.asciidoc
new file mode 100644
index 00000000000..3c078a4d4fb
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleConsumer]]++DoubleConsumer++::
+* ++[[painless-api-reference-DoubleConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/DoubleConsumer.html#accept%2Ddouble%2D[accept](double)++ (link:{java9-javadoc}/java/util/function/DoubleConsumer.html#accept%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleConsumer-andThen-1]]<<painless-api-reference-DoubleConsumer,DoubleConsumer>> link:{java8-javadoc}/java/util/function/DoubleConsumer.html#andThen%2Djava.util.function.DoubleConsumer%2D[andThen](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/function/DoubleConsumer.html#andThen%2Djava.util.function.DoubleConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleFunction.asciidoc b/docs/reference/painless-api-reference/DoubleFunction.asciidoc
new file mode 100644
index 00000000000..082a7000132
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleFunction]]++DoubleFunction++::
+* ++[[painless-api-reference-DoubleFunction-apply-1]]def link:{java8-javadoc}/java/util/function/DoubleFunction.html#apply%2Ddouble%2D[apply](double)++ (link:{java9-javadoc}/java/util/function/DoubleFunction.html#apply%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoublePredicate.asciidoc b/docs/reference/painless-api-reference/DoublePredicate.asciidoc
new file mode 100644
index 00000000000..48cfd3c5929
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoublePredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoublePredicate]]++DoublePredicate++::
+* ++[[painless-api-reference-DoublePredicate-and-1]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#and%2Djava.util.function.DoublePredicate%2D[and](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#and%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-negate-0]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-or-1]]<<painless-api-reference-DoublePredicate,DoublePredicate>> link:{java8-javadoc}/java/util/function/DoublePredicate.html#or%2Djava.util.function.DoublePredicate%2D[or](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#or%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoublePredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/DoublePredicate.html#test%2Ddouble%2D[test](double)++ (link:{java9-javadoc}/java/util/function/DoublePredicate.html#test%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleStream.Builder.asciidoc b/docs/reference/painless-api-reference/DoubleStream.Builder.asciidoc
new file mode 100644
index 00000000000..278a3f99a9a
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleStream-Builder]]++DoubleStream.Builder++::
+* ++[[painless-api-reference-DoubleStream-Builder-add-1]]<<painless-api-reference-DoubleStream-Builder,DoubleStream.Builder>> link:{java8-javadoc}/java/util/stream/DoubleStream$Builder.html#add%2Ddouble%2D[add](double)++ (link:{java9-javadoc}/java/util/stream/DoubleStream$Builder.html#add%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-Builder-build-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream$Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DoubleConsumer,DoubleConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleStream.asciidoc b/docs/reference/painless-api-reference/DoubleStream.asciidoc
new file mode 100644
index 00000000000..d3f517e29e4
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleStream.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleStream]]++DoubleStream++::
+* ++[[painless-api-reference-DoubleStream-builder-0]]static <<painless-api-reference-DoubleStream-Builder,DoubleStream.Builder>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-concat-2]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#concat%2Djava.util.stream.DoubleStream%2Djava.util.stream.DoubleStream%2D[concat](<<painless-api-reference-DoubleStream,DoubleStream>>, <<painless-api-reference-DoubleStream,DoubleStream>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#concat%2Djava.util.stream.DoubleStream%2Djava.util.stream.DoubleStream%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-empty-0]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-of-1]]static <<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#of%2Ddouble:A%2D[of](double[])++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#of%2Ddouble:A%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#allMatch%2Djava.util.function.DoublePredicate%2D[allMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#allMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#anyMatch%2Djava.util.function.DoublePredicate%2D[anyMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#anyMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-collect-3]]def link:{java8-javadoc}/java/util/stream/DoubleStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjDoubleConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjDoubleConsumer,ObjDoubleConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjDoubleConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-count-0]]long link:{java8-javadoc}/java/util/stream/DoubleStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-distinct-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-filter-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#filter%2Djava.util.function.DoublePredicate%2D[filter](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#filter%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-findAny-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-findFirst-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-flatMap-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#flatMap%2Djava.util.function.DoubleFunction%2D[flatMap](<<painless-api-reference-DoubleFunction,DoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#flatMap%2Djava.util.function.DoubleFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/DoubleStream.html#forEach%2Djava.util.function.DoubleConsumer%2D[forEach](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#forEach%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/DoubleStream.html#forEachOrdered%2Djava.util.function.DoubleConsumer%2D[forEachOrdered](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#forEachOrdered%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfDouble,PrimitiveIterator.OfDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-limit-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-map-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#map%2Djava.util.function.DoubleUnaryOperator%2D[map](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#map%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToInt%2Djava.util.function.DoubleToIntFunction%2D[mapToInt](<<painless-api-reference-DoubleToIntFunction,DoubleToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToInt%2Djava.util.function.DoubleToIntFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToLong%2Djava.util.function.DoubleToLongFunction%2D[mapToLong](<<painless-api-reference-DoubleToLongFunction,DoubleToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToLong%2Djava.util.function.DoubleToLongFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#mapToObj%2Djava.util.function.DoubleFunction%2D[mapToObj](<<painless-api-reference-DoubleFunction,DoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#mapToObj%2Djava.util.function.DoubleFunction%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-max-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-min-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/DoubleStream.html#noneMatch%2Djava.util.function.DoublePredicate%2D[noneMatch](<<painless-api-reference-DoublePredicate,DoublePredicate>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#noneMatch%2Djava.util.function.DoublePredicate%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-peek-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#peek%2Djava.util.function.DoubleConsumer%2D[peek](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#peek%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-reduce-1]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#reduce%2Djava.util.function.DoubleBinaryOperator%2D[reduce](<<painless-api-reference-DoubleBinaryOperator,DoubleBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#reduce%2Djava.util.function.DoubleBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-reduce-2]]double link:{java8-javadoc}/java/util/stream/DoubleStream.html#reduce%2Ddouble%2Djava.util.function.DoubleBinaryOperator%2D[reduce](double, <<painless-api-reference-DoubleBinaryOperator,DoubleBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#reduce%2Ddouble%2Djava.util.function.DoubleBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sequential-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-skip-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sorted-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-spliterator-0]]<<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-sum-0]]double link:{java8-javadoc}/java/util/stream/DoubleStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-summaryStatistics-0]]<<painless-api-reference-DoubleSummaryStatistics,DoubleSummaryStatistics>> link:{java8-javadoc}/java/util/stream/DoubleStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleStream-toArray-0]]double[] link:{java8-javadoc}/java/util/stream/DoubleStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/DoubleStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleSummaryStatistics.asciidoc b/docs/reference/painless-api-reference/DoubleSummaryStatistics.asciidoc
new file mode 100644
index 00000000000..976eb984404
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleSummaryStatistics]]++DoubleSummaryStatistics++::
+* ++[[painless-api-reference-DoubleSummaryStatistics-DoubleSummaryStatistics-0]]link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#DoubleSummaryStatistics%2D%2D[DoubleSummaryStatistics]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#DoubleSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#combine%2Djava.util.DoubleSummaryStatistics%2D[combine](<<painless-api-reference-DoubleSummaryStatistics,DoubleSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#combine%2Djava.util.DoubleSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getMax-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getMin-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleSummaryStatistics-getSum-0]]double link:{java8-javadoc}/java/util/DoubleSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/DoubleSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DoubleConsumer,DoubleConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleSupplier.asciidoc b/docs/reference/painless-api-reference/DoubleSupplier.asciidoc
new file mode 100644
index 00000000000..1de478ab50f
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleSupplier]]++DoubleSupplier++::
+* ++[[painless-api-reference-DoubleSupplier-getAsDouble-0]]double link:{java8-javadoc}/java/util/function/DoubleSupplier.html#getAsDouble%2D%2D[getAsDouble]()++ (link:{java9-javadoc}/java/util/function/DoubleSupplier.html#getAsDouble%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleToIntFunction.asciidoc b/docs/reference/painless-api-reference/DoubleToIntFunction.asciidoc
new file mode 100644
index 00000000000..9448a88faf3
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleToIntFunction]]++DoubleToIntFunction++::
+* ++[[painless-api-reference-DoubleToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/DoubleToIntFunction.html#applyAsInt%2Ddouble%2D[applyAsInt](double)++ (link:{java9-javadoc}/java/util/function/DoubleToIntFunction.html#applyAsInt%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleToLongFunction.asciidoc b/docs/reference/painless-api-reference/DoubleToLongFunction.asciidoc
new file mode 100644
index 00000000000..62a3f841b33
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleToLongFunction]]++DoubleToLongFunction++::
+* ++[[painless-api-reference-DoubleToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/DoubleToLongFunction.html#applyAsLong%2Ddouble%2D[applyAsLong](double)++ (link:{java9-javadoc}/java/util/function/DoubleToLongFunction.html#applyAsLong%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DoubleUnaryOperator.asciidoc b/docs/reference/painless-api-reference/DoubleUnaryOperator.asciidoc
new file mode 100644
index 00000000000..a78b8ac5d0b
--- /dev/null
+++ b/docs/reference/painless-api-reference/DoubleUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DoubleUnaryOperator]]++DoubleUnaryOperator++::
+* ++[[painless-api-reference-DoubleUnaryOperator-identity-0]]static <<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-andThen-1]]<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#andThen%2Djava.util.function.DoubleUnaryOperator%2D[andThen](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#andThen%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#applyAsDouble%2Ddouble%2D[applyAsDouble](double)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#applyAsDouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-DoubleUnaryOperator-compose-1]]<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>> link:{java8-javadoc}/java/util/function/DoubleUnaryOperator.html#compose%2Djava.util.function.DoubleUnaryOperator%2D[compose](<<painless-api-reference-DoubleUnaryOperator,DoubleUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/DoubleUnaryOperator.html#compose%2Djava.util.function.DoubleUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/DuplicateFormatFlagsException.asciidoc b/docs/reference/painless-api-reference/DuplicateFormatFlagsException.asciidoc
new file mode 100644
index 00000000000..f26103fdc19
--- /dev/null
+++ b/docs/reference/painless-api-reference/DuplicateFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-DuplicateFormatFlagsException]]++DuplicateFormatFlagsException++::
+* ++[[painless-api-reference-DuplicateFormatFlagsException-DuplicateFormatFlagsException-1]]link:{java8-javadoc}/java/util/DuplicateFormatFlagsException.html#DuplicateFormatFlagsException%2Djava.lang.String%2D[DuplicateFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/DuplicateFormatFlagsException.html#DuplicateFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-DuplicateFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/DuplicateFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/DuplicateFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Duration.asciidoc b/docs/reference/painless-api-reference/Duration.asciidoc
new file mode 100644
index 00000000000..ef9ecf720a0
--- /dev/null
+++ b/docs/reference/painless-api-reference/Duration.asciidoc
@@ -0,0 +1,51 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Duration]]++Duration++::
+** [[painless-api-reference-Duration-ZERO]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ZERO[ZERO] (link:{java9-javadoc}/java/time/Duration.html#ZERO[java 9])
+* ++[[painless-api-reference-Duration-between-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[between](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/Duration.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-Duration-from-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#from%2Djava.time.temporal.TemporalAmount%2D[from](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Duration.html#from%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Duration-of-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#of%2Dlong%2Djava.time.temporal.TemporalUnit%2D[of](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#of%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-ofDays-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofDays%2Dlong%2D[ofDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofHours-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofHours%2Dlong%2D[ofHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofMillis-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofMillis%2Dlong%2D[ofMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofMinutes-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofMinutes%2Dlong%2D[ofMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofNanos-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofNanos%2Dlong%2D[ofNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofSeconds-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2D[ofSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-ofSeconds-2]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2Dlong%2D[ofSeconds](long, long)++ (link:{java9-javadoc}/java/time/Duration.html#ofSeconds%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-parse-1]]static <<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Duration.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Duration-abs-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#abs%2D%2D[abs]()++ (link:{java9-javadoc}/java/time/Duration.html#abs%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-compareTo-1]]int link:{java8-javadoc}/java/time/Duration.html#compareTo%2Djava.time.Duration%2D[compareTo](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#compareTo%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-dividedBy-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#dividedBy%2Dlong%2D[dividedBy](long)++ (link:{java9-javadoc}/java/time/Duration.html#dividedBy%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-getNano-0]]int link:{java8-javadoc}/java/time/Duration.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/Duration.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-getSeconds-0]]long link:{java8-javadoc}/java/time/Duration.html#getSeconds%2D%2D[getSeconds]()++ (link:{java9-javadoc}/java/time/Duration.html#getSeconds%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-isNegative-0]]boolean link:{java8-javadoc}/java/time/Duration.html#isNegative%2D%2D[isNegative]()++ (link:{java9-javadoc}/java/time/Duration.html#isNegative%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-isZero-0]]boolean link:{java8-javadoc}/java/time/Duration.html#isZero%2D%2D[isZero]()++ (link:{java9-javadoc}/java/time/Duration.html#isZero%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-minus-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minus%2Djava.time.Duration%2D[minus](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#minus%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-minus-2]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-minusDays-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusHours-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusMillis-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusMillis%2Dlong%2D[minusMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusMinutes-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-minusSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-multipliedBy-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#multipliedBy%2Dlong%2D[multipliedBy](long)++ (link:{java9-javadoc}/java/time/Duration.html#multipliedBy%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-negated-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/Duration.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-plus-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plus%2Djava.time.Duration%2D[plus](<<painless-api-reference-Duration,Duration>>)++ (link:{java9-javadoc}/java/time/Duration.html#plus%2Djava.time.Duration%2D[java 9])
+* ++[[painless-api-reference-Duration-plus-2]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Duration.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Duration-plusDays-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusHours-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusMillis-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusMillis%2Dlong%2D[plusMillis](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusMinutes-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-plusSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Duration-toDays-0]]long link:{java8-javadoc}/java/time/Duration.html#toDays%2D%2D[toDays]()++ (link:{java9-javadoc}/java/time/Duration.html#toDays%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toHours-0]]long link:{java8-javadoc}/java/time/Duration.html#toHours%2D%2D[toHours]()++ (link:{java9-javadoc}/java/time/Duration.html#toHours%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toMillis-0]]long link:{java8-javadoc}/java/time/Duration.html#toMillis%2D%2D[toMillis]()++ (link:{java9-javadoc}/java/time/Duration.html#toMillis%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toMinutes-0]]long link:{java8-javadoc}/java/time/Duration.html#toMinutes%2D%2D[toMinutes]()++ (link:{java9-javadoc}/java/time/Duration.html#toMinutes%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-toNanos-0]]long link:{java8-javadoc}/java/time/Duration.html#toNanos%2D%2D[toNanos]()++ (link:{java9-javadoc}/java/time/Duration.html#toNanos%2D%2D[java 9])
+* ++[[painless-api-reference-Duration-withNanos-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#withNanos%2Dint%2D[withNanos](int)++ (link:{java9-javadoc}/java/time/Duration.html#withNanos%2Dint%2D[java 9])
+* ++[[painless-api-reference-Duration-withSeconds-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/Duration.html#withSeconds%2Dlong%2D[withSeconds](long)++ (link:{java9-javadoc}/java/time/Duration.html#withSeconds%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/reference/painless-api-reference/EmptyStackException.asciidoc b/docs/reference/painless-api-reference/EmptyStackException.asciidoc
new file mode 100644
index 00000000000..09ca4ca96be
--- /dev/null
+++ b/docs/reference/painless-api-reference/EmptyStackException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EmptyStackException]]++EmptyStackException++::
+* ++[[painless-api-reference-EmptyStackException-EmptyStackException-0]]link:{java8-javadoc}/java/util/EmptyStackException.html#EmptyStackException%2D%2D[EmptyStackException]()++ (link:{java9-javadoc}/java/util/EmptyStackException.html#EmptyStackException%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Enum.asciidoc b/docs/reference/painless-api-reference/Enum.asciidoc
new file mode 100644
index 00000000000..077044dbfb9
--- /dev/null
+++ b/docs/reference/painless-api-reference/Enum.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Enum]]++Enum++::
+* ++[[painless-api-reference-Enum-compareTo-1]]int link:{java8-javadoc}/java/lang/Enum.html#compareTo%2Djava.lang.Enum%2D[compareTo](<<painless-api-reference-Enum,Enum>>)++ (link:{java9-javadoc}/java/lang/Enum.html#compareTo%2Djava.lang.Enum%2D[java 9])
+* ++[[painless-api-reference-Enum-name-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Enum.html#name%2D%2D[name]()++ (link:{java9-javadoc}/java/lang/Enum.html#name%2D%2D[java 9])
+* ++[[painless-api-reference-Enum-ordinal-0]]int link:{java8-javadoc}/java/lang/Enum.html#ordinal%2D%2D[ordinal]()++ (link:{java9-javadoc}/java/lang/Enum.html#ordinal%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/EnumConstantNotPresentException.asciidoc b/docs/reference/painless-api-reference/EnumConstantNotPresentException.asciidoc
new file mode 100644
index 00000000000..66a709379d7
--- /dev/null
+++ b/docs/reference/painless-api-reference/EnumConstantNotPresentException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EnumConstantNotPresentException]]++EnumConstantNotPresentException++::
+* ++[[painless-api-reference-EnumConstantNotPresentException-constantName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/EnumConstantNotPresentException.html#constantName%2D%2D[constantName]()++ (link:{java9-javadoc}/java/lang/EnumConstantNotPresentException.html#constantName%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Enumeration.asciidoc b/docs/reference/painless-api-reference/Enumeration.asciidoc
new file mode 100644
index 00000000000..51849ec27de
--- /dev/null
+++ b/docs/reference/painless-api-reference/Enumeration.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Enumeration]]++Enumeration++::
+* ++[[painless-api-reference-Enumeration-hasMoreElements-0]]boolean link:{java8-javadoc}/java/util/Enumeration.html#hasMoreElements%2D%2D[hasMoreElements]()++ (link:{java9-javadoc}/java/util/Enumeration.html#hasMoreElements%2D%2D[java 9])
+* ++[[painless-api-reference-Enumeration-nextElement-0]]def link:{java8-javadoc}/java/util/Enumeration.html#nextElement%2D%2D[nextElement]()++ (link:{java9-javadoc}/java/util/Enumeration.html#nextElement%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Era.asciidoc b/docs/reference/painless-api-reference/Era.asciidoc
new file mode 100644
index 00000000000..db5ee6aa86b
--- /dev/null
+++ b/docs/reference/painless-api-reference/Era.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Era]]++Era++::
+* ++[[painless-api-reference-Era-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/chrono/Era.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/chrono/Era.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Era-getValue-0]]int link:{java8-javadoc}/java/time/chrono/Era.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/Era.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/EventListener.asciidoc b/docs/reference/painless-api-reference/EventListener.asciidoc
new file mode 100644
index 00000000000..499095ae020
--- /dev/null
+++ b/docs/reference/painless-api-reference/EventListener.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventListener]]++EventListener++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/EventListenerProxy.asciidoc b/docs/reference/painless-api-reference/EventListenerProxy.asciidoc
new file mode 100644
index 00000000000..a3a4958ec71
--- /dev/null
+++ b/docs/reference/painless-api-reference/EventListenerProxy.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventListenerProxy]]++EventListenerProxy++::
+* ++[[painless-api-reference-EventListenerProxy-getListener-0]]<<painless-api-reference-EventListener,EventListener>> link:{java8-javadoc}/java/util/EventListenerProxy.html#getListener%2D%2D[getListener]()++ (link:{java9-javadoc}/java/util/EventListenerProxy.html#getListener%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/EventObject.asciidoc b/docs/reference/painless-api-reference/EventObject.asciidoc
new file mode 100644
index 00000000000..4ac6b14150c
--- /dev/null
+++ b/docs/reference/painless-api-reference/EventObject.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-EventObject]]++EventObject++::
+* ++[[painless-api-reference-EventObject-EventObject-1]]link:{java8-javadoc}/java/util/EventObject.html#EventObject%2Djava.lang.Object%2D[EventObject](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/EventObject.html#EventObject%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-EventObject-getSource-0]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/util/EventObject.html#getSource%2D%2D[getSource]()++ (link:{java9-javadoc}/java/util/EventObject.html#getSource%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Exception.asciidoc b/docs/reference/painless-api-reference/Exception.asciidoc
new file mode 100644
index 00000000000..d6ff7b0a7e5
--- /dev/null
+++ b/docs/reference/painless-api-reference/Exception.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Exception]]++Exception++::
+* ++[[painless-api-reference-Exception-Exception-0]]link:{java8-javadoc}/java/lang/Exception.html#Exception%2D%2D[Exception]()++ (link:{java9-javadoc}/java/lang/Exception.html#Exception%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-Exception-1]]link:{java8-javadoc}/java/lang/Exception.html#Exception%2Djava.lang.String%2D[Exception](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Exception.html#Exception%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Exception-getLocalizedMessage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Exception.html#getLocalizedMessage%2D%2D[getLocalizedMessage]()++ (link:{java9-javadoc}/java/lang/Exception.html#getLocalizedMessage%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-getMessage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Exception.html#getMessage%2D%2D[getMessage]()++ (link:{java9-javadoc}/java/lang/Exception.html#getMessage%2D%2D[java 9])
+* ++[[painless-api-reference-Exception-getStackTrace-0]]<<painless-api-reference-StackTraceElement,StackTraceElement>>[] link:{java8-javadoc}/java/lang/Exception.html#getStackTrace%2D%2D[getStackTrace]()++ (link:{java9-javadoc}/java/lang/Exception.html#getStackTrace%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/FieldPosition.asciidoc b/docs/reference/painless-api-reference/FieldPosition.asciidoc
new file mode 100644
index 00000000000..7a4b5d78407
--- /dev/null
+++ b/docs/reference/painless-api-reference/FieldPosition.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FieldPosition]]++FieldPosition++::
+* ++[[painless-api-reference-FieldPosition-FieldPosition-1]]link:{java8-javadoc}/java/text/FieldPosition.html#FieldPosition%2Dint%2D[FieldPosition](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#FieldPosition%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-FieldPosition-2]]link:{java8-javadoc}/java/text/FieldPosition.html#FieldPosition%2Djava.text.Format$Field%2Dint%2D[FieldPosition](<<painless-api-reference-Format-Field,Format.Field>>, int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#FieldPosition%2Djava.text.Format$Field%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getBeginIndex-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getBeginIndex%2D%2D[getBeginIndex]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getBeginIndex%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getEndIndex-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getEndIndex%2D%2D[getEndIndex]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getEndIndex%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getField-0]]int link:{java8-javadoc}/java/text/FieldPosition.html#getField%2D%2D[getField]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getField%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-getFieldAttribute-0]]<<painless-api-reference-Format-Field,Format.Field>> link:{java8-javadoc}/java/text/FieldPosition.html#getFieldAttribute%2D%2D[getFieldAttribute]()++ (link:{java9-javadoc}/java/text/FieldPosition.html#getFieldAttribute%2D%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-setBeginIndex-1]]void link:{java8-javadoc}/java/text/FieldPosition.html#setBeginIndex%2Dint%2D[setBeginIndex](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#setBeginIndex%2Dint%2D[java 9])
+* ++[[painless-api-reference-FieldPosition-setEndIndex-1]]void link:{java8-javadoc}/java/text/FieldPosition.html#setEndIndex%2Dint%2D[setEndIndex](int)++ (link:{java9-javadoc}/java/text/FieldPosition.html#setEndIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Float.asciidoc b/docs/reference/painless-api-reference/Float.asciidoc
new file mode 100644
index 00000000000..1d20357842c
--- /dev/null
+++ b/docs/reference/painless-api-reference/Float.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Float]]++Float++::
+** [[painless-api-reference-Float-BYTES]]static int link:{java8-javadoc}/java/lang/Float.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Float.html#BYTES[java 9])
+** [[painless-api-reference-Float-MAX_EXPONENT]]static int link:{java8-javadoc}/java/lang/Float.html#MAX_EXPONENT[MAX_EXPONENT] (link:{java9-javadoc}/java/lang/Float.html#MAX_EXPONENT[java 9])
+** [[painless-api-reference-Float-MAX_VALUE]]static float link:{java8-javadoc}/java/lang/Float.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Float.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Float-MIN_EXPONENT]]static int link:{java8-javadoc}/java/lang/Float.html#MIN_EXPONENT[MIN_EXPONENT] (link:{java9-javadoc}/java/lang/Float.html#MIN_EXPONENT[java 9])
+** [[painless-api-reference-Float-MIN_NORMAL]]static float link:{java8-javadoc}/java/lang/Float.html#MIN_NORMAL[MIN_NORMAL] (link:{java9-javadoc}/java/lang/Float.html#MIN_NORMAL[java 9])
+** [[painless-api-reference-Float-MIN_VALUE]]static float link:{java8-javadoc}/java/lang/Float.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Float.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Float-NEGATIVE_INFINITY]]static float link:{java8-javadoc}/java/lang/Float.html#NEGATIVE_INFINITY[NEGATIVE_INFINITY] (link:{java9-javadoc}/java/lang/Float.html#NEGATIVE_INFINITY[java 9])
+** [[painless-api-reference-Float-NaN]]static float link:{java8-javadoc}/java/lang/Float.html#NaN[NaN] (link:{java9-javadoc}/java/lang/Float.html#NaN[java 9])
+** [[painless-api-reference-Float-POSITIVE_INFINITY]]static float link:{java8-javadoc}/java/lang/Float.html#POSITIVE_INFINITY[POSITIVE_INFINITY] (link:{java9-javadoc}/java/lang/Float.html#POSITIVE_INFINITY[java 9])
+** [[painless-api-reference-Float-SIZE]]static int link:{java8-javadoc}/java/lang/Float.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Float.html#SIZE[java 9])
+* ++[[painless-api-reference-Float-compare-2]]static int link:{java8-javadoc}/java/lang/Float.html#compare%2Dfloat%2Dfloat%2D[compare](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#compare%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-floatToIntBits-1]]static int link:{java8-javadoc}/java/lang/Float.html#floatToIntBits%2Dfloat%2D[floatToIntBits](float)++ (link:{java9-javadoc}/java/lang/Float.html#floatToIntBits%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-floatToRawIntBits-1]]static int link:{java8-javadoc}/java/lang/Float.html#floatToRawIntBits%2Dfloat%2D[floatToRawIntBits](float)++ (link:{java9-javadoc}/java/lang/Float.html#floatToRawIntBits%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-hashCode-1]]static int link:{java8-javadoc}/java/lang/Float.html#hashCode%2Dfloat%2D[hashCode](float)++ (link:{java9-javadoc}/java/lang/Float.html#hashCode%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-intBitsToFloat-1]]static float link:{java8-javadoc}/java/lang/Float.html#intBitsToFloat%2Dint%2D[intBitsToFloat](int)++ (link:{java9-javadoc}/java/lang/Float.html#intBitsToFloat%2Dint%2D[java 9])
+* ++[[painless-api-reference-Float-isFinite-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isFinite%2Dfloat%2D[isFinite](float)++ (link:{java9-javadoc}/java/lang/Float.html#isFinite%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-isInfinite-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isInfinite%2Dfloat%2D[isInfinite](float)++ (link:{java9-javadoc}/java/lang/Float.html#isInfinite%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-isNaN-1]]static boolean link:{java8-javadoc}/java/lang/Float.html#isNaN%2Dfloat%2D[isNaN](float)++ (link:{java9-javadoc}/java/lang/Float.html#isNaN%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-max-2]]static float link:{java8-javadoc}/java/lang/Float.html#max%2Dfloat%2Dfloat%2D[max](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#max%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-min-2]]static float link:{java8-javadoc}/java/lang/Float.html#min%2Dfloat%2Dfloat%2D[min](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#min%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-parseFloat-1]]static float link:{java8-javadoc}/java/lang/Float.html#parseFloat%2Djava.lang.String%2D[parseFloat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Float.html#parseFloat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Float-sum-2]]static float link:{java8-javadoc}/java/lang/Float.html#sum%2Dfloat%2Dfloat%2D[sum](float, float)++ (link:{java9-javadoc}/java/lang/Float.html#sum%2Dfloat%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Float.html#toHexString%2Dfloat%2D[toHexString](float)++ (link:{java9-javadoc}/java/lang/Float.html#toHexString%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Float.html#toString%2Dfloat%2D[toString](float)++ (link:{java9-javadoc}/java/lang/Float.html#toString%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-valueOf-1]]static <<painless-api-reference-Float,Float>> link:{java8-javadoc}/java/lang/Float.html#valueOf%2Dfloat%2D[valueOf](float)++ (link:{java9-javadoc}/java/lang/Float.html#valueOf%2Dfloat%2D[java 9])
+* ++[[painless-api-reference-Float-compareTo-1]]int link:{java8-javadoc}/java/lang/Float.html#compareTo%2Djava.lang.Float%2D[compareTo](<<painless-api-reference-Float,Float>>)++ (link:{java9-javadoc}/java/lang/Float.html#compareTo%2Djava.lang.Float%2D[java 9])
+* ++[[painless-api-reference-Float-isInfinite-0]]boolean link:{java8-javadoc}/java/lang/Float.html#isInfinite%2D%2D[isInfinite]()++ (link:{java9-javadoc}/java/lang/Float.html#isInfinite%2D%2D[java 9])
+* ++[[painless-api-reference-Float-isNaN-0]]boolean link:{java8-javadoc}/java/lang/Float.html#isNaN%2D%2D[isNaN]()++ (link:{java9-javadoc}/java/lang/Float.html#isNaN%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Format.Field.asciidoc b/docs/reference/painless-api-reference/Format.Field.asciidoc
new file mode 100644
index 00000000000..3ddf6ec6d2f
--- /dev/null
+++ b/docs/reference/painless-api-reference/Format.Field.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Format-Field]]++Format.Field++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Format.asciidoc b/docs/reference/painless-api-reference/Format.asciidoc
new file mode 100644
index 00000000000..afa42a0598d
--- /dev/null
+++ b/docs/reference/painless-api-reference/Format.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Format]]++Format++::
+* ++[[painless-api-reference-Format-clone-0]]def link:{java8-javadoc}/java/text/Format.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/text/Format.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Format-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2D[format](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Format-format-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2Djava.lang.StringBuffer%2Djava.text.FieldPosition%2D[format](<<painless-api-reference-Object,Object>>, <<painless-api-reference-StringBuffer,StringBuffer>>, <<painless-api-reference-FieldPosition,FieldPosition>>)++ (link:{java9-javadoc}/java/text/Format.html#format%2Djava.lang.Object%2Djava.lang.StringBuffer%2Djava.text.FieldPosition%2D[java 9])
+* ++[[painless-api-reference-Format-formatToCharacterIterator-1]]<<painless-api-reference-AttributedCharacterIterator,AttributedCharacterIterator>> link:{java8-javadoc}/java/text/Format.html#formatToCharacterIterator%2Djava.lang.Object%2D[formatToCharacterIterator](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/text/Format.html#formatToCharacterIterator%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Format-parseObject-1]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2D[parseObject](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Format-parseObject-2]]<<painless-api-reference-Object,Object>> link:{java8-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2Djava.text.ParsePosition%2D[parseObject](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/Format.html#parseObject%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc b/docs/reference/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc
new file mode 100644
index 00000000000..b72d827b196
--- /dev/null
+++ b/docs/reference/painless-api-reference/FormatFlagsConversionMismatchException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatFlagsConversionMismatchException]]++FormatFlagsConversionMismatchException++::
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-FormatFlagsConversionMismatchException-2]]link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#FormatFlagsConversionMismatchException%2Djava.lang.String%2Dchar%2D[FormatFlagsConversionMismatchException](<<painless-api-reference-String,String>>, char)++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#FormatFlagsConversionMismatchException%2Djava.lang.String%2Dchar%2D[java 9])
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-getConversion-0]]char link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getConversion%2D%2D[java 9])
+* ++[[painless-api-reference-FormatFlagsConversionMismatchException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/FormatFlagsConversionMismatchException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/FormatStyle.asciidoc b/docs/reference/painless-api-reference/FormatStyle.asciidoc
new file mode 100644
index 00000000000..4db787c051a
--- /dev/null
+++ b/docs/reference/painless-api-reference/FormatStyle.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatStyle]]++FormatStyle++::
+** [[painless-api-reference-FormatStyle-FULL]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#FULL[FULL] (link:{java9-javadoc}/java/time/format/FormatStyle.html#FULL[java 9])
+** [[painless-api-reference-FormatStyle-LONG]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#LONG[LONG] (link:{java9-javadoc}/java/time/format/FormatStyle.html#LONG[java 9])
+** [[painless-api-reference-FormatStyle-MEDIUM]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#MEDIUM[MEDIUM] (link:{java9-javadoc}/java/time/format/FormatStyle.html#MEDIUM[java 9])
+** [[painless-api-reference-FormatStyle-SHORT]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#SHORT[SHORT] (link:{java9-javadoc}/java/time/format/FormatStyle.html#SHORT[java 9])
+* ++[[painless-api-reference-FormatStyle-valueOf-1]]static <<painless-api-reference-FormatStyle,FormatStyle>> link:{java8-javadoc}/java/time/format/FormatStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/FormatStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-FormatStyle-values-0]]static <<painless-api-reference-FormatStyle,FormatStyle>>[] link:{java8-javadoc}/java/time/format/FormatStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/FormatStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Formattable.asciidoc b/docs/reference/painless-api-reference/Formattable.asciidoc
new file mode 100644
index 00000000000..ab70d1319e9
--- /dev/null
+++ b/docs/reference/painless-api-reference/Formattable.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formattable]]++Formattable++::
+* ++[[painless-api-reference-Formattable-formatTo-4]]void link:{java8-javadoc}/java/util/Formattable.html#formatTo%2Djava.util.Formatter%2Dint%2Dint%2Dint%2D[formatTo](<<painless-api-reference-Formatter,Formatter>>, int, int, int)++ (link:{java9-javadoc}/java/util/Formattable.html#formatTo%2Djava.util.Formatter%2Dint%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/FormattableFlags.asciidoc b/docs/reference/painless-api-reference/FormattableFlags.asciidoc
new file mode 100644
index 00000000000..dec639fe70b
--- /dev/null
+++ b/docs/reference/painless-api-reference/FormattableFlags.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormattableFlags]]++FormattableFlags++::
+** [[painless-api-reference-FormattableFlags-ALTERNATE]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#ALTERNATE[ALTERNATE] (link:{java9-javadoc}/java/util/FormattableFlags.html#ALTERNATE[java 9])
+** [[painless-api-reference-FormattableFlags-LEFT_JUSTIFY]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#LEFT_JUSTIFY[LEFT_JUSTIFY] (link:{java9-javadoc}/java/util/FormattableFlags.html#LEFT_JUSTIFY[java 9])
+** [[painless-api-reference-FormattableFlags-UPPERCASE]]static int link:{java8-javadoc}/java/util/FormattableFlags.html#UPPERCASE[UPPERCASE] (link:{java9-javadoc}/java/util/FormattableFlags.html#UPPERCASE[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc b/docs/reference/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc
new file mode 100644
index 00000000000..92d8852e13c
--- /dev/null
+++ b/docs/reference/painless-api-reference/Formatter.BigDecimalLayoutForm.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formatter-BigDecimalLayoutForm]]++Formatter.BigDecimalLayoutForm++::
+** [[painless-api-reference-Formatter-BigDecimalLayoutForm-DECIMAL_FLOAT]]static <<painless-api-reference-Formatter-BigDecimalLayoutForm,Formatter.BigDecimalLayoutForm>> link:{java8-javadoc}/java/util/Formatter$BigDecimalLayoutForm.html#DECIMAL_FLOAT[DECIMAL_FLOAT] (link:{java9-javadoc}/java/util/Formatter$BigDecimalLayoutForm.html#DECIMAL_FLOAT[java 9])
+** [[painless-api-reference-Formatter-BigDecimalLayoutForm-SCIENTIFIC]]static <<painless-api-reference-Formatter-BigDecimalLayoutForm,Formatter.BigDecimalLayoutForm>> link:{java8-javadoc}/java/util/Formatter$BigDecimalLayoutForm.html#SCIENTIFIC[SCIENTIFIC] (link:{java9-javadoc}/java/util/Formatter$BigDecimalLayoutForm.html#SCIENTIFIC[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Formatter.asciidoc b/docs/reference/painless-api-reference/Formatter.asciidoc
new file mode 100644
index 00000000000..4cad434f130
--- /dev/null
+++ b/docs/reference/painless-api-reference/Formatter.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Formatter]]++Formatter++::
+* ++[[painless-api-reference-Formatter-Formatter-0]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2D%2D[Formatter]()++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2D%2D[java 9])
+* ++[[painless-api-reference-Formatter-Formatter-1]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2D[Formatter](<<painless-api-reference-Appendable,Appendable>>)++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2D[java 9])
+* ++[[painless-api-reference-Formatter-Formatter-2]]link:{java8-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2Djava.util.Locale%2D[Formatter](<<painless-api-reference-Appendable,Appendable>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Formatter.html#Formatter%2Djava.lang.Appendable%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Formatter-format-2]]<<painless-api-reference-Formatter,Formatter>> link:{java8-javadoc}/java/util/Formatter.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/util/Formatter.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Formatter-format-3]]<<painless-api-reference-Formatter,Formatter>> link:{java8-javadoc}/java/util/Formatter.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-Locale,Locale>>, <<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/util/Formatter.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Formatter-locale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Formatter.html#locale%2D%2D[locale]()++ (link:{java9-javadoc}/java/util/Formatter.html#locale%2D%2D[java 9])
+* ++[[painless-api-reference-Formatter-out-0]]<<painless-api-reference-Appendable,Appendable>> link:{java8-javadoc}/java/util/Formatter.html#out%2D%2D[out]()++ (link:{java9-javadoc}/java/util/Formatter.html#out%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/FormatterClosedException.asciidoc b/docs/reference/painless-api-reference/FormatterClosedException.asciidoc
new file mode 100644
index 00000000000..00c4f5971d3
--- /dev/null
+++ b/docs/reference/painless-api-reference/FormatterClosedException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-FormatterClosedException]]++FormatterClosedException++::
+* ++[[painless-api-reference-FormatterClosedException-FormatterClosedException-0]]link:{java8-javadoc}/java/util/FormatterClosedException.html#FormatterClosedException%2D%2D[FormatterClosedException]()++ (link:{java9-javadoc}/java/util/FormatterClosedException.html#FormatterClosedException%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Function.asciidoc b/docs/reference/painless-api-reference/Function.asciidoc
new file mode 100644
index 00000000000..61fe3b6b5da
--- /dev/null
+++ b/docs/reference/painless-api-reference/Function.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Function]]++Function++::
+* ++[[painless-api-reference-Function-identity-0]]static <<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/Function.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-Function-andThen-1]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#andThen%2Djava.util.function.Function%2D[andThen](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/Function.html#andThen%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Function-apply-1]]def link:{java8-javadoc}/java/util/function/Function.html#apply%2Djava.lang.Object%2D[apply](def)++ (link:{java9-javadoc}/java/util/function/Function.html#apply%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Function-compose-1]]<<painless-api-reference-Function,Function>> link:{java8-javadoc}/java/util/function/Function.html#compose%2Djava.util.function.Function%2D[compose](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/function/Function.html#compose%2Djava.util.function.Function%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/GregorianCalendar.asciidoc b/docs/reference/painless-api-reference/GregorianCalendar.asciidoc
new file mode 100644
index 00000000000..8f7b9ccecc7
--- /dev/null
+++ b/docs/reference/painless-api-reference/GregorianCalendar.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-GregorianCalendar]]++GregorianCalendar++::
+** [[painless-api-reference-GregorianCalendar-AD]]static int link:{java8-javadoc}/java/util/GregorianCalendar.html#AD[AD] (link:{java9-javadoc}/java/util/GregorianCalendar.html#AD[java 9])
+** [[painless-api-reference-GregorianCalendar-BC]]static int link:{java8-javadoc}/java/util/GregorianCalendar.html#BC[BC] (link:{java9-javadoc}/java/util/GregorianCalendar.html#BC[java 9])
+* ++[[painless-api-reference-GregorianCalendar-from-1]]static <<painless-api-reference-GregorianCalendar,GregorianCalendar>> link:{java8-javadoc}/java/util/GregorianCalendar.html#from%2Djava.time.ZonedDateTime%2D[from](<<painless-api-reference-ZonedDateTime,ZonedDateTime>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#from%2Djava.time.ZonedDateTime%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-0]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2D%2D[GregorianCalendar]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2D%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-1]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2D[GregorianCalendar](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-2]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2Djava.util.Locale%2D[GregorianCalendar](<<painless-api-reference-TimeZone,TimeZone>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Djava.util.TimeZone%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-3]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-5]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-GregorianCalendar-6]]link:{java8-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[GregorianCalendar](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#GregorianCalendar%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-getGregorianChange-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/util/GregorianCalendar.html#getGregorianChange%2D%2D[getGregorianChange]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#getGregorianChange%2D%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-isLeapYear-1]]boolean link:{java8-javadoc}/java/util/GregorianCalendar.html#isLeapYear%2Dint%2D[isLeapYear](int)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#isLeapYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-setGregorianChange-1]]void link:{java8-javadoc}/java/util/GregorianCalendar.html#setGregorianChange%2Djava.util.Date%2D[setGregorianChange](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#setGregorianChange%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-GregorianCalendar-toZonedDateTime-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/util/GregorianCalendar.html#toZonedDateTime%2D%2D[toZonedDateTime]()++ (link:{java9-javadoc}/java/util/GregorianCalendar.html#toZonedDateTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Calendar,Calendar>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/HashMap.asciidoc b/docs/reference/painless-api-reference/HashMap.asciidoc
new file mode 100644
index 00000000000..f67e75d702f
--- /dev/null
+++ b/docs/reference/painless-api-reference/HashMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HashMap]]++HashMap++::
+* ++[[painless-api-reference-HashMap-HashMap-0]]link:{java8-javadoc}/java/util/HashMap.html#HashMap%2D%2D[HashMap]()++ (link:{java9-javadoc}/java/util/HashMap.html#HashMap%2D%2D[java 9])
+* ++[[painless-api-reference-HashMap-HashMap-1]]link:{java8-javadoc}/java/util/HashMap.html#HashMap%2Djava.util.Map%2D[HashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/HashMap.html#HashMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-HashMap-clone-0]]def link:{java8-javadoc}/java/util/HashMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/HashMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/HashSet.asciidoc b/docs/reference/painless-api-reference/HashSet.asciidoc
new file mode 100644
index 00000000000..af4171def22
--- /dev/null
+++ b/docs/reference/painless-api-reference/HashSet.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HashSet]]++HashSet++::
+* ++[[painless-api-reference-HashSet-HashSet-0]]link:{java8-javadoc}/java/util/HashSet.html#HashSet%2D%2D[HashSet]()++ (link:{java9-javadoc}/java/util/HashSet.html#HashSet%2D%2D[java 9])
+* ++[[painless-api-reference-HashSet-HashSet-1]]link:{java8-javadoc}/java/util/HashSet.html#HashSet%2Djava.util.Collection%2D[HashSet](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/HashSet.html#HashSet%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-HashSet-clone-0]]def link:{java8-javadoc}/java/util/HashSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/HashSet.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/reference/painless-api-reference/Hashtable.asciidoc b/docs/reference/painless-api-reference/Hashtable.asciidoc
new file mode 100644
index 00000000000..34702e84660
--- /dev/null
+++ b/docs/reference/painless-api-reference/Hashtable.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Hashtable]]++Hashtable++::
+* ++[[painless-api-reference-Hashtable-Hashtable-0]]link:{java8-javadoc}/java/util/Hashtable.html#Hashtable%2D%2D[Hashtable]()++ (link:{java9-javadoc}/java/util/Hashtable.html#Hashtable%2D%2D[java 9])
+* ++[[painless-api-reference-Hashtable-Hashtable-1]]link:{java8-javadoc}/java/util/Hashtable.html#Hashtable%2Djava.util.Map%2D[Hashtable](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Hashtable.html#Hashtable%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Hashtable-clone-0]]def link:{java8-javadoc}/java/util/Hashtable.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Hashtable.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Dictionary,Dictionary>>++, ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/HijrahChronology.asciidoc b/docs/reference/painless-api-reference/HijrahChronology.asciidoc
new file mode 100644
index 00000000000..b7577fbdfd7
--- /dev/null
+++ b/docs/reference/painless-api-reference/HijrahChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahChronology]]++HijrahChronology++::
+** [[painless-api-reference-HijrahChronology-INSTANCE]]static <<painless-api-reference-HijrahChronology,HijrahChronology>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-3]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-date-4]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateEpochDay-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateYearDay-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-dateYearDay-3]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-eraOf-1]]<<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahChronology-resolveDate-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/HijrahDate.asciidoc b/docs/reference/painless-api-reference/HijrahDate.asciidoc
new file mode 100644
index 00000000000..09ebf31682a
--- /dev/null
+++ b/docs/reference/painless-api-reference/HijrahDate.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahDate]]++HijrahDate++::
+* ++[[painless-api-reference-HijrahDate-from-1]]static <<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-of-3]]static <<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-getChronology-0]]<<painless-api-reference-HijrahChronology,HijrahChronology>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-getEra-0]]<<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-minus-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-minus-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-plus-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-plus-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-with-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-with-2]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-HijrahDate-withVariant-1]]<<painless-api-reference-HijrahDate,HijrahDate>> link:{java8-javadoc}/java/time/chrono/HijrahDate.html#withVariant%2Djava.time.chrono.HijrahChronology%2D[withVariant](<<painless-api-reference-HijrahChronology,HijrahChronology>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahDate.html#withVariant%2Djava.time.chrono.HijrahChronology%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/HijrahEra.asciidoc b/docs/reference/painless-api-reference/HijrahEra.asciidoc
new file mode 100644
index 00000000000..9bfe561d2fa
--- /dev/null
+++ b/docs/reference/painless-api-reference/HijrahEra.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-HijrahEra]]++HijrahEra++::
+** [[painless-api-reference-HijrahEra-AH]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#AH[AH] (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#AH[java 9])
+* ++[[painless-api-reference-HijrahEra-of-1]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-valueOf-1]]static <<painless-api-reference-HijrahEra,HijrahEra>> link:{java8-javadoc}/java/time/chrono/HijrahEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-values-0]]static <<painless-api-reference-HijrahEra,HijrahEra>>[] link:{java8-javadoc}/java/time/chrono/HijrahEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-HijrahEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/HijrahEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/HijrahEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/IdentityHashMap.asciidoc b/docs/reference/painless-api-reference/IdentityHashMap.asciidoc
new file mode 100644
index 00000000000..050d0eaeb1c
--- /dev/null
+++ b/docs/reference/painless-api-reference/IdentityHashMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IdentityHashMap]]++IdentityHashMap++::
+* ++[[painless-api-reference-IdentityHashMap-IdentityHashMap-0]]link:{java8-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2D%2D[IdentityHashMap]()++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2D%2D[java 9])
+* ++[[painless-api-reference-IdentityHashMap-IdentityHashMap-1]]link:{java8-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2Djava.util.Map%2D[IdentityHashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#IdentityHashMap%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-IdentityHashMap-clone-0]]def link:{java8-javadoc}/java/util/IdentityHashMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/IdentityHashMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalAccessException.asciidoc b/docs/reference/painless-api-reference/IllegalAccessException.asciidoc
new file mode 100644
index 00000000000..30c8ba4536d
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalAccessException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalAccessException]]++IllegalAccessException++::
+* ++[[painless-api-reference-IllegalAccessException-IllegalAccessException-0]]link:{java8-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2D%2D[IllegalAccessException]()++ (link:{java9-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalAccessException-IllegalAccessException-1]]link:{java8-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2Djava.lang.String%2D[IllegalAccessException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalAccessException.html#IllegalAccessException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalArgumentException.asciidoc b/docs/reference/painless-api-reference/IllegalArgumentException.asciidoc
new file mode 100644
index 00000000000..538fa46a656
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalArgumentException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalArgumentException]]++IllegalArgumentException++::
+* ++[[painless-api-reference-IllegalArgumentException-IllegalArgumentException-0]]link:{java8-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2D%2D[IllegalArgumentException]()++ (link:{java9-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalArgumentException-IllegalArgumentException-1]]link:{java8-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2Djava.lang.String%2D[IllegalArgumentException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalArgumentException.html#IllegalArgumentException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatCodePointException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatCodePointException.asciidoc
new file mode 100644
index 00000000000..2a533384198
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatCodePointException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatCodePointException]]++IllegalFormatCodePointException++::
+* ++[[painless-api-reference-IllegalFormatCodePointException-IllegalFormatCodePointException-1]]link:{java8-javadoc}/java/util/IllegalFormatCodePointException.html#IllegalFormatCodePointException%2Dint%2D[IllegalFormatCodePointException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatCodePointException.html#IllegalFormatCodePointException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatCodePointException-getCodePoint-0]]int link:{java8-javadoc}/java/util/IllegalFormatCodePointException.html#getCodePoint%2D%2D[getCodePoint]()++ (link:{java9-javadoc}/java/util/IllegalFormatCodePointException.html#getCodePoint%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatConversionException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatConversionException.asciidoc
new file mode 100644
index 00000000000..2a36b5367a3
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatConversionException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatConversionException]]++IllegalFormatConversionException++::
+* ++[[painless-api-reference-IllegalFormatConversionException-getConversion-0]]char link:{java8-javadoc}/java/util/IllegalFormatConversionException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/IllegalFormatConversionException.html#getConversion%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatException.asciidoc
new file mode 100644
index 00000000000..98a58ceddd2
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatException.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatException]]++IllegalFormatException++::
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatFlagsException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatFlagsException.asciidoc
new file mode 100644
index 00000000000..9ed7810f0e6
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatFlagsException]]++IllegalFormatFlagsException++::
+* ++[[painless-api-reference-IllegalFormatFlagsException-IllegalFormatFlagsException-1]]link:{java8-javadoc}/java/util/IllegalFormatFlagsException.html#IllegalFormatFlagsException%2Djava.lang.String%2D[IllegalFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/IllegalFormatFlagsException.html#IllegalFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/IllegalFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/IllegalFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatPrecisionException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatPrecisionException.asciidoc
new file mode 100644
index 00000000000..318b80dcfaa
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatPrecisionException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatPrecisionException]]++IllegalFormatPrecisionException++::
+* ++[[painless-api-reference-IllegalFormatPrecisionException-IllegalFormatPrecisionException-1]]link:{java8-javadoc}/java/util/IllegalFormatPrecisionException.html#IllegalFormatPrecisionException%2Dint%2D[IllegalFormatPrecisionException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatPrecisionException.html#IllegalFormatPrecisionException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatPrecisionException-getPrecision-0]]int link:{java8-javadoc}/java/util/IllegalFormatPrecisionException.html#getPrecision%2D%2D[getPrecision]()++ (link:{java9-javadoc}/java/util/IllegalFormatPrecisionException.html#getPrecision%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalFormatWidthException.asciidoc b/docs/reference/painless-api-reference/IllegalFormatWidthException.asciidoc
new file mode 100644
index 00000000000..96f008b4a53
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalFormatWidthException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalFormatWidthException]]++IllegalFormatWidthException++::
+* ++[[painless-api-reference-IllegalFormatWidthException-IllegalFormatWidthException-1]]link:{java8-javadoc}/java/util/IllegalFormatWidthException.html#IllegalFormatWidthException%2Dint%2D[IllegalFormatWidthException](int)++ (link:{java9-javadoc}/java/util/IllegalFormatWidthException.html#IllegalFormatWidthException%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllegalFormatWidthException-getWidth-0]]int link:{java8-javadoc}/java/util/IllegalFormatWidthException.html#getWidth%2D%2D[getWidth]()++ (link:{java9-javadoc}/java/util/IllegalFormatWidthException.html#getWidth%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalMonitorStateException.asciidoc b/docs/reference/painless-api-reference/IllegalMonitorStateException.asciidoc
new file mode 100644
index 00000000000..057c58f948b
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalMonitorStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalMonitorStateException]]++IllegalMonitorStateException++::
+* ++[[painless-api-reference-IllegalMonitorStateException-IllegalMonitorStateException-0]]link:{java8-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2D%2D[IllegalMonitorStateException]()++ (link:{java9-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalMonitorStateException-IllegalMonitorStateException-1]]link:{java8-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2Djava.lang.String%2D[IllegalMonitorStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalMonitorStateException.html#IllegalMonitorStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalStateException.asciidoc b/docs/reference/painless-api-reference/IllegalStateException.asciidoc
new file mode 100644
index 00000000000..fa8ddf72fa5
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalStateException]]++IllegalStateException++::
+* ++[[painless-api-reference-IllegalStateException-IllegalStateException-0]]link:{java8-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2D%2D[IllegalStateException]()++ (link:{java9-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalStateException-IllegalStateException-1]]link:{java8-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2Djava.lang.String%2D[IllegalStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalStateException.html#IllegalStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllegalThreadStateException.asciidoc b/docs/reference/painless-api-reference/IllegalThreadStateException.asciidoc
new file mode 100644
index 00000000000..5fc8293fe09
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllegalThreadStateException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllegalThreadStateException]]++IllegalThreadStateException++::
+* ++[[painless-api-reference-IllegalThreadStateException-IllegalThreadStateException-0]]link:{java8-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2D%2D[IllegalThreadStateException]()++ (link:{java9-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2D%2D[java 9])
+* ++[[painless-api-reference-IllegalThreadStateException-IllegalThreadStateException-1]]link:{java8-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2Djava.lang.String%2D[IllegalThreadStateException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IllegalThreadStateException.html#IllegalThreadStateException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IllformedLocaleException.asciidoc b/docs/reference/painless-api-reference/IllformedLocaleException.asciidoc
new file mode 100644
index 00000000000..59a814d9520
--- /dev/null
+++ b/docs/reference/painless-api-reference/IllformedLocaleException.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IllformedLocaleException]]++IllformedLocaleException++::
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-0]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2D%2D[IllformedLocaleException]()++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2D%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-1]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2D[IllformedLocaleException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-IllformedLocaleException-2]]link:{java8-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2Dint%2D[IllformedLocaleException](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#IllformedLocaleException%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-IllformedLocaleException-getErrorIndex-0]]int link:{java8-javadoc}/java/util/IllformedLocaleException.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/util/IllformedLocaleException.html#getErrorIndex%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IndexOutOfBoundsException.asciidoc b/docs/reference/painless-api-reference/IndexOutOfBoundsException.asciidoc
new file mode 100644
index 00000000000..a141ec6b041
--- /dev/null
+++ b/docs/reference/painless-api-reference/IndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IndexOutOfBoundsException]]++IndexOutOfBoundsException++::
+* ++[[painless-api-reference-IndexOutOfBoundsException-IndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2D%2D[IndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-IndexOutOfBoundsException-IndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2Djava.lang.String%2D[IndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/IndexOutOfBoundsException.html#IndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/InputMismatchException.asciidoc b/docs/reference/painless-api-reference/InputMismatchException.asciidoc
new file mode 100644
index 00000000000..9497cf9486e
--- /dev/null
+++ b/docs/reference/painless-api-reference/InputMismatchException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InputMismatchException]]++InputMismatchException++::
+* ++[[painless-api-reference-InputMismatchException-InputMismatchException-0]]link:{java8-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2D%2D[InputMismatchException]()++ (link:{java9-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2D%2D[java 9])
+* ++[[painless-api-reference-InputMismatchException-InputMismatchException-1]]link:{java8-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2Djava.lang.String%2D[InputMismatchException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/InputMismatchException.html#InputMismatchException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Instant.asciidoc b/docs/reference/painless-api-reference/Instant.asciidoc
new file mode 100644
index 00000000000..4a7ffe45d99
--- /dev/null
+++ b/docs/reference/painless-api-reference/Instant.asciidoc
@@ -0,0 +1,36 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Instant]]++Instant++::
+** [[painless-api-reference-Instant-EPOCH]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#EPOCH[EPOCH] (link:{java9-javadoc}/java/time/Instant.html#EPOCH[java 9])
+** [[painless-api-reference-Instant-MAX]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#MAX[MAX] (link:{java9-javadoc}/java/time/Instant.html#MAX[java 9])
+** [[painless-api-reference-Instant-MIN]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#MIN[MIN] (link:{java9-javadoc}/java/time/Instant.html#MIN[java 9])
+* ++[[painless-api-reference-Instant-from-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Instant.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochMilli-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochMilli%2Dlong%2D[ofEpochMilli](long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochMilli%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochSecond-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2D[ofEpochSecond](long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-ofEpochSecond-2]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2Dlong%2D[ofEpochSecond](long, long)++ (link:{java9-javadoc}/java/time/Instant.html#ofEpochSecond%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-parse-1]]static <<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Instant.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Instant-atOffset-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/Instant.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/Instant.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-Instant-atZone-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/Instant.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/Instant.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-Instant-compareTo-1]]int link:{java8-javadoc}/java/time/Instant.html#compareTo%2Djava.time.Instant%2D[compareTo](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#compareTo%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-getEpochSecond-0]]long link:{java8-javadoc}/java/time/Instant.html#getEpochSecond%2D%2D[getEpochSecond]()++ (link:{java9-javadoc}/java/time/Instant.html#getEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-getNano-0]]int link:{java8-javadoc}/java/time/Instant.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/Instant.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-isAfter-1]]boolean link:{java8-javadoc}/java/time/Instant.html#isAfter%2Djava.time.Instant%2D[isAfter](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#isAfter%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-isBefore-1]]boolean link:{java8-javadoc}/java/time/Instant.html#isBefore%2Djava.time.Instant%2D[isBefore](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/Instant.html#isBefore%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-Instant-minus-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Instant.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Instant-minus-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-minusMillis-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusMillis%2Dlong%2D[minusMillis](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-minusNanos-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-minusSeconds-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/Instant.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plus-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Instant.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Instant-plus-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-plusMillis-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusMillis%2Dlong%2D[plusMillis](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusMillis%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plusNanos-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-plusSeconds-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/Instant.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Instant-toEpochMilli-0]]long link:{java8-javadoc}/java/time/Instant.html#toEpochMilli%2D%2D[toEpochMilli]()++ (link:{java9-javadoc}/java/time/Instant.html#toEpochMilli%2D%2D[java 9])
+* ++[[painless-api-reference-Instant-truncatedTo-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Instant.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Instant-with-1]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Instant-with-2]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/Instant.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/InstantiationException.asciidoc b/docs/reference/painless-api-reference/InstantiationException.asciidoc
new file mode 100644
index 00000000000..06d88eb654f
--- /dev/null
+++ b/docs/reference/painless-api-reference/InstantiationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InstantiationException]]++InstantiationException++::
+* ++[[painless-api-reference-InstantiationException-InstantiationException-0]]link:{java8-javadoc}/java/lang/InstantiationException.html#InstantiationException%2D%2D[InstantiationException]()++ (link:{java9-javadoc}/java/lang/InstantiationException.html#InstantiationException%2D%2D[java 9])
+* ++[[painless-api-reference-InstantiationException-InstantiationException-1]]link:{java8-javadoc}/java/lang/InstantiationException.html#InstantiationException%2Djava.lang.String%2D[InstantiationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/InstantiationException.html#InstantiationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntBinaryOperator.asciidoc b/docs/reference/painless-api-reference/IntBinaryOperator.asciidoc
new file mode 100644
index 00000000000..ca3c64ab3d8
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntBinaryOperator]]++IntBinaryOperator++::
+* ++[[painless-api-reference-IntBinaryOperator-applyAsInt-2]]int link:{java8-javadoc}/java/util/function/IntBinaryOperator.html#applyAsInt%2Dint%2Dint%2D[applyAsInt](int, int)++ (link:{java9-javadoc}/java/util/function/IntBinaryOperator.html#applyAsInt%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntConsumer.asciidoc b/docs/reference/painless-api-reference/IntConsumer.asciidoc
new file mode 100644
index 00000000000..e8325838dac
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntConsumer]]++IntConsumer++::
+* ++[[painless-api-reference-IntConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/IntConsumer.html#accept%2Dint%2D[accept](int)++ (link:{java9-javadoc}/java/util/function/IntConsumer.html#accept%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntConsumer-andThen-1]]<<painless-api-reference-IntConsumer,IntConsumer>> link:{java8-javadoc}/java/util/function/IntConsumer.html#andThen%2Djava.util.function.IntConsumer%2D[andThen](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/function/IntConsumer.html#andThen%2Djava.util.function.IntConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntFunction.asciidoc b/docs/reference/painless-api-reference/IntFunction.asciidoc
new file mode 100644
index 00000000000..336c1f7c3cb
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntFunction]]++IntFunction++::
+* ++[[painless-api-reference-IntFunction-apply-1]]def link:{java8-javadoc}/java/util/function/IntFunction.html#apply%2Dint%2D[apply](int)++ (link:{java9-javadoc}/java/util/function/IntFunction.html#apply%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntPredicate.asciidoc b/docs/reference/painless-api-reference/IntPredicate.asciidoc
new file mode 100644
index 00000000000..a9c45c95af8
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntPredicate]]++IntPredicate++::
+* ++[[painless-api-reference-IntPredicate-and-1]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#and%2Djava.util.function.IntPredicate%2D[and](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#and%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-negate-0]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-or-1]]<<painless-api-reference-IntPredicate,IntPredicate>> link:{java8-javadoc}/java/util/function/IntPredicate.html#or%2Djava.util.function.IntPredicate%2D[or](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#or%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntPredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/IntPredicate.html#test%2Dint%2D[test](int)++ (link:{java9-javadoc}/java/util/function/IntPredicate.html#test%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntStream.Builder.asciidoc b/docs/reference/painless-api-reference/IntStream.Builder.asciidoc
new file mode 100644
index 00000000000..d9991388173
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntStream-Builder]]++IntStream.Builder++::
+* ++[[painless-api-reference-IntStream-Builder-add-1]]<<painless-api-reference-IntStream-Builder,IntStream.Builder>> link:{java8-javadoc}/java/util/stream/IntStream$Builder.html#add%2Dint%2D[add](int)++ (link:{java9-javadoc}/java/util/stream/IntStream$Builder.html#add%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-Builder-build-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/IntStream$Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-IntConsumer,IntConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntStream.asciidoc b/docs/reference/painless-api-reference/IntStream.asciidoc
new file mode 100644
index 00000000000..10e11492689
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntStream.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntStream]]++IntStream++::
+* ++[[painless-api-reference-IntStream-builder-0]]static <<painless-api-reference-IntStream-Builder,IntStream.Builder>> link:{java8-javadoc}/java/util/stream/IntStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-concat-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#concat%2Djava.util.stream.IntStream%2Djava.util.stream.IntStream%2D[concat](<<painless-api-reference-IntStream,IntStream>>, <<painless-api-reference-IntStream,IntStream>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#concat%2Djava.util.stream.IntStream%2Djava.util.stream.IntStream%2D[java 9])
+* ++[[painless-api-reference-IntStream-empty-0]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-of-1]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#of%2Dint:A%2D[of](int[])++ (link:{java9-javadoc}/java/util/stream/IntStream.html#of%2Dint:A%2D[java 9])
+* ++[[painless-api-reference-IntStream-range-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#range%2Dint%2Dint%2D[range](int, int)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#range%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-rangeClosed-2]]static <<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#rangeClosed%2Dint%2Dint%2D[rangeClosed](int, int)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#rangeClosed%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#allMatch%2Djava.util.function.IntPredicate%2D[allMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#allMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#anyMatch%2Djava.util.function.IntPredicate%2D[anyMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#anyMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-asDoubleStream-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#asDoubleStream%2D%2D[asDoubleStream]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#asDoubleStream%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-asLongStream-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#asLongStream%2D%2D[asLongStream]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#asLongStream%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/IntStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/IntStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-collect-3]]def link:{java8-javadoc}/java/util/stream/IntStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjIntConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjIntConsumer,ObjIntConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjIntConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-count-0]]long link:{java8-javadoc}/java/util/stream/IntStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-distinct-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-filter-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#filter%2Djava.util.function.IntPredicate%2D[filter](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#filter%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-findAny-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-findFirst-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-flatMap-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#flatMap%2Djava.util.function.IntFunction%2D[flatMap](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#flatMap%2Djava.util.function.IntFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/IntStream.html#forEach%2Djava.util.function.IntConsumer%2D[forEach](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#forEach%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/IntStream.html#forEachOrdered%2Djava.util.function.IntConsumer%2D[forEachOrdered](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#forEachOrdered%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfInt,PrimitiveIterator.OfInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-limit-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IntStream-map-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#map%2Djava.util.function.IntUnaryOperator%2D[map](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#map%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToDouble%2Djava.util.function.IntToDoubleFunction%2D[mapToDouble](<<painless-api-reference-IntToDoubleFunction,IntToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToDouble%2Djava.util.function.IntToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToLong%2Djava.util.function.IntToLongFunction%2D[mapToLong](<<painless-api-reference-IntToLongFunction,IntToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToLong%2Djava.util.function.IntToLongFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/IntStream.html#mapToObj%2Djava.util.function.IntFunction%2D[mapToObj](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#mapToObj%2Djava.util.function.IntFunction%2D[java 9])
+* ++[[painless-api-reference-IntStream-max-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-min-0]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/IntStream.html#noneMatch%2Djava.util.function.IntPredicate%2D[noneMatch](<<painless-api-reference-IntPredicate,IntPredicate>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#noneMatch%2Djava.util.function.IntPredicate%2D[java 9])
+* ++[[painless-api-reference-IntStream-peek-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#peek%2Djava.util.function.IntConsumer%2D[peek](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#peek%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-IntStream-reduce-1]]<<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#reduce%2Djava.util.function.IntBinaryOperator%2D[reduce](<<painless-api-reference-IntBinaryOperator,IntBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#reduce%2Djava.util.function.IntBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-reduce-2]]int link:{java8-javadoc}/java/util/stream/IntStream.html#reduce%2Dint%2Djava.util.function.IntBinaryOperator%2D[reduce](int, <<painless-api-reference-IntBinaryOperator,IntBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#reduce%2Dint%2Djava.util.function.IntBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntStream-sequential-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-skip-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/IntStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IntStream-sorted-0]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/IntStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-spliterator-0]]<<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/stream/IntStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-sum-0]]int link:{java8-javadoc}/java/util/stream/IntStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-summaryStatistics-0]]<<painless-api-reference-IntSummaryStatistics,IntSummaryStatistics>> link:{java8-javadoc}/java/util/stream/IntStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-IntStream-toArray-0]]int[] link:{java8-javadoc}/java/util/stream/IntStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/IntStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntSummaryStatistics.asciidoc b/docs/reference/painless-api-reference/IntSummaryStatistics.asciidoc
new file mode 100644
index 00000000000..e13d47ad7c4
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntSummaryStatistics]]++IntSummaryStatistics++::
+* ++[[painless-api-reference-IntSummaryStatistics-IntSummaryStatistics-0]]link:{java8-javadoc}/java/util/IntSummaryStatistics.html#IntSummaryStatistics%2D%2D[IntSummaryStatistics]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#IntSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/IntSummaryStatistics.html#combine%2Djava.util.IntSummaryStatistics%2D[combine](<<painless-api-reference-IntSummaryStatistics,IntSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#combine%2Djava.util.IntSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getMax-0]]int link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getMin-0]]int link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-IntSummaryStatistics-getSum-0]]long link:{java8-javadoc}/java/util/IntSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/IntSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-IntConsumer,IntConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntSupplier.asciidoc b/docs/reference/painless-api-reference/IntSupplier.asciidoc
new file mode 100644
index 00000000000..bca332f7e6c
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntSupplier]]++IntSupplier++::
+* ++[[painless-api-reference-IntSupplier-getAsInt-0]]int link:{java8-javadoc}/java/util/function/IntSupplier.html#getAsInt%2D%2D[getAsInt]()++ (link:{java9-javadoc}/java/util/function/IntSupplier.html#getAsInt%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntToDoubleFunction.asciidoc b/docs/reference/painless-api-reference/IntToDoubleFunction.asciidoc
new file mode 100644
index 00000000000..1cfadec16b2
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntToDoubleFunction]]++IntToDoubleFunction++::
+* ++[[painless-api-reference-IntToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/IntToDoubleFunction.html#applyAsDouble%2Dint%2D[applyAsDouble](int)++ (link:{java9-javadoc}/java/util/function/IntToDoubleFunction.html#applyAsDouble%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntToLongFunction.asciidoc b/docs/reference/painless-api-reference/IntToLongFunction.asciidoc
new file mode 100644
index 00000000000..3cffe6cddea
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntToLongFunction]]++IntToLongFunction++::
+* ++[[painless-api-reference-IntToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/IntToLongFunction.html#applyAsLong%2Dint%2D[applyAsLong](int)++ (link:{java9-javadoc}/java/util/function/IntToLongFunction.html#applyAsLong%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IntUnaryOperator.asciidoc b/docs/reference/painless-api-reference/IntUnaryOperator.asciidoc
new file mode 100644
index 00000000000..a1bb1651fee
--- /dev/null
+++ b/docs/reference/painless-api-reference/IntUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IntUnaryOperator]]++IntUnaryOperator++::
+* ++[[painless-api-reference-IntUnaryOperator-identity-0]]static <<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-andThen-1]]<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#andThen%2Djava.util.function.IntUnaryOperator%2D[andThen](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#andThen%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#applyAsInt%2Dint%2D[applyAsInt](int)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#applyAsInt%2Dint%2D[java 9])
+* ++[[painless-api-reference-IntUnaryOperator-compose-1]]<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>> link:{java8-javadoc}/java/util/function/IntUnaryOperator.html#compose%2Djava.util.function.IntUnaryOperator%2D[compose](<<painless-api-reference-IntUnaryOperator,IntUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/IntUnaryOperator.html#compose%2Djava.util.function.IntUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Integer.asciidoc b/docs/reference/painless-api-reference/Integer.asciidoc
new file mode 100644
index 00000000000..2f608bcc662
--- /dev/null
+++ b/docs/reference/painless-api-reference/Integer.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Integer]]++Integer++::
+** [[painless-api-reference-Integer-BYTES]]static int link:{java8-javadoc}/java/lang/Integer.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Integer.html#BYTES[java 9])
+** [[painless-api-reference-Integer-MAX_VALUE]]static int link:{java8-javadoc}/java/lang/Integer.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Integer.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Integer-MIN_VALUE]]static int link:{java8-javadoc}/java/lang/Integer.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Integer.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Integer-SIZE]]static int link:{java8-javadoc}/java/lang/Integer.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Integer.html#SIZE[java 9])
+* ++[[painless-api-reference-Integer-bitCount-1]]static int link:{java8-javadoc}/java/lang/Integer.html#bitCount%2Dint%2D[bitCount](int)++ (link:{java9-javadoc}/java/lang/Integer.html#bitCount%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compare-2]]static int link:{java8-javadoc}/java/lang/Integer.html#compare%2Dint%2Dint%2D[compare](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#compare%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compareUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#compareUnsigned%2Dint%2Dint%2D[compareUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#compareUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-decode-1]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-divideUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#divideUnsigned%2Dint%2Dint%2D[divideUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#divideUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-hashCode-1]]static int link:{java8-javadoc}/java/lang/Integer.html#hashCode%2Dint%2D[hashCode](int)++ (link:{java9-javadoc}/java/lang/Integer.html#hashCode%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-highestOneBit-1]]static int link:{java8-javadoc}/java/lang/Integer.html#highestOneBit%2Dint%2D[highestOneBit](int)++ (link:{java9-javadoc}/java/lang/Integer.html#highestOneBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-lowestOneBit-1]]static int link:{java8-javadoc}/java/lang/Integer.html#lowestOneBit%2Dint%2D[lowestOneBit](int)++ (link:{java9-javadoc}/java/lang/Integer.html#lowestOneBit%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-max-2]]static int link:{java8-javadoc}/java/lang/Integer.html#max%2Dint%2Dint%2D[max](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#max%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-min-2]]static int link:{java8-javadoc}/java/lang/Integer.html#min%2Dint%2Dint%2D[min](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#min%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-numberOfLeadingZeros-1]]static int link:{java8-javadoc}/java/lang/Integer.html#numberOfLeadingZeros%2Dint%2D[numberOfLeadingZeros](int)++ (link:{java9-javadoc}/java/lang/Integer.html#numberOfLeadingZeros%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-numberOfTrailingZeros-1]]static int link:{java8-javadoc}/java/lang/Integer.html#numberOfTrailingZeros%2Dint%2D[numberOfTrailingZeros](int)++ (link:{java9-javadoc}/java/lang/Integer.html#numberOfTrailingZeros%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-parseInt-1]]static int link:{java8-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2D[parseInt](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-parseInt-2]]static int link:{java8-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2Dint%2D[parseInt](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#parseInt%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-parseUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2D[parseUnsignedInt](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Integer-parseUnsignedInt-2]]static int link:{java8-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2Dint%2D[parseUnsignedInt](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#parseUnsignedInt%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-remainderUnsigned-2]]static int link:{java8-javadoc}/java/lang/Integer.html#remainderUnsigned%2Dint%2Dint%2D[remainderUnsigned](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#remainderUnsigned%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-reverse-1]]static int link:{java8-javadoc}/java/lang/Integer.html#reverse%2Dint%2D[reverse](int)++ (link:{java9-javadoc}/java/lang/Integer.html#reverse%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-reverseBytes-1]]static int link:{java8-javadoc}/java/lang/Integer.html#reverseBytes%2Dint%2D[reverseBytes](int)++ (link:{java9-javadoc}/java/lang/Integer.html#reverseBytes%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-rotateLeft-2]]static int link:{java8-javadoc}/java/lang/Integer.html#rotateLeft%2Dint%2Dint%2D[rotateLeft](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#rotateLeft%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-rotateRight-2]]static int link:{java8-javadoc}/java/lang/Integer.html#rotateRight%2Dint%2Dint%2D[rotateRight](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#rotateRight%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-signum-1]]static int link:{java8-javadoc}/java/lang/Integer.html#signum%2Dint%2D[signum](int)++ (link:{java9-javadoc}/java/lang/Integer.html#signum%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toBinaryString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toBinaryString%2Dint%2D[toBinaryString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toBinaryString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toHexString%2Dint%2D[toHexString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toHexString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toOctalString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toOctalString%2Dint%2D[toOctalString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toOctalString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toString%2Dint%2D[toString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toString%2Dint%2Dint%2D[toString](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#toString%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Integer.html#toUnsignedLong%2Dint%2D[toUnsignedLong](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedLong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2D[toUnsignedString](int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-toUnsignedString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2Dint%2D[toUnsignedString](int, int)++ (link:{java9-javadoc}/java/lang/Integer.html#toUnsignedString%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-valueOf-1]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#valueOf%2Dint%2D[valueOf](int)++ (link:{java9-javadoc}/java/lang/Integer.html#valueOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-valueOf-2]]static <<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/lang/Integer.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Integer.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Integer-compareTo-1]]int link:{java8-javadoc}/java/lang/Integer.html#compareTo%2Djava.lang.Integer%2D[compareTo](<<painless-api-reference-Integer,Integer>>)++ (link:{java9-javadoc}/java/lang/Integer.html#compareTo%2Djava.lang.Integer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/InterruptedException.asciidoc b/docs/reference/painless-api-reference/InterruptedException.asciidoc
new file mode 100644
index 00000000000..cd0ef457f76
--- /dev/null
+++ b/docs/reference/painless-api-reference/InterruptedException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-InterruptedException]]++InterruptedException++::
+* ++[[painless-api-reference-InterruptedException-InterruptedException-0]]link:{java8-javadoc}/java/lang/InterruptedException.html#InterruptedException%2D%2D[InterruptedException]()++ (link:{java9-javadoc}/java/lang/InterruptedException.html#InterruptedException%2D%2D[java 9])
+* ++[[painless-api-reference-InterruptedException-InterruptedException-1]]link:{java8-javadoc}/java/lang/InterruptedException.html#InterruptedException%2Djava.lang.String%2D[InterruptedException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/InterruptedException.html#InterruptedException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/IsoChronology.asciidoc b/docs/reference/painless-api-reference/IsoChronology.asciidoc
new file mode 100644
index 00000000000..3990839b041
--- /dev/null
+++ b/docs/reference/painless-api-reference/IsoChronology.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoChronology]]++IsoChronology++::
+** [[painless-api-reference-IsoChronology-INSTANCE]]static <<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-IsoChronology-date-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-date-3]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-date-4]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateEpochDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateYearDay-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-dateYearDay-3]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-eraOf-1]]<<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-localDateTime-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[localDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#localDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-period-3]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#period%2Dint%2Dint%2Dint%2D[period](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#period%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-resolveDate-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-zonedDateTime-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[zonedDateTime](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-IsoChronology-zonedDateTime-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[zonedDateTime](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/chrono/IsoChronology.html#zonedDateTime%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/IsoEra.asciidoc b/docs/reference/painless-api-reference/IsoEra.asciidoc
new file mode 100644
index 00000000000..975d7d1153a
--- /dev/null
+++ b/docs/reference/painless-api-reference/IsoEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoEra]]++IsoEra++::
+** [[painless-api-reference-IsoEra-BCE]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#BCE[BCE] (link:{java9-javadoc}/java/time/chrono/IsoEra.html#BCE[java 9])
+** [[painless-api-reference-IsoEra-CE]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#CE[CE] (link:{java9-javadoc}/java/time/chrono/IsoEra.html#CE[java 9])
+* ++[[painless-api-reference-IsoEra-of-1]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-IsoEra-valueOf-1]]static <<painless-api-reference-IsoEra,IsoEra>> link:{java8-javadoc}/java/time/chrono/IsoEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-IsoEra-values-0]]static <<painless-api-reference-IsoEra,IsoEra>>[] link:{java8-javadoc}/java/time/chrono/IsoEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-IsoEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/IsoEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/IsoEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/IsoFields.asciidoc b/docs/reference/painless-api-reference/IsoFields.asciidoc
new file mode 100644
index 00000000000..db3f5bbefbb
--- /dev/null
+++ b/docs/reference/painless-api-reference/IsoFields.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-IsoFields]]++IsoFields++::
+** [[painless-api-reference-IsoFields-DAY_OF_QUARTER]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#DAY_OF_QUARTER[DAY_OF_QUARTER] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#DAY_OF_QUARTER[java 9])
+** [[painless-api-reference-IsoFields-QUARTER_OF_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#QUARTER_OF_YEAR[QUARTER_OF_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#QUARTER_OF_YEAR[java 9])
+** [[painless-api-reference-IsoFields-QUARTER_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#QUARTER_YEARS[QUARTER_YEARS] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#QUARTER_YEARS[java 9])
+** [[painless-api-reference-IsoFields-WEEK_BASED_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEAR[WEEK_BASED_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEAR[java 9])
+** [[painless-api-reference-IsoFields-WEEK_BASED_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEARS[WEEK_BASED_YEARS] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_BASED_YEARS[java 9])
+** [[painless-api-reference-IsoFields-WEEK_OF_WEEK_BASED_YEAR]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/IsoFields.html#WEEK_OF_WEEK_BASED_YEAR[WEEK_OF_WEEK_BASED_YEAR] (link:{java9-javadoc}/java/time/temporal/IsoFields.html#WEEK_OF_WEEK_BASED_YEAR[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Iterable.asciidoc b/docs/reference/painless-api-reference/Iterable.asciidoc
new file mode 100644
index 00000000000..b7d626767c7
--- /dev/null
+++ b/docs/reference/painless-api-reference/Iterable.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Iterable]]++Iterable++::
+* ++[[painless-api-reference-Iterable-any-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#any%2Djava.lang.Iterable%2Djava.util.function.Predicate%2D[any](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Iterable-asCollection-0]]<<painless-api-reference-Collection,Collection>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#asCollection%2Djava.lang.Iterable%2D[asCollection]()++
+* ++[[painless-api-reference-Iterable-asList-0]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#asList%2Djava.lang.Iterable%2D[asList]()++
+* ++[[painless-api-reference-Iterable-each-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#each%2Djava.lang.Iterable%2Djava.util.function.Consumer%2D[each](<<painless-api-reference-Consumer,Consumer>>)++
+* ++[[painless-api-reference-Iterable-eachWithIndex-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#eachWithIndex%2Djava.lang.Iterable%2Djava.util.function.ObjIntConsumer%2D[eachWithIndex](<<painless-api-reference-ObjIntConsumer,ObjIntConsumer>>)++
+* ++[[painless-api-reference-Iterable-every-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#every%2Djava.lang.Iterable%2Djava.util.function.Predicate%2D[every](<<painless-api-reference-Predicate,Predicate>>)++
+* ++[[painless-api-reference-Iterable-findResults-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResults%2Djava.lang.Iterable%2Djava.util.function.Function%2D[findResults](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Iterable-forEach-1]]void link:{java8-javadoc}/java/lang/Iterable.html#forEach%2Djava.util.function.Consumer%2D[forEach](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/lang/Iterable.html#forEach%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Iterable-groupBy-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#groupBy%2Djava.lang.Iterable%2Djava.util.function.Function%2D[groupBy](<<painless-api-reference-Function,Function>>)++
+* ++[[painless-api-reference-Iterable-iterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/lang/Iterable.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/lang/Iterable.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-Iterable-join-1]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#join%2Djava.lang.Iterable%2Djava.lang.String%2D[join](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-Iterable-spliterator-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/lang/Iterable.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/lang/Iterable.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Iterable-sum-0]]double link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#sum%2Djava.lang.Iterable%2D[sum]()++
+* ++[[painless-api-reference-Iterable-sum-1]]double link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#sum%2Djava.lang.Iterable%2Djava.util.function.ToDoubleFunction%2D[sum](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Iterator.asciidoc b/docs/reference/painless-api-reference/Iterator.asciidoc
new file mode 100644
index 00000000000..9dd3f5601d5
--- /dev/null
+++ b/docs/reference/painless-api-reference/Iterator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Iterator]]++Iterator++::
+* ++[[painless-api-reference-Iterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Iterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[forEachRemaining](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Iterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Iterator-hasNext-0]]boolean link:{java8-javadoc}/java/util/Iterator.html#hasNext%2D%2D[hasNext]()++ (link:{java9-javadoc}/java/util/Iterator.html#hasNext%2D%2D[java 9])
+* ++[[painless-api-reference-Iterator-next-0]]def link:{java8-javadoc}/java/util/Iterator.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/Iterator.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-Iterator-remove-0]]void link:{java8-javadoc}/java/util/Iterator.html#remove%2D%2D[remove]()++ (link:{java9-javadoc}/java/util/Iterator.html#remove%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/JapaneseChronology.asciidoc b/docs/reference/painless-api-reference/JapaneseChronology.asciidoc
new file mode 100644
index 00000000000..166f3ca176d
--- /dev/null
+++ b/docs/reference/painless-api-reference/JapaneseChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseChronology]]++JapaneseChronology++::
+** [[painless-api-reference-JapaneseChronology-INSTANCE]]static <<painless-api-reference-JapaneseChronology,JapaneseChronology>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-3]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-date-4]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateEpochDay-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateYearDay-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-dateYearDay-3]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-eraOf-1]]<<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseChronology-resolveDate-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/JapaneseDate.asciidoc b/docs/reference/painless-api-reference/JapaneseDate.asciidoc
new file mode 100644
index 00000000000..560ebc7a664
--- /dev/null
+++ b/docs/reference/painless-api-reference/JapaneseDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseDate]]++JapaneseDate++::
+* ++[[painless-api-reference-JapaneseDate-from-1]]static <<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-of-3]]static <<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-getChronology-0]]<<painless-api-reference-JapaneseChronology,JapaneseChronology>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-getEra-0]]<<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-minus-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-minus-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-plus-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-plus-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-with-1]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-JapaneseDate-with-2]]<<painless-api-reference-JapaneseDate,JapaneseDate>> link:{java8-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/JapaneseDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/JapaneseEra.asciidoc b/docs/reference/painless-api-reference/JapaneseEra.asciidoc
new file mode 100644
index 00000000000..b5a9c7f63bc
--- /dev/null
+++ b/docs/reference/painless-api-reference/JapaneseEra.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JapaneseEra]]++JapaneseEra++::
+** [[painless-api-reference-JapaneseEra-HEISEI]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#HEISEI[HEISEI] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#HEISEI[java 9])
+** [[painless-api-reference-JapaneseEra-MEIJI]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#MEIJI[MEIJI] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#MEIJI[java 9])
+** [[painless-api-reference-JapaneseEra-SHOWA]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#SHOWA[SHOWA] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#SHOWA[java 9])
+** [[painless-api-reference-JapaneseEra-TAISHO]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#TAISHO[TAISHO] (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#TAISHO[java 9])
+* ++[[painless-api-reference-JapaneseEra-of-1]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-valueOf-1]]static <<painless-api-reference-JapaneseEra,JapaneseEra>> link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-values-0]]static <<painless-api-reference-JapaneseEra,JapaneseEra>>[] link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-JapaneseEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/JapaneseEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/JapaneseEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/JulianFields.asciidoc b/docs/reference/painless-api-reference/JulianFields.asciidoc
new file mode 100644
index 00000000000..19ee6c5e2eb
--- /dev/null
+++ b/docs/reference/painless-api-reference/JulianFields.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-JulianFields]]++JulianFields++::
+** [[painless-api-reference-JulianFields-JULIAN_DAY]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#JULIAN_DAY[JULIAN_DAY] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#JULIAN_DAY[java 9])
+** [[painless-api-reference-JulianFields-MODIFIED_JULIAN_DAY]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#MODIFIED_JULIAN_DAY[MODIFIED_JULIAN_DAY] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#MODIFIED_JULIAN_DAY[java 9])
+** [[painless-api-reference-JulianFields-RATA_DIE]]static <<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/JulianFields.html#RATA_DIE[RATA_DIE] (link:{java9-javadoc}/java/time/temporal/JulianFields.html#RATA_DIE[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LinkedHashMap.asciidoc b/docs/reference/painless-api-reference/LinkedHashMap.asciidoc
new file mode 100644
index 00000000000..bda2d45f22c
--- /dev/null
+++ b/docs/reference/painless-api-reference/LinkedHashMap.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedHashMap]]++LinkedHashMap++::
+* ++[[painless-api-reference-LinkedHashMap-LinkedHashMap-0]]link:{java8-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2D%2D[LinkedHashMap]()++ (link:{java9-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedHashMap-LinkedHashMap-1]]link:{java8-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2Djava.util.Map%2D[LinkedHashMap](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/LinkedHashMap.html#LinkedHashMap%2Djava.util.Map%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-HashMap,HashMap>>++, ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LinkedHashSet.asciidoc b/docs/reference/painless-api-reference/LinkedHashSet.asciidoc
new file mode 100644
index 00000000000..3a7710771e8
--- /dev/null
+++ b/docs/reference/painless-api-reference/LinkedHashSet.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedHashSet]]++LinkedHashSet++::
+* ++[[painless-api-reference-LinkedHashSet-LinkedHashSet-0]]link:{java8-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2D%2D[LinkedHashSet]()++ (link:{java9-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedHashSet-LinkedHashSet-1]]link:{java8-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2Djava.util.Collection%2D[LinkedHashSet](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/LinkedHashSet.html#LinkedHashSet%2Djava.util.Collection%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-HashSet,HashSet>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/reference/painless-api-reference/LinkedList.asciidoc b/docs/reference/painless-api-reference/LinkedList.asciidoc
new file mode 100644
index 00000000000..5d18b373ba3
--- /dev/null
+++ b/docs/reference/painless-api-reference/LinkedList.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LinkedList]]++LinkedList++::
+* ++[[painless-api-reference-LinkedList-LinkedList-0]]link:{java8-javadoc}/java/util/LinkedList.html#LinkedList%2D%2D[LinkedList]()++ (link:{java9-javadoc}/java/util/LinkedList.html#LinkedList%2D%2D[java 9])
+* ++[[painless-api-reference-LinkedList-LinkedList-1]]link:{java8-javadoc}/java/util/LinkedList.html#LinkedList%2Djava.util.Collection%2D[LinkedList](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/LinkedList.html#LinkedList%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-LinkedList-clone-0]]def link:{java8-javadoc}/java/util/LinkedList.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/LinkedList.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Deque,Deque>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/reference/painless-api-reference/List.asciidoc b/docs/reference/painless-api-reference/List.asciidoc
new file mode 100644
index 00000000000..abe438451b8
--- /dev/null
+++ b/docs/reference/painless-api-reference/List.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-List]]++List++::
+* ++[[painless-api-reference-List-add-2]]void link:{java8-javadoc}/java/util/List.html#add%2Dint%2Djava.lang.Object%2D[add](int, def)++ (link:{java9-javadoc}/java/util/List.html#add%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-addAll-2]]boolean link:{java8-javadoc}/java/util/List.html#addAll%2Dint%2Djava.util.Collection%2D[addAll](int, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/List.html#addAll%2Dint%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-List-equals-1]]boolean link:{java8-javadoc}/java/util/List.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/List.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-get-1]]def link:{java8-javadoc}/java/util/List.html#get%2Dint%2D[get](int)++ (link:{java9-javadoc}/java/util/List.html#get%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-getLength-0]]int link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#getLength%2Djava.util.List%2D[getLength]()++
+* ++[[painless-api-reference-List-hashCode-0]]int link:{java8-javadoc}/java/util/List.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/List.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-List-indexOf-1]]int link:{java8-javadoc}/java/util/List.html#indexOf%2Djava.lang.Object%2D[indexOf](def)++ (link:{java9-javadoc}/java/util/List.html#indexOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-lastIndexOf-1]]int link:{java8-javadoc}/java/util/List.html#lastIndexOf%2Djava.lang.Object%2D[lastIndexOf](def)++ (link:{java9-javadoc}/java/util/List.html#lastIndexOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-listIterator-0]]<<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/List.html#listIterator%2D%2D[listIterator]()++ (link:{java9-javadoc}/java/util/List.html#listIterator%2D%2D[java 9])
+* ++[[painless-api-reference-List-listIterator-1]]<<painless-api-reference-ListIterator,ListIterator>> link:{java8-javadoc}/java/util/List.html#listIterator%2Dint%2D[listIterator](int)++ (link:{java9-javadoc}/java/util/List.html#listIterator%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-remove-1]]def link:{java8-javadoc}/java/util/List.html#remove%2Dint%2D[remove](int)++ (link:{java9-javadoc}/java/util/List.html#remove%2Dint%2D[java 9])
+* ++[[painless-api-reference-List-replaceAll-1]]void link:{java8-javadoc}/java/util/List.html#replaceAll%2Djava.util.function.UnaryOperator%2D[replaceAll](<<painless-api-reference-UnaryOperator,UnaryOperator>>)++ (link:{java9-javadoc}/java/util/List.html#replaceAll%2Djava.util.function.UnaryOperator%2D[java 9])
+* ++[[painless-api-reference-List-set-2]]def link:{java8-javadoc}/java/util/List.html#set%2Dint%2Djava.lang.Object%2D[set](int, def)++ (link:{java9-javadoc}/java/util/List.html#set%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-List-sort-1]]void link:{java8-javadoc}/java/util/List.html#sort%2Djava.util.Comparator%2D[sort](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/List.html#sort%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-List-subList-2]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/List.html#subList%2Dint%2Dint%2D[subList](int, int)++ (link:{java9-javadoc}/java/util/List.html#subList%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ListIterator.asciidoc b/docs/reference/painless-api-reference/ListIterator.asciidoc
new file mode 100644
index 00000000000..37d04b2a755
--- /dev/null
+++ b/docs/reference/painless-api-reference/ListIterator.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ListIterator]]++ListIterator++::
+* ++[[painless-api-reference-ListIterator-add-1]]void link:{java8-javadoc}/java/util/ListIterator.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/ListIterator.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-ListIterator-hasPrevious-0]]boolean link:{java8-javadoc}/java/util/ListIterator.html#hasPrevious%2D%2D[hasPrevious]()++ (link:{java9-javadoc}/java/util/ListIterator.html#hasPrevious%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-nextIndex-0]]int link:{java8-javadoc}/java/util/ListIterator.html#nextIndex%2D%2D[nextIndex]()++ (link:{java9-javadoc}/java/util/ListIterator.html#nextIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-previousIndex-0]]int link:{java8-javadoc}/java/util/ListIterator.html#previousIndex%2D%2D[previousIndex]()++ (link:{java9-javadoc}/java/util/ListIterator.html#previousIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ListIterator-set-1]]void link:{java8-javadoc}/java/util/ListIterator.html#set%2Djava.lang.Object%2D[set](def)++ (link:{java9-javadoc}/java/util/ListIterator.html#set%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LocalDate.asciidoc b/docs/reference/painless-api-reference/LocalDate.asciidoc
new file mode 100644
index 00000000000..8498474d02a
--- /dev/null
+++ b/docs/reference/painless-api-reference/LocalDate.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalDate]]++LocalDate++::
+** [[painless-api-reference-LocalDate-MAX]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalDate.html#MAX[java 9])
+** [[painless-api-reference-LocalDate-MIN]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalDate.html#MIN[java 9])
+* ++[[painless-api-reference-LocalDate-from-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalDate-of-3]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-ofEpochDay-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#ofEpochDay%2Dlong%2D[ofEpochDay](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#ofEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-ofYearDay-2]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#ofYearDay%2Dint%2Dint%2D[ofYearDay](int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#ofYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-parse-1]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalDate-parse-2]]static <<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atStartOfDay-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atStartOfDay%2D%2D[atStartOfDay]()++ (link:{java9-javadoc}/java/time/LocalDate.html#atStartOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atStartOfDay-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atStartOfDay%2Djava.time.ZoneId%2D[atStartOfDay](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#atStartOfDay%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Djava.time.LocalTime%2D[atTime](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2D[atTime](int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-3]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2D[atTime](int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-atTime-4]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2Dint%2D[atTime](int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDate.html#atTime%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getChronology-0]]<<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/LocalDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/LocalDate.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getDayOfYear-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/LocalDate.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getMonthValue-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-getYear-0]]int link:{java8-javadoc}/java/time/LocalDate.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/LocalDate.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minus-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minus-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusDays-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusMonths-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusWeeks-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-minusYears-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plus-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plus-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusDays-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusMonths-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusWeeks-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-plusYears-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/LocalDate.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-until-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/LocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[until](<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#until%2Djava.time.chrono.ChronoLocalDate%2D[java 9])
+* ++[[painless-api-reference-LocalDate-with-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalDate-with-2]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withDayOfMonth-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withDayOfYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withMonth-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDate-withYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDate.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/LocalDate.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/LocalDateTime.asciidoc b/docs/reference/painless-api-reference/LocalDateTime.asciidoc
new file mode 100644
index 00000000000..9507ff1dd52
--- /dev/null
+++ b/docs/reference/painless-api-reference/LocalDateTime.asciidoc
@@ -0,0 +1,61 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalDateTime]]++LocalDateTime++::
+** [[painless-api-reference-LocalDateTime-MAX]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalDateTime.html#MAX[java 9])
+** [[painless-api-reference-LocalDateTime-MIN]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalDateTime.html#MIN[java 9])
+* ++[[painless-api-reference-LocalDateTime-from-1]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-5]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-6]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-of-7]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-ofEpochSecond-3]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#ofEpochSecond%2Dlong%2Dint%2Djava.time.ZoneOffset%2D[ofEpochSecond](long, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#ofEpochSecond%2Dlong%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-ofInstant-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-parse-1]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-parse-2]]static <<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-atOffset-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-atZone-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#atZone%2Djava.time.ZoneId%2D[atZone](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#atZone%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getHour-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/LocalDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getNano-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-getYear-0]]int link:{java8-javadoc}/java/time/LocalDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minus-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minus-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusDays-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusHours-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusMinutes-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusMonths-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusNanos-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusSeconds-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusWeeks-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-minusYears-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plus-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plus-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusDays-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusHours-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusMinutes-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusMonths-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusNanos-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusSeconds-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusWeeks-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-plusYears-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/LocalDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/LocalDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-truncatedTo-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-with-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-with-2]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withDayOfMonth-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withDayOfYear-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withHour-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withMinute-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withMonth-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withSecond-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalDateTime-withYear-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/LocalDateTime.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDateTime,ChronoLocalDateTime>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/LocalTime.asciidoc b/docs/reference/painless-api-reference/LocalTime.asciidoc
new file mode 100644
index 00000000000..f12e13f4e78
--- /dev/null
+++ b/docs/reference/painless-api-reference/LocalTime.asciidoc
@@ -0,0 +1,50 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LocalTime]]++LocalTime++::
+** [[painless-api-reference-LocalTime-MAX]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/LocalTime.html#MAX[java 9])
+** [[painless-api-reference-LocalTime-MIDNIGHT]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MIDNIGHT[MIDNIGHT] (link:{java9-javadoc}/java/time/LocalTime.html#MIDNIGHT[java 9])
+** [[painless-api-reference-LocalTime-MIN]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/LocalTime.html#MIN[java 9])
+** [[painless-api-reference-LocalTime-NOON]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#NOON[NOON] (link:{java9-javadoc}/java/time/LocalTime.html#NOON[java 9])
+* ++[[painless-api-reference-LocalTime-from-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-2]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-3]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-of-4]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2Dint%2D[of](int, int, int, int)++ (link:{java9-javadoc}/java/time/LocalTime.html#of%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-ofNanoOfDay-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#ofNanoOfDay%2Dlong%2D[ofNanoOfDay](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#ofNanoOfDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-ofSecondOfDay-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#ofSecondOfDay%2Dlong%2D[ofSecondOfDay](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#ofSecondOfDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-parse-1]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-LocalTime-parse-2]]static <<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalTime-atDate-1]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/LocalTime.html#atDate%2Djava.time.LocalDate%2D[atDate](<<painless-api-reference-LocalDate,LocalDate>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#atDate%2Djava.time.LocalDate%2D[java 9])
+* ++[[painless-api-reference-LocalTime-atOffset-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/LocalTime.html#atOffset%2Djava.time.ZoneOffset%2D[atOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#atOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-LocalTime-compareTo-1]]int link:{java8-javadoc}/java/time/LocalTime.html#compareTo%2Djava.time.LocalTime%2D[compareTo](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#compareTo%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/LocalTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getHour-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getMinute-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getNano-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-getSecond-0]]int link:{java8-javadoc}/java/time/LocalTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/LocalTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/LocalTime.html#isAfter%2Djava.time.LocalTime%2D[isAfter](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#isAfter%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/LocalTime.html#isBefore%2Djava.time.LocalTime%2D[isBefore](<<painless-api-reference-LocalTime,LocalTime>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#isBefore%2Djava.time.LocalTime%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minus-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minus-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusHours-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusMinutes-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusNanos-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-minusSeconds-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plus-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plus-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusHours-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusMinutes-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusNanos-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-plusSeconds-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/LocalTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-toNanoOfDay-0]]long link:{java8-javadoc}/java/time/LocalTime.html#toNanoOfDay%2D%2D[toNanoOfDay]()++ (link:{java9-javadoc}/java/time/LocalTime.html#toNanoOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-toSecondOfDay-0]]int link:{java8-javadoc}/java/time/LocalTime.html#toSecondOfDay%2D%2D[toSecondOfDay]()++ (link:{java9-javadoc}/java/time/LocalTime.html#toSecondOfDay%2D%2D[java 9])
+* ++[[painless-api-reference-LocalTime-truncatedTo-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-LocalTime-with-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-LocalTime-with-2]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/LocalTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withHour-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withMinute-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withNano-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-LocalTime-withSecond-1]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/LocalTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/LocalTime.html#withSecond%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/Locale.Builder.asciidoc b/docs/reference/painless-api-reference/Locale.Builder.asciidoc
new file mode 100644
index 00000000000..8fad8099de9
--- /dev/null
+++ b/docs/reference/painless-api-reference/Locale.Builder.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-Builder]]++Locale.Builder++::
+* ++[[painless-api-reference-Locale-Builder-Locale.Builder-0]]link:{java8-javadoc}/java/util/Locale$Builder.html#Locale.Builder%2D%2D[Locale.Builder]()++ (link:{java9-javadoc}/java/util/Locale$Builder.html#Locale.Builder%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-addUnicodeLocaleAttribute-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#addUnicodeLocaleAttribute%2Djava.lang.String%2D[addUnicodeLocaleAttribute](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#addUnicodeLocaleAttribute%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-build-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/Locale$Builder.html#build%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-clear-0]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Locale$Builder.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-clearExtensions-0]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#clearExtensions%2D%2D[clearExtensions]()++ (link:{java9-javadoc}/java/util/Locale$Builder.html#clearExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-removeUnicodeLocaleAttribute-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#removeUnicodeLocaleAttribute%2Djava.lang.String%2D[removeUnicodeLocaleAttribute](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#removeUnicodeLocaleAttribute%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setExtension-2]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setExtension%2Dchar%2Djava.lang.String%2D[setExtension](char, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setExtension%2Dchar%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLanguage-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setLanguage%2Djava.lang.String%2D[setLanguage](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setLanguage%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLanguageTag-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setLanguageTag%2Djava.lang.String%2D[setLanguageTag](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setLanguageTag%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setLocale-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setRegion-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setRegion%2Djava.lang.String%2D[setRegion](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setRegion%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setScript-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setScript%2Djava.lang.String%2D[setScript](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setScript%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setUnicodeLocaleKeyword-2]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setUnicodeLocaleKeyword%2Djava.lang.String%2Djava.lang.String%2D[setUnicodeLocaleKeyword](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setUnicodeLocaleKeyword%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Builder-setVariant-1]]<<painless-api-reference-Locale-Builder,Locale.Builder>> link:{java8-javadoc}/java/util/Locale$Builder.html#setVariant%2Djava.lang.String%2D[setVariant](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Builder.html#setVariant%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Locale.Category.asciidoc b/docs/reference/painless-api-reference/Locale.Category.asciidoc
new file mode 100644
index 00000000000..37a57018963
--- /dev/null
+++ b/docs/reference/painless-api-reference/Locale.Category.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-Category]]++Locale.Category++::
+** [[painless-api-reference-Locale-Category-DISPLAY]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale$Category.html#DISPLAY[DISPLAY] (link:{java9-javadoc}/java/util/Locale$Category.html#DISPLAY[java 9])
+** [[painless-api-reference-Locale-Category-FORMAT]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale$Category.html#FORMAT[FORMAT] (link:{java9-javadoc}/java/util/Locale$Category.html#FORMAT[java 9])
+* ++[[painless-api-reference-Locale-Category-valueOf-1]]static <<painless-api-reference-Locale-Category,Locale.Category>> link:{java8-javadoc}/java/util/Locale$Category.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$Category.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Category-values-0]]static <<painless-api-reference-Locale-Category,Locale.Category>>[] link:{java8-javadoc}/java/util/Locale$Category.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Locale$Category.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Locale.FilteringMode.asciidoc b/docs/reference/painless-api-reference/Locale.FilteringMode.asciidoc
new file mode 100644
index 00000000000..e4399e146fc
--- /dev/null
+++ b/docs/reference/painless-api-reference/Locale.FilteringMode.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-FilteringMode]]++Locale.FilteringMode++::
+** [[painless-api-reference-Locale-FilteringMode-AUTOSELECT_FILTERING]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#AUTOSELECT_FILTERING[AUTOSELECT_FILTERING] (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#AUTOSELECT_FILTERING[java 9])
+** [[painless-api-reference-Locale-FilteringMode-EXTENDED_FILTERING]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#EXTENDED_FILTERING[EXTENDED_FILTERING] (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#EXTENDED_FILTERING[java 9])
+** [[painless-api-reference-Locale-FilteringMode-IGNORE_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#IGNORE_EXTENDED_RANGES[IGNORE_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#IGNORE_EXTENDED_RANGES[java 9])
+** [[painless-api-reference-Locale-FilteringMode-MAP_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#MAP_EXTENDED_RANGES[MAP_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#MAP_EXTENDED_RANGES[java 9])
+** [[painless-api-reference-Locale-FilteringMode-REJECT_EXTENDED_RANGES]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#REJECT_EXTENDED_RANGES[REJECT_EXTENDED_RANGES] (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#REJECT_EXTENDED_RANGES[java 9])
+* ++[[painless-api-reference-Locale-FilteringMode-valueOf-1]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>> link:{java8-javadoc}/java/util/Locale$FilteringMode.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-FilteringMode-values-0]]static <<painless-api-reference-Locale-FilteringMode,Locale.FilteringMode>>[] link:{java8-javadoc}/java/util/Locale$FilteringMode.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Locale$FilteringMode.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Locale.LanguageRange.asciidoc b/docs/reference/painless-api-reference/Locale.LanguageRange.asciidoc
new file mode 100644
index 00000000000..0e0038d3dd6
--- /dev/null
+++ b/docs/reference/painless-api-reference/Locale.LanguageRange.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale-LanguageRange]]++Locale.LanguageRange++::
+** [[painless-api-reference-Locale-LanguageRange-MAX_WEIGHT]]static double link:{java8-javadoc}/java/util/Locale$LanguageRange.html#MAX_WEIGHT[MAX_WEIGHT] (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#MAX_WEIGHT[java 9])
+** [[painless-api-reference-Locale-LanguageRange-MIN_WEIGHT]]static double link:{java8-javadoc}/java/util/Locale$LanguageRange.html#MIN_WEIGHT[MIN_WEIGHT] (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#MIN_WEIGHT[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-mapEquivalents-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale$LanguageRange.html#mapEquivalents%2Djava.util.List%2Djava.util.Map%2D[mapEquivalents](<<painless-api-reference-List,List>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#mapEquivalents%2Djava.util.List%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-parse-1]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale$LanguageRange.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-parse-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale$LanguageRange.html#parse%2Djava.lang.String%2Djava.util.Map%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#parse%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-Locale.LanguageRange-1]]link:{java8-javadoc}/java/util/Locale$LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2D[Locale.LanguageRange](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-Locale.LanguageRange-2]]link:{java8-javadoc}/java/util/Locale$LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2Ddouble%2D[Locale.LanguageRange](<<painless-api-reference-String,String>>, double)++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#Locale.LanguageRange%2Djava.lang.String%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-getRange-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale$LanguageRange.html#getRange%2D%2D[getRange]()++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#getRange%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-LanguageRange-getWeight-0]]double link:{java8-javadoc}/java/util/Locale$LanguageRange.html#getWeight%2D%2D[getWeight]()++ (link:{java9-javadoc}/java/util/Locale$LanguageRange.html#getWeight%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Locale.asciidoc b/docs/reference/painless-api-reference/Locale.asciidoc
new file mode 100644
index 00000000000..d041c6c1f7f
--- /dev/null
+++ b/docs/reference/painless-api-reference/Locale.asciidoc
@@ -0,0 +1,69 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Locale]]++Locale++::
+** [[painless-api-reference-Locale-CANADA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CANADA[CANADA] (link:{java9-javadoc}/java/util/Locale.html#CANADA[java 9])
+** [[painless-api-reference-Locale-CANADA_FRENCH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CANADA_FRENCH[CANADA_FRENCH] (link:{java9-javadoc}/java/util/Locale.html#CANADA_FRENCH[java 9])
+** [[painless-api-reference-Locale-CHINA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CHINA[CHINA] (link:{java9-javadoc}/java/util/Locale.html#CHINA[java 9])
+** [[painless-api-reference-Locale-CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#CHINESE[CHINESE] (link:{java9-javadoc}/java/util/Locale.html#CHINESE[java 9])
+** [[painless-api-reference-Locale-ENGLISH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ENGLISH[ENGLISH] (link:{java9-javadoc}/java/util/Locale.html#ENGLISH[java 9])
+** [[painless-api-reference-Locale-FRANCE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#FRANCE[FRANCE] (link:{java9-javadoc}/java/util/Locale.html#FRANCE[java 9])
+** [[painless-api-reference-Locale-FRENCH]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#FRENCH[FRENCH] (link:{java9-javadoc}/java/util/Locale.html#FRENCH[java 9])
+** [[painless-api-reference-Locale-GERMAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#GERMAN[GERMAN] (link:{java9-javadoc}/java/util/Locale.html#GERMAN[java 9])
+** [[painless-api-reference-Locale-GERMANY]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#GERMANY[GERMANY] (link:{java9-javadoc}/java/util/Locale.html#GERMANY[java 9])
+** [[painless-api-reference-Locale-ITALIAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ITALIAN[ITALIAN] (link:{java9-javadoc}/java/util/Locale.html#ITALIAN[java 9])
+** [[painless-api-reference-Locale-ITALY]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ITALY[ITALY] (link:{java9-javadoc}/java/util/Locale.html#ITALY[java 9])
+** [[painless-api-reference-Locale-JAPAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#JAPAN[JAPAN] (link:{java9-javadoc}/java/util/Locale.html#JAPAN[java 9])
+** [[painless-api-reference-Locale-JAPANESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#JAPANESE[JAPANESE] (link:{java9-javadoc}/java/util/Locale.html#JAPANESE[java 9])
+** [[painless-api-reference-Locale-KOREA]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#KOREA[KOREA] (link:{java9-javadoc}/java/util/Locale.html#KOREA[java 9])
+** [[painless-api-reference-Locale-KOREAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#KOREAN[KOREAN] (link:{java9-javadoc}/java/util/Locale.html#KOREAN[java 9])
+** [[painless-api-reference-Locale-PRC]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#PRC[PRC] (link:{java9-javadoc}/java/util/Locale.html#PRC[java 9])
+** [[painless-api-reference-Locale-PRIVATE_USE_EXTENSION]]static char link:{java8-javadoc}/java/util/Locale.html#PRIVATE_USE_EXTENSION[PRIVATE_USE_EXTENSION] (link:{java9-javadoc}/java/util/Locale.html#PRIVATE_USE_EXTENSION[java 9])
+** [[painless-api-reference-Locale-ROOT]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#ROOT[ROOT] (link:{java9-javadoc}/java/util/Locale.html#ROOT[java 9])
+** [[painless-api-reference-Locale-SIMPLIFIED_CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#SIMPLIFIED_CHINESE[SIMPLIFIED_CHINESE] (link:{java9-javadoc}/java/util/Locale.html#SIMPLIFIED_CHINESE[java 9])
+** [[painless-api-reference-Locale-TAIWAN]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#TAIWAN[TAIWAN] (link:{java9-javadoc}/java/util/Locale.html#TAIWAN[java 9])
+** [[painless-api-reference-Locale-TRADITIONAL_CHINESE]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#TRADITIONAL_CHINESE[TRADITIONAL_CHINESE] (link:{java9-javadoc}/java/util/Locale.html#TRADITIONAL_CHINESE[java 9])
+** [[painless-api-reference-Locale-UK]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#UK[UK] (link:{java9-javadoc}/java/util/Locale.html#UK[java 9])
+** [[painless-api-reference-Locale-UNICODE_LOCALE_EXTENSION]]static char link:{java8-javadoc}/java/util/Locale.html#UNICODE_LOCALE_EXTENSION[UNICODE_LOCALE_EXTENSION] (link:{java9-javadoc}/java/util/Locale.html#UNICODE_LOCALE_EXTENSION[java 9])
+** [[painless-api-reference-Locale-US]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#US[US] (link:{java9-javadoc}/java/util/Locale.html#US[java 9])
+* ++[[painless-api-reference-Locale-filter-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.html#filter%2Djava.util.List%2Djava.util.Collection%2D[filter](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#filter%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-filterTags-2]]static <<painless-api-reference-List,List>> link:{java8-javadoc}/java/util/Locale.html#filterTags%2Djava.util.List%2Djava.util.Collection%2D[filterTags](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#filterTags%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-forLanguageTag-1]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#forLanguageTag%2Djava.lang.String%2D[forLanguageTag](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#forLanguageTag%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/util/Locale.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/util/Locale.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDefault-0]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#getDefault%2D%2D[getDefault]()++ (link:{java9-javadoc}/java/util/Locale.html#getDefault%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDefault-1]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#getDefault%2Djava.util.Locale$Category%2D[getDefault](<<painless-api-reference-Locale-Category,Locale.Category>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDefault%2Djava.util.Locale$Category%2D[java 9])
+* ++[[painless-api-reference-Locale-getISOCountries-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/Locale.html#getISOCountries%2D%2D[getISOCountries]()++ (link:{java9-javadoc}/java/util/Locale.html#getISOCountries%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISOLanguages-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/Locale.html#getISOLanguages%2D%2D[getISOLanguages]()++ (link:{java9-javadoc}/java/util/Locale.html#getISOLanguages%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-lookup-2]]static <<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#lookup%2Djava.util.List%2Djava.util.Collection%2D[lookup](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#lookup%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-lookupTag-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#lookupTag%2Djava.util.List%2Djava.util.Collection%2D[lookupTag](<<painless-api-reference-List,List>>, <<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Locale.html#lookupTag%2Djava.util.List%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-1]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-2]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-Locale-3]]link:{java8-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[Locale](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#Locale%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-clone-0]]def link:{java8-javadoc}/java/util/Locale.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Locale.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getCountry-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getCountry%2D%2D[getCountry]()++ (link:{java9-javadoc}/java/util/Locale.html#getCountry%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayCountry-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayCountry%2D%2D[getDisplayCountry]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayCountry%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayCountry-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayCountry%2Djava.util.Locale%2D[getDisplayCountry](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayCountry%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayLanguage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayLanguage%2D%2D[getDisplayLanguage]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayLanguage%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayLanguage-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayLanguage%2Djava.util.Locale%2D[getDisplayLanguage](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayLanguage%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayScript-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayScript%2D%2D[getDisplayScript]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayScript%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayScript-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayScript%2Djava.util.Locale%2D[getDisplayScript](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayScript%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayVariant-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayVariant%2D%2D[getDisplayVariant]()++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayVariant%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getDisplayVariant-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getDisplayVariant%2Djava.util.Locale%2D[getDisplayVariant](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/Locale.html#getDisplayVariant%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Locale-getExtension-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getExtension%2Dchar%2D[getExtension](char)++ (link:{java9-javadoc}/java/util/Locale.html#getExtension%2Dchar%2D[java 9])
+* ++[[painless-api-reference-Locale-getExtensionKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getExtensionKeys%2D%2D[getExtensionKeys]()++ (link:{java9-javadoc}/java/util/Locale.html#getExtensionKeys%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISO3Country-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getISO3Country%2D%2D[getISO3Country]()++ (link:{java9-javadoc}/java/util/Locale.html#getISO3Country%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getISO3Language-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getISO3Language%2D%2D[getISO3Language]()++ (link:{java9-javadoc}/java/util/Locale.html#getISO3Language%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getLanguage-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getLanguage%2D%2D[getLanguage]()++ (link:{java9-javadoc}/java/util/Locale.html#getLanguage%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getScript-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getScript%2D%2D[getScript]()++ (link:{java9-javadoc}/java/util/Locale.html#getScript%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleAttributes-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleAttributes%2D%2D[getUnicodeLocaleAttributes]()++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleAttributes%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleKeys-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleKeys%2D%2D[getUnicodeLocaleKeys]()++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleKeys%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-getUnicodeLocaleType-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getUnicodeLocaleType%2Djava.lang.String%2D[getUnicodeLocaleType](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Locale.html#getUnicodeLocaleType%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Locale-getVariant-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#getVariant%2D%2D[getVariant]()++ (link:{java9-javadoc}/java/util/Locale.html#getVariant%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-hasExtensions-0]]boolean link:{java8-javadoc}/java/util/Locale.html#hasExtensions%2D%2D[hasExtensions]()++ (link:{java9-javadoc}/java/util/Locale.html#hasExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-stripExtensions-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/util/Locale.html#stripExtensions%2D%2D[stripExtensions]()++ (link:{java9-javadoc}/java/util/Locale.html#stripExtensions%2D%2D[java 9])
+* ++[[painless-api-reference-Locale-toLanguageTag-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Locale.html#toLanguageTag%2D%2D[toLanguageTag]()++ (link:{java9-javadoc}/java/util/Locale.html#toLanguageTag%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Long.asciidoc b/docs/reference/painless-api-reference/Long.asciidoc
new file mode 100644
index 00000000000..e35c80f63b9
--- /dev/null
+++ b/docs/reference/painless-api-reference/Long.asciidoc
@@ -0,0 +1,44 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Long]]++Long++::
+** [[painless-api-reference-Long-BYTES]]static int link:{java8-javadoc}/java/lang/Long.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Long.html#BYTES[java 9])
+** [[painless-api-reference-Long-MAX_VALUE]]static long link:{java8-javadoc}/java/lang/Long.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Long.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Long-MIN_VALUE]]static long link:{java8-javadoc}/java/lang/Long.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Long.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Long-SIZE]]static int link:{java8-javadoc}/java/lang/Long.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Long.html#SIZE[java 9])
+* ++[[painless-api-reference-Long-bitCount-1]]static int link:{java8-javadoc}/java/lang/Long.html#bitCount%2Dlong%2D[bitCount](long)++ (link:{java9-javadoc}/java/lang/Long.html#bitCount%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-compare-2]]static int link:{java8-javadoc}/java/lang/Long.html#compare%2Dlong%2Dlong%2D[compare](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#compare%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-compareUnsigned-2]]static int link:{java8-javadoc}/java/lang/Long.html#compareUnsigned%2Dlong%2Dlong%2D[compareUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#compareUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-decode-1]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-divideUnsigned-2]]static long link:{java8-javadoc}/java/lang/Long.html#divideUnsigned%2Dlong%2Dlong%2D[divideUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#divideUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-hashCode-1]]static int link:{java8-javadoc}/java/lang/Long.html#hashCode%2Dlong%2D[hashCode](long)++ (link:{java9-javadoc}/java/lang/Long.html#hashCode%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-highestOneBit-1]]static long link:{java8-javadoc}/java/lang/Long.html#highestOneBit%2Dlong%2D[highestOneBit](long)++ (link:{java9-javadoc}/java/lang/Long.html#highestOneBit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-lowestOneBit-1]]static long link:{java8-javadoc}/java/lang/Long.html#lowestOneBit%2Dlong%2D[lowestOneBit](long)++ (link:{java9-javadoc}/java/lang/Long.html#lowestOneBit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-max-2]]static long link:{java8-javadoc}/java/lang/Long.html#max%2Dlong%2Dlong%2D[max](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#max%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-min-2]]static long link:{java8-javadoc}/java/lang/Long.html#min%2Dlong%2Dlong%2D[min](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#min%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-numberOfLeadingZeros-1]]static int link:{java8-javadoc}/java/lang/Long.html#numberOfLeadingZeros%2Dlong%2D[numberOfLeadingZeros](long)++ (link:{java9-javadoc}/java/lang/Long.html#numberOfLeadingZeros%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-numberOfTrailingZeros-1]]static int link:{java8-javadoc}/java/lang/Long.html#numberOfTrailingZeros%2Dlong%2D[numberOfTrailingZeros](long)++ (link:{java9-javadoc}/java/lang/Long.html#numberOfTrailingZeros%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-parseLong-1]]static long link:{java8-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2D[parseLong](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-parseLong-2]]static long link:{java8-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2Dint%2D[parseLong](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#parseLong%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-parseUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2D[parseUnsignedLong](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Long-parseUnsignedLong-2]]static long link:{java8-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2Dint%2D[parseUnsignedLong](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#parseUnsignedLong%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-remainderUnsigned-2]]static long link:{java8-javadoc}/java/lang/Long.html#remainderUnsigned%2Dlong%2Dlong%2D[remainderUnsigned](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#remainderUnsigned%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-reverse-1]]static long link:{java8-javadoc}/java/lang/Long.html#reverse%2Dlong%2D[reverse](long)++ (link:{java9-javadoc}/java/lang/Long.html#reverse%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-reverseBytes-1]]static long link:{java8-javadoc}/java/lang/Long.html#reverseBytes%2Dlong%2D[reverseBytes](long)++ (link:{java9-javadoc}/java/lang/Long.html#reverseBytes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-rotateLeft-2]]static long link:{java8-javadoc}/java/lang/Long.html#rotateLeft%2Dlong%2Dint%2D[rotateLeft](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#rotateLeft%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-rotateRight-2]]static long link:{java8-javadoc}/java/lang/Long.html#rotateRight%2Dlong%2Dint%2D[rotateRight](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#rotateRight%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-signum-1]]static int link:{java8-javadoc}/java/lang/Long.html#signum%2Dlong%2D[signum](long)++ (link:{java9-javadoc}/java/lang/Long.html#signum%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-sum-2]]static long link:{java8-javadoc}/java/lang/Long.html#sum%2Dlong%2Dlong%2D[sum](long, long)++ (link:{java9-javadoc}/java/lang/Long.html#sum%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toBinaryString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toBinaryString%2Dlong%2D[toBinaryString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toBinaryString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toHexString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toHexString%2Dlong%2D[toHexString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toHexString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toOctalString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toOctalString%2Dlong%2D[toOctalString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toOctalString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toString%2Dlong%2D[toString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toString%2Dlong%2Dint%2D[toString](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#toString%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-toUnsignedString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2D[toUnsignedString](long)++ (link:{java9-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-toUnsignedString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2Dint%2D[toUnsignedString](long, int)++ (link:{java9-javadoc}/java/lang/Long.html#toUnsignedString%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-valueOf-1]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#valueOf%2Dlong%2D[valueOf](long)++ (link:{java9-javadoc}/java/lang/Long.html#valueOf%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Long-valueOf-2]]static <<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/lang/Long.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Long.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Long-compareTo-1]]int link:{java8-javadoc}/java/lang/Long.html#compareTo%2Djava.lang.Long%2D[compareTo](<<painless-api-reference-Long,Long>>)++ (link:{java9-javadoc}/java/lang/Long.html#compareTo%2Djava.lang.Long%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongBinaryOperator.asciidoc b/docs/reference/painless-api-reference/LongBinaryOperator.asciidoc
new file mode 100644
index 00000000000..45a777a6971
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongBinaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongBinaryOperator]]++LongBinaryOperator++::
+* ++[[painless-api-reference-LongBinaryOperator-applyAsLong-2]]long link:{java8-javadoc}/java/util/function/LongBinaryOperator.html#applyAsLong%2Dlong%2Dlong%2D[applyAsLong](long, long)++ (link:{java9-javadoc}/java/util/function/LongBinaryOperator.html#applyAsLong%2Dlong%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongConsumer.asciidoc b/docs/reference/painless-api-reference/LongConsumer.asciidoc
new file mode 100644
index 00000000000..28bbc92c031
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongConsumer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongConsumer]]++LongConsumer++::
+* ++[[painless-api-reference-LongConsumer-accept-1]]void link:{java8-javadoc}/java/util/function/LongConsumer.html#accept%2Dlong%2D[accept](long)++ (link:{java9-javadoc}/java/util/function/LongConsumer.html#accept%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongConsumer-andThen-1]]<<painless-api-reference-LongConsumer,LongConsumer>> link:{java8-javadoc}/java/util/function/LongConsumer.html#andThen%2Djava.util.function.LongConsumer%2D[andThen](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/function/LongConsumer.html#andThen%2Djava.util.function.LongConsumer%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongFunction.asciidoc b/docs/reference/painless-api-reference/LongFunction.asciidoc
new file mode 100644
index 00000000000..033392914a5
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongFunction]]++LongFunction++::
+* ++[[painless-api-reference-LongFunction-apply-1]]def link:{java8-javadoc}/java/util/function/LongFunction.html#apply%2Dlong%2D[apply](long)++ (link:{java9-javadoc}/java/util/function/LongFunction.html#apply%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongPredicate.asciidoc b/docs/reference/painless-api-reference/LongPredicate.asciidoc
new file mode 100644
index 00000000000..59b719572e3
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongPredicate.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongPredicate]]++LongPredicate++::
+* ++[[painless-api-reference-LongPredicate-and-1]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#and%2Djava.util.function.LongPredicate%2D[and](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#and%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-negate-0]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-or-1]]<<painless-api-reference-LongPredicate,LongPredicate>> link:{java8-javadoc}/java/util/function/LongPredicate.html#or%2Djava.util.function.LongPredicate%2D[or](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#or%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongPredicate-test-1]]boolean link:{java8-javadoc}/java/util/function/LongPredicate.html#test%2Dlong%2D[test](long)++ (link:{java9-javadoc}/java/util/function/LongPredicate.html#test%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongStream.Builder.asciidoc b/docs/reference/painless-api-reference/LongStream.Builder.asciidoc
new file mode 100644
index 00000000000..98c55b149ae
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongStream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongStream-Builder]]++LongStream.Builder++::
+* ++[[painless-api-reference-LongStream-Builder-add-1]]<<painless-api-reference-LongStream-Builder,LongStream.Builder>> link:{java8-javadoc}/java/util/stream/LongStream$Builder.html#add%2Dlong%2D[add](long)++ (link:{java9-javadoc}/java/util/stream/LongStream$Builder.html#add%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-Builder-build-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/LongStream$Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-LongConsumer,LongConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongStream.asciidoc b/docs/reference/painless-api-reference/LongStream.asciidoc
new file mode 100644
index 00000000000..08ea178451a
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongStream.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongStream]]++LongStream++::
+* ++[[painless-api-reference-LongStream-builder-0]]static <<painless-api-reference-LongStream-Builder,LongStream.Builder>> link:{java8-javadoc}/java/util/stream/LongStream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-concat-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#concat%2Djava.util.stream.LongStream%2Djava.util.stream.LongStream%2D[concat](<<painless-api-reference-LongStream,LongStream>>, <<painless-api-reference-LongStream,LongStream>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#concat%2Djava.util.stream.LongStream%2Djava.util.stream.LongStream%2D[java 9])
+* ++[[painless-api-reference-LongStream-empty-0]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-of-1]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#of%2Dlong:A%2D[of](long[])++ (link:{java9-javadoc}/java/util/stream/LongStream.html#of%2Dlong:A%2D[java 9])
+* ++[[painless-api-reference-LongStream-range-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#range%2Dlong%2Dlong%2D[range](long, long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#range%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-rangeClosed-2]]static <<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#rangeClosed%2Dlong%2Dlong%2D[rangeClosed](long, long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#rangeClosed%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#allMatch%2Djava.util.function.LongPredicate%2D[allMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#allMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#anyMatch%2Djava.util.function.LongPredicate%2D[anyMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#anyMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-asDoubleStream-0]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#asDoubleStream%2D%2D[asDoubleStream]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#asDoubleStream%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-average-0]]<<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/stream/LongStream.html#average%2D%2D[average]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#average%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-boxed-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/LongStream.html#boxed%2D%2D[boxed]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#boxed%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-collect-3]]def link:{java8-javadoc}/java/util/stream/LongStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjLongConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-ObjLongConsumer,ObjLongConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.ObjLongConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-count-0]]long link:{java8-javadoc}/java/util/stream/LongStream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-distinct-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-filter-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#filter%2Djava.util.function.LongPredicate%2D[filter](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#filter%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-findAny-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-findFirst-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-flatMap-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#flatMap%2Djava.util.function.LongFunction%2D[flatMap](<<painless-api-reference-LongFunction,LongFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#flatMap%2Djava.util.function.LongFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-forEach-1]]void link:{java8-javadoc}/java/util/stream/LongStream.html#forEach%2Djava.util.function.LongConsumer%2D[forEach](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#forEach%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/LongStream.html#forEachOrdered%2Djava.util.function.LongConsumer%2D[forEachOrdered](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#forEachOrdered%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-iterator-0]]<<painless-api-reference-PrimitiveIterator-OfLong,PrimitiveIterator.OfLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#iterator%2D%2D[iterator]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#iterator%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-limit-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-map-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#map%2Djava.util.function.LongUnaryOperator%2D[map](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#map%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToDouble%2Djava.util.function.LongToDoubleFunction%2D[mapToDouble](<<painless-api-reference-LongToDoubleFunction,LongToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToDouble%2Djava.util.function.LongToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToInt%2Djava.util.function.LongToIntFunction%2D[mapToInt](<<painless-api-reference-LongToIntFunction,LongToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToInt%2Djava.util.function.LongToIntFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-mapToObj-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/LongStream.html#mapToObj%2Djava.util.function.LongFunction%2D[mapToObj](<<painless-api-reference-LongFunction,LongFunction>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#mapToObj%2Djava.util.function.LongFunction%2D[java 9])
+* ++[[painless-api-reference-LongStream-max-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#max%2D%2D[max]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#max%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-min-0]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#min%2D%2D[min]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#min%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/LongStream.html#noneMatch%2Djava.util.function.LongPredicate%2D[noneMatch](<<painless-api-reference-LongPredicate,LongPredicate>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#noneMatch%2Djava.util.function.LongPredicate%2D[java 9])
+* ++[[painless-api-reference-LongStream-peek-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#peek%2Djava.util.function.LongConsumer%2D[peek](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#peek%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-LongStream-reduce-1]]<<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#reduce%2Djava.util.function.LongBinaryOperator%2D[reduce](<<painless-api-reference-LongBinaryOperator,LongBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#reduce%2Djava.util.function.LongBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-reduce-2]]long link:{java8-javadoc}/java/util/stream/LongStream.html#reduce%2Dlong%2Djava.util.function.LongBinaryOperator%2D[reduce](long, <<painless-api-reference-LongBinaryOperator,LongBinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#reduce%2Dlong%2Djava.util.function.LongBinaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongStream-sequential-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#sequential%2D%2D[sequential]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sequential%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-skip-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/LongStream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongStream-sorted-0]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/LongStream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-spliterator-0]]<<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/stream/LongStream.html#spliterator%2D%2D[spliterator]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#spliterator%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-sum-0]]long link:{java8-javadoc}/java/util/stream/LongStream.html#sum%2D%2D[sum]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#sum%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-summaryStatistics-0]]<<painless-api-reference-LongSummaryStatistics,LongSummaryStatistics>> link:{java8-javadoc}/java/util/stream/LongStream.html#summaryStatistics%2D%2D[summaryStatistics]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#summaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-LongStream-toArray-0]]long[] link:{java8-javadoc}/java/util/stream/LongStream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/LongStream.html#toArray%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongSummaryStatistics.asciidoc b/docs/reference/painless-api-reference/LongSummaryStatistics.asciidoc
new file mode 100644
index 00000000000..b678d51efa4
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongSummaryStatistics.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongSummaryStatistics]]++LongSummaryStatistics++::
+* ++[[painless-api-reference-LongSummaryStatistics-LongSummaryStatistics-0]]link:{java8-javadoc}/java/util/LongSummaryStatistics.html#LongSummaryStatistics%2D%2D[LongSummaryStatistics]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#LongSummaryStatistics%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-combine-1]]void link:{java8-javadoc}/java/util/LongSummaryStatistics.html#combine%2Djava.util.LongSummaryStatistics%2D[combine](<<painless-api-reference-LongSummaryStatistics,LongSummaryStatistics>>)++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#combine%2Djava.util.LongSummaryStatistics%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getAverage-0]]double link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getAverage%2D%2D[getAverage]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getAverage%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getCount-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getCount%2D%2D[getCount]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getCount%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getMax-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getMax%2D%2D[getMax]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getMax%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getMin-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getMin%2D%2D[getMin]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getMin%2D%2D[java 9])
+* ++[[painless-api-reference-LongSummaryStatistics-getSum-0]]long link:{java8-javadoc}/java/util/LongSummaryStatistics.html#getSum%2D%2D[getSum]()++ (link:{java9-javadoc}/java/util/LongSummaryStatistics.html#getSum%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-LongConsumer,LongConsumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongSupplier.asciidoc b/docs/reference/painless-api-reference/LongSupplier.asciidoc
new file mode 100644
index 00000000000..6693e51f17a
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongSupplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongSupplier]]++LongSupplier++::
+* ++[[painless-api-reference-LongSupplier-getAsLong-0]]long link:{java8-javadoc}/java/util/function/LongSupplier.html#getAsLong%2D%2D[getAsLong]()++ (link:{java9-javadoc}/java/util/function/LongSupplier.html#getAsLong%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongToDoubleFunction.asciidoc b/docs/reference/painless-api-reference/LongToDoubleFunction.asciidoc
new file mode 100644
index 00000000000..0bed3833371
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongToDoubleFunction]]++LongToDoubleFunction++::
+* ++[[painless-api-reference-LongToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/LongToDoubleFunction.html#applyAsDouble%2Dlong%2D[applyAsDouble](long)++ (link:{java9-javadoc}/java/util/function/LongToDoubleFunction.html#applyAsDouble%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongToIntFunction.asciidoc b/docs/reference/painless-api-reference/LongToIntFunction.asciidoc
new file mode 100644
index 00000000000..2586ba85511
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongToIntFunction]]++LongToIntFunction++::
+* ++[[painless-api-reference-LongToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/LongToIntFunction.html#applyAsInt%2Dlong%2D[applyAsInt](long)++ (link:{java9-javadoc}/java/util/function/LongToIntFunction.html#applyAsInt%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/LongUnaryOperator.asciidoc b/docs/reference/painless-api-reference/LongUnaryOperator.asciidoc
new file mode 100644
index 00000000000..eb84a3688bd
--- /dev/null
+++ b/docs/reference/painless-api-reference/LongUnaryOperator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-LongUnaryOperator]]++LongUnaryOperator++::
+* ++[[painless-api-reference-LongUnaryOperator-identity-0]]static <<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#identity%2D%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-andThen-1]]<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#andThen%2Djava.util.function.LongUnaryOperator%2D[andThen](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#andThen%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#applyAsLong%2Dlong%2D[applyAsLong](long)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#applyAsLong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-LongUnaryOperator-compose-1]]<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>> link:{java8-javadoc}/java/util/function/LongUnaryOperator.html#compose%2Djava.util.function.LongUnaryOperator%2D[compose](<<painless-api-reference-LongUnaryOperator,LongUnaryOperator>>)++ (link:{java9-javadoc}/java/util/function/LongUnaryOperator.html#compose%2Djava.util.function.LongUnaryOperator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Map.Entry.asciidoc b/docs/reference/painless-api-reference/Map.Entry.asciidoc
new file mode 100644
index 00000000000..f40e61a70b0
--- /dev/null
+++ b/docs/reference/painless-api-reference/Map.Entry.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Map-Entry]]++Map.Entry++::
+* ++[[painless-api-reference-Map-Entry-comparingByKey-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map$Entry.html#comparingByKey%2D%2D[comparingByKey]()++ (link:{java9-javadoc}/java/util/Map$Entry.html#comparingByKey%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByKey-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map$Entry.html#comparingByKey%2Djava.util.Comparator%2D[comparingByKey](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Map$Entry.html#comparingByKey%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByValue-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map$Entry.html#comparingByValue%2D%2D[comparingByValue]()++ (link:{java9-javadoc}/java/util/Map$Entry.html#comparingByValue%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-comparingByValue-1]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Map$Entry.html#comparingByValue%2Djava.util.Comparator%2D[comparingByValue](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Map$Entry.html#comparingByValue%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-equals-1]]boolean link:{java8-javadoc}/java/util/Map$Entry.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Map$Entry.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-getKey-0]]def link:{java8-javadoc}/java/util/Map$Entry.html#getKey%2D%2D[getKey]()++ (link:{java9-javadoc}/java/util/Map$Entry.html#getKey%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-getValue-0]]def link:{java8-javadoc}/java/util/Map$Entry.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/util/Map$Entry.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-hashCode-0]]int link:{java8-javadoc}/java/util/Map$Entry.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/Map$Entry.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Map-Entry-setValue-1]]def link:{java8-javadoc}/java/util/Map$Entry.html#setValue%2Djava.lang.Object%2D[setValue](def)++ (link:{java9-javadoc}/java/util/Map$Entry.html#setValue%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Map.asciidoc b/docs/reference/painless-api-reference/Map.asciidoc
new file mode 100644
index 00000000000..a32c9c137d8
--- /dev/null
+++ b/docs/reference/painless-api-reference/Map.asciidoc
@@ -0,0 +1,42 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Map]]++Map++::
+* ++[[painless-api-reference-Map-clear-0]]void link:{java8-javadoc}/java/util/Map.html#clear%2D%2D[clear]()++ (link:{java9-javadoc}/java/util/Map.html#clear%2D%2D[java 9])
+* ++[[painless-api-reference-Map-collect-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Map%2Djava.util.function.BiFunction%2D[collect](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-collect-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#collect%2Djava.util.Map%2Djava.util.Collection%2Djava.util.function.BiFunction%2D[collect](<<painless-api-reference-Collection,Collection>>, <<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-compute-2]]def link:{java8-javadoc}/java/util/Map.html#compute%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[compute](def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#compute%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-computeIfAbsent-2]]def link:{java8-javadoc}/java/util/Map.html#computeIfAbsent%2Djava.lang.Object%2Djava.util.function.Function%2D[computeIfAbsent](def, <<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Map.html#computeIfAbsent%2Djava.lang.Object%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Map-computeIfPresent-2]]def link:{java8-javadoc}/java/util/Map.html#computeIfPresent%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[computeIfPresent](def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#computeIfPresent%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-containsKey-1]]boolean link:{java8-javadoc}/java/util/Map.html#containsKey%2Djava.lang.Object%2D[containsKey](def)++ (link:{java9-javadoc}/java/util/Map.html#containsKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-containsValue-1]]boolean link:{java8-javadoc}/java/util/Map.html#containsValue%2Djava.lang.Object%2D[containsValue](def)++ (link:{java9-javadoc}/java/util/Map.html#containsValue%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-count-1]]int link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#count%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[count](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-each-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#each%2Djava.util.Map%2Djava.util.function.BiConsumer%2D[each](<<painless-api-reference-BiConsumer,BiConsumer>>)++
+* ++[[painless-api-reference-Map-entrySet-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Map.html#entrySet%2D%2D[entrySet]()++ (link:{java9-javadoc}/java/util/Map.html#entrySet%2D%2D[java 9])
+* ++[[painless-api-reference-Map-equals-1]]boolean link:{java8-javadoc}/java/util/Map.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Map.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-every-1]]boolean link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#every%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[every](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-find-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#find%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[find](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-findAll-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findAll%2Djava.util.Map%2Djava.util.function.BiPredicate%2D[findAll](<<painless-api-reference-BiPredicate,BiPredicate>>)++
+* ++[[painless-api-reference-Map-findResult-1]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Map%2Djava.util.function.BiFunction%2D[findResult](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-findResult-2]]def link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResult%2Djava.util.Map%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[findResult](def, <<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-findResults-1]]<<painless-api-reference-List,List>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#findResults%2Djava.util.Map%2Djava.util.function.BiFunction%2D[findResults](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-forEach-1]]void link:{java8-javadoc}/java/util/Map.html#forEach%2Djava.util.function.BiConsumer%2D[forEach](<<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/Map.html#forEach%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-Map-get-1]]def link:{java8-javadoc}/java/util/Map.html#get%2Djava.lang.Object%2D[get](def)++ (link:{java9-javadoc}/java/util/Map.html#get%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-getOrDefault-2]]def link:{java8-javadoc}/java/util/Map.html#getOrDefault%2Djava.lang.Object%2Djava.lang.Object%2D[getOrDefault](def, def)++ (link:{java9-javadoc}/java/util/Map.html#getOrDefault%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-groupBy-1]]<<painless-api-reference-Map,Map>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#groupBy%2Djava.util.Map%2Djava.util.function.BiFunction%2D[groupBy](<<painless-api-reference-BiFunction,BiFunction>>)++
+* ++[[painless-api-reference-Map-isEmpty-0]]boolean link:{java8-javadoc}/java/util/Map.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/util/Map.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-Map-keySet-0]]<<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/util/Map.html#keySet%2D%2D[keySet]()++ (link:{java9-javadoc}/java/util/Map.html#keySet%2D%2D[java 9])
+* ++[[painless-api-reference-Map-merge-3]]def link:{java8-javadoc}/java/util/Map.html#merge%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[merge](def, def, <<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#merge%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-put-2]]def link:{java8-javadoc}/java/util/Map.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[put](def, def)++ (link:{java9-javadoc}/java/util/Map.html#put%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-putAll-1]]void link:{java8-javadoc}/java/util/Map.html#putAll%2Djava.util.Map%2D[putAll](<<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/util/Map.html#putAll%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-Map-putIfAbsent-2]]def link:{java8-javadoc}/java/util/Map.html#putIfAbsent%2Djava.lang.Object%2Djava.lang.Object%2D[putIfAbsent](def, def)++ (link:{java9-javadoc}/java/util/Map.html#putIfAbsent%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-remove-1]]def link:{java8-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-remove-2]]boolean link:{java8-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2Djava.lang.Object%2D[remove](def, def)++ (link:{java9-javadoc}/java/util/Map.html#remove%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replace-2]]def link:{java8-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2D[replace](def, def)++ (link:{java9-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replace-3]]boolean link:{java8-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2Djava.lang.Object%2D[replace](def, def, def)++ (link:{java9-javadoc}/java/util/Map.html#replace%2Djava.lang.Object%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Map-replaceAll-1]]void link:{java8-javadoc}/java/util/Map.html#replaceAll%2Djava.util.function.BiFunction%2D[replaceAll](<<painless-api-reference-BiFunction,BiFunction>>)++ (link:{java9-javadoc}/java/util/Map.html#replaceAll%2Djava.util.function.BiFunction%2D[java 9])
+* ++[[painless-api-reference-Map-size-0]]int link:{java8-javadoc}/java/util/Map.html#size%2D%2D[size]()++ (link:{java9-javadoc}/java/util/Map.html#size%2D%2D[java 9])
+* ++[[painless-api-reference-Map-values-0]]<<painless-api-reference-Collection,Collection>> link:{java8-javadoc}/java/util/Map.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/util/Map.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Matcher.asciidoc b/docs/reference/painless-api-reference/Matcher.asciidoc
new file mode 100644
index 00000000000..85198e0bc8d
--- /dev/null
+++ b/docs/reference/painless-api-reference/Matcher.asciidoc
@@ -0,0 +1,34 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Matcher]]++Matcher++::
+* ++[[painless-api-reference-Matcher-quoteReplacement-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#quoteReplacement%2Djava.lang.String%2D[quoteReplacement](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#quoteReplacement%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-end-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#end%2D%2D[end]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#end%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-end-1]]int link:{java8-javadoc}/java/util/regex/Matcher.html#end%2Dint%2D[end](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#end%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-find-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#find%2D%2D[find]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#find%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-find-1]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#find%2Dint%2D[find](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#find%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-group-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#group%2D%2D[group]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#group%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-group-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#group%2Dint%2D[group](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#group%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-groupCount-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#groupCount%2D%2D[groupCount]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#groupCount%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hasAnchoringBounds-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hasAnchoringBounds%2D%2D[hasAnchoringBounds]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hasAnchoringBounds%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hasTransparentBounds-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hasTransparentBounds%2D%2D[hasTransparentBounds]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hasTransparentBounds%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-hitEnd-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#hitEnd%2D%2D[hitEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#hitEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-lookingAt-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#lookingAt%2D%2D[lookingAt]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#lookingAt%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-matches-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#matches%2D%2D[matches]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#matches%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-namedGroup-1]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#namedGroup%2Djava.util.regex.Matcher%2Djava.lang.String%2D[namedGroup](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-Matcher-pattern-0]]<<painless-api-reference-Pattern,Pattern>> link:{java8-javadoc}/java/util/regex/Matcher.html#pattern%2D%2D[pattern]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#pattern%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-region-2]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#region%2Dint%2Dint%2D[region](int, int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#region%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-regionEnd-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#regionEnd%2D%2D[regionEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#regionEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-regionStart-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#regionStart%2D%2D[regionStart]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#regionStart%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-replaceAll-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#replaceAll%2Djava.lang.String%2D[replaceAll](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#replaceAll%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-replaceFirst-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Matcher.html#replaceFirst%2Djava.lang.String%2D[replaceFirst](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#replaceFirst%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Matcher-requireEnd-0]]boolean link:{java8-javadoc}/java/util/regex/Matcher.html#requireEnd%2D%2D[requireEnd]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#requireEnd%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-reset-0]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#reset%2D%2D[reset]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#reset%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-start-0]]int link:{java8-javadoc}/java/util/regex/Matcher.html#start%2D%2D[start]()++ (link:{java9-javadoc}/java/util/regex/Matcher.html#start%2D%2D[java 9])
+* ++[[painless-api-reference-Matcher-start-1]]int link:{java8-javadoc}/java/util/regex/Matcher.html#start%2Dint%2D[start](int)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#start%2Dint%2D[java 9])
+* ++[[painless-api-reference-Matcher-useAnchoringBounds-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#useAnchoringBounds%2Dboolean%2D[useAnchoringBounds](boolean)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#useAnchoringBounds%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Matcher-usePattern-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#usePattern%2Djava.util.regex.Pattern%2D[usePattern](<<painless-api-reference-Pattern,Pattern>>)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#usePattern%2Djava.util.regex.Pattern%2D[java 9])
+* ++[[painless-api-reference-Matcher-useTransparentBounds-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Matcher.html#useTransparentBounds%2Dboolean%2D[useTransparentBounds](boolean)++ (link:{java9-javadoc}/java/util/regex/Matcher.html#useTransparentBounds%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Math.asciidoc b/docs/reference/painless-api-reference/Math.asciidoc
new file mode 100644
index 00000000000..4b8322c15fc
--- /dev/null
+++ b/docs/reference/painless-api-reference/Math.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Math]]++Math++::
+** [[painless-api-reference-Math-E]]static double link:{java8-javadoc}/java/lang/Math.html#E[E] (link:{java9-javadoc}/java/lang/Math.html#E[java 9])
+** [[painless-api-reference-Math-PI]]static double link:{java8-javadoc}/java/lang/Math.html#PI[PI] (link:{java9-javadoc}/java/lang/Math.html#PI[java 9])
+* ++[[painless-api-reference-Math-IEEEremainder-2]]static double link:{java8-javadoc}/java/lang/Math.html#IEEEremainder%2Ddouble%2Ddouble%2D[IEEEremainder](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#IEEEremainder%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-abs-1]]static double link:{java8-javadoc}/java/lang/Math.html#abs%2Ddouble%2D[abs](double)++ (link:{java9-javadoc}/java/lang/Math.html#abs%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-acos-1]]static double link:{java8-javadoc}/java/lang/Math.html#acos%2Ddouble%2D[acos](double)++ (link:{java9-javadoc}/java/lang/Math.html#acos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-asin-1]]static double link:{java8-javadoc}/java/lang/Math.html#asin%2Ddouble%2D[asin](double)++ (link:{java9-javadoc}/java/lang/Math.html#asin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-atan-1]]static double link:{java8-javadoc}/java/lang/Math.html#atan%2Ddouble%2D[atan](double)++ (link:{java9-javadoc}/java/lang/Math.html#atan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-atan2-2]]static double link:{java8-javadoc}/java/lang/Math.html#atan2%2Ddouble%2Ddouble%2D[atan2](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#atan2%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cbrt-1]]static double link:{java8-javadoc}/java/lang/Math.html#cbrt%2Ddouble%2D[cbrt](double)++ (link:{java9-javadoc}/java/lang/Math.html#cbrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-ceil-1]]static double link:{java8-javadoc}/java/lang/Math.html#ceil%2Ddouble%2D[ceil](double)++ (link:{java9-javadoc}/java/lang/Math.html#ceil%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-copySign-2]]static double link:{java8-javadoc}/java/lang/Math.html#copySign%2Ddouble%2Ddouble%2D[copySign](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#copySign%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cos-1]]static double link:{java8-javadoc}/java/lang/Math.html#cos%2Ddouble%2D[cos](double)++ (link:{java9-javadoc}/java/lang/Math.html#cos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-cosh-1]]static double link:{java8-javadoc}/java/lang/Math.html#cosh%2Ddouble%2D[cosh](double)++ (link:{java9-javadoc}/java/lang/Math.html#cosh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-exp-1]]static double link:{java8-javadoc}/java/lang/Math.html#exp%2Ddouble%2D[exp](double)++ (link:{java9-javadoc}/java/lang/Math.html#exp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-expm1-1]]static double link:{java8-javadoc}/java/lang/Math.html#expm1%2Ddouble%2D[expm1](double)++ (link:{java9-javadoc}/java/lang/Math.html#expm1%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-floor-1]]static double link:{java8-javadoc}/java/lang/Math.html#floor%2Ddouble%2D[floor](double)++ (link:{java9-javadoc}/java/lang/Math.html#floor%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-hypot-2]]static double link:{java8-javadoc}/java/lang/Math.html#hypot%2Ddouble%2Ddouble%2D[hypot](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#hypot%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log-1]]static double link:{java8-javadoc}/java/lang/Math.html#log%2Ddouble%2D[log](double)++ (link:{java9-javadoc}/java/lang/Math.html#log%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log10-1]]static double link:{java8-javadoc}/java/lang/Math.html#log10%2Ddouble%2D[log10](double)++ (link:{java9-javadoc}/java/lang/Math.html#log10%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-log1p-1]]static double link:{java8-javadoc}/java/lang/Math.html#log1p%2Ddouble%2D[log1p](double)++ (link:{java9-javadoc}/java/lang/Math.html#log1p%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-max-2]]static double link:{java8-javadoc}/java/lang/Math.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-min-2]]static double link:{java8-javadoc}/java/lang/Math.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextAfter-2]]static double link:{java8-javadoc}/java/lang/Math.html#nextAfter%2Ddouble%2Ddouble%2D[nextAfter](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#nextAfter%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextDown-1]]static double link:{java8-javadoc}/java/lang/Math.html#nextDown%2Ddouble%2D[nextDown](double)++ (link:{java9-javadoc}/java/lang/Math.html#nextDown%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-nextUp-1]]static double link:{java8-javadoc}/java/lang/Math.html#nextUp%2Ddouble%2D[nextUp](double)++ (link:{java9-javadoc}/java/lang/Math.html#nextUp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-pow-2]]static double link:{java8-javadoc}/java/lang/Math.html#pow%2Ddouble%2Ddouble%2D[pow](double, double)++ (link:{java9-javadoc}/java/lang/Math.html#pow%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-random-0]]static double link:{java8-javadoc}/java/lang/Math.html#random%2D%2D[random]()++ (link:{java9-javadoc}/java/lang/Math.html#random%2D%2D[java 9])
+* ++[[painless-api-reference-Math-rint-1]]static double link:{java8-javadoc}/java/lang/Math.html#rint%2Ddouble%2D[rint](double)++ (link:{java9-javadoc}/java/lang/Math.html#rint%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-round-1]]static long link:{java8-javadoc}/java/lang/Math.html#round%2Ddouble%2D[round](double)++ (link:{java9-javadoc}/java/lang/Math.html#round%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-scalb-2]]static double link:{java8-javadoc}/java/lang/Math.html#scalb%2Ddouble%2Dint%2D[scalb](double, int)++ (link:{java9-javadoc}/java/lang/Math.html#scalb%2Ddouble%2Dint%2D[java 9])
+* ++[[painless-api-reference-Math-signum-1]]static double link:{java8-javadoc}/java/lang/Math.html#signum%2Ddouble%2D[signum](double)++ (link:{java9-javadoc}/java/lang/Math.html#signum%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sin-1]]static double link:{java8-javadoc}/java/lang/Math.html#sin%2Ddouble%2D[sin](double)++ (link:{java9-javadoc}/java/lang/Math.html#sin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sinh-1]]static double link:{java8-javadoc}/java/lang/Math.html#sinh%2Ddouble%2D[sinh](double)++ (link:{java9-javadoc}/java/lang/Math.html#sinh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-sqrt-1]]static double link:{java8-javadoc}/java/lang/Math.html#sqrt%2Ddouble%2D[sqrt](double)++ (link:{java9-javadoc}/java/lang/Math.html#sqrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-tan-1]]static double link:{java8-javadoc}/java/lang/Math.html#tan%2Ddouble%2D[tan](double)++ (link:{java9-javadoc}/java/lang/Math.html#tan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-tanh-1]]static double link:{java8-javadoc}/java/lang/Math.html#tanh%2Ddouble%2D[tanh](double)++ (link:{java9-javadoc}/java/lang/Math.html#tanh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-toDegrees-1]]static double link:{java8-javadoc}/java/lang/Math.html#toDegrees%2Ddouble%2D[toDegrees](double)++ (link:{java9-javadoc}/java/lang/Math.html#toDegrees%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-toRadians-1]]static double link:{java8-javadoc}/java/lang/Math.html#toRadians%2Ddouble%2D[toRadians](double)++ (link:{java9-javadoc}/java/lang/Math.html#toRadians%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Math-ulp-1]]static double link:{java8-javadoc}/java/lang/Math.html#ulp%2Ddouble%2D[ulp](double)++ (link:{java9-javadoc}/java/lang/Math.html#ulp%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MathContext.asciidoc b/docs/reference/painless-api-reference/MathContext.asciidoc
new file mode 100644
index 00000000000..c8167367fd5
--- /dev/null
+++ b/docs/reference/painless-api-reference/MathContext.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MathContext]]++MathContext++::
+** [[painless-api-reference-MathContext-DECIMAL128]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL128[DECIMAL128] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL128[java 9])
+** [[painless-api-reference-MathContext-DECIMAL32]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL32[DECIMAL32] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL32[java 9])
+** [[painless-api-reference-MathContext-DECIMAL64]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#DECIMAL64[DECIMAL64] (link:{java9-javadoc}/java/math/MathContext.html#DECIMAL64[java 9])
+** [[painless-api-reference-MathContext-UNLIMITED]]static <<painless-api-reference-MathContext,MathContext>> link:{java8-javadoc}/java/math/MathContext.html#UNLIMITED[UNLIMITED] (link:{java9-javadoc}/java/math/MathContext.html#UNLIMITED[java 9])
+* ++[[painless-api-reference-MathContext-MathContext-1]]link:{java8-javadoc}/java/math/MathContext.html#MathContext%2Dint%2D[MathContext](int)++ (link:{java9-javadoc}/java/math/MathContext.html#MathContext%2Dint%2D[java 9])
+* ++[[painless-api-reference-MathContext-MathContext-2]]link:{java8-javadoc}/java/math/MathContext.html#MathContext%2Dint%2Djava.math.RoundingMode%2D[MathContext](int, <<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/math/MathContext.html#MathContext%2Dint%2Djava.math.RoundingMode%2D[java 9])
+* ++[[painless-api-reference-MathContext-getPrecision-0]]int link:{java8-javadoc}/java/math/MathContext.html#getPrecision%2D%2D[getPrecision]()++ (link:{java9-javadoc}/java/math/MathContext.html#getPrecision%2D%2D[java 9])
+* ++[[painless-api-reference-MathContext-getRoundingMode-0]]<<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/MathContext.html#getRoundingMode%2D%2D[getRoundingMode]()++ (link:{java9-javadoc}/java/math/MathContext.html#getRoundingMode%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MessageFormat.Field.asciidoc b/docs/reference/painless-api-reference/MessageFormat.Field.asciidoc
new file mode 100644
index 00000000000..6b0e71e0a5f
--- /dev/null
+++ b/docs/reference/painless-api-reference/MessageFormat.Field.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MessageFormat-Field]]++MessageFormat.Field++::
+** [[painless-api-reference-MessageFormat-Field-ARGUMENT]]static <<painless-api-reference-MessageFormat-Field,MessageFormat.Field>> link:{java8-javadoc}/java/text/MessageFormat$Field.html#ARGUMENT[ARGUMENT] (link:{java9-javadoc}/java/text/MessageFormat$Field.html#ARGUMENT[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MessageFormat.asciidoc b/docs/reference/painless-api-reference/MessageFormat.asciidoc
new file mode 100644
index 00000000000..eaed777d9f0
--- /dev/null
+++ b/docs/reference/painless-api-reference/MessageFormat.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MessageFormat]]++MessageFormat++::
+* ++[[painless-api-reference-MessageFormat-format-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/MessageFormat.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, <<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getFormats-0]]<<painless-api-reference-Format,Format>>[] link:{java8-javadoc}/java/text/MessageFormat.html#getFormats%2D%2D[getFormats]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getFormats%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getFormatsByArgumentIndex-0]]<<painless-api-reference-Format,Format>>[] link:{java8-javadoc}/java/text/MessageFormat.html#getFormatsByArgumentIndex%2D%2D[getFormatsByArgumentIndex]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getFormatsByArgumentIndex%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-getLocale-0]]<<painless-api-reference-Locale,Locale>> link:{java8-javadoc}/java/text/MessageFormat.html#getLocale%2D%2D[getLocale]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#getLocale%2D%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-parse-1]]<<painless-api-reference-Object,Object>>[] link:{java8-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-parse-2]]<<painless-api-reference-Object,Object>>[] link:{java8-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormat-2]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormat%2Dint%2Djava.text.Format%2D[setFormat](int, <<painless-api-reference-Format,Format>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormat%2Dint%2Djava.text.Format%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormatByArgumentIndex-2]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormatByArgumentIndex%2Dint%2Djava.text.Format%2D[setFormatByArgumentIndex](int, <<painless-api-reference-Format,Format>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormatByArgumentIndex%2Dint%2Djava.text.Format%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormats-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormats%2Djava.text.Format:A%2D[setFormats](<<painless-api-reference-Format,Format>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormats%2Djava.text.Format:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setFormatsByArgumentIndex-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setFormatsByArgumentIndex%2Djava.text.Format:A%2D[setFormatsByArgumentIndex](<<painless-api-reference-Format,Format>>[])++ (link:{java9-javadoc}/java/text/MessageFormat.html#setFormatsByArgumentIndex%2Djava.text.Format:A%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-setLocale-1]]void link:{java8-javadoc}/java/text/MessageFormat.html#setLocale%2Djava.util.Locale%2D[setLocale](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/MessageFormat.html#setLocale%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-MessageFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/MessageFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/MessageFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MinguoChronology.asciidoc b/docs/reference/painless-api-reference/MinguoChronology.asciidoc
new file mode 100644
index 00000000000..9e9afb2cedf
--- /dev/null
+++ b/docs/reference/painless-api-reference/MinguoChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoChronology]]++MinguoChronology++::
+** [[painless-api-reference-MinguoChronology-INSTANCE]]static <<painless-api-reference-MinguoChronology,MinguoChronology>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-3]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-date-4]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateEpochDay-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateYearDay-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-dateYearDay-3]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-eraOf-1]]<<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoChronology-resolveDate-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/MinguoDate.asciidoc b/docs/reference/painless-api-reference/MinguoDate.asciidoc
new file mode 100644
index 00000000000..38d0b87da76
--- /dev/null
+++ b/docs/reference/painless-api-reference/MinguoDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoDate]]++MinguoDate++::
+* ++[[painless-api-reference-MinguoDate-from-1]]static <<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-of-3]]static <<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-getChronology-0]]<<painless-api-reference-MinguoChronology,MinguoChronology>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-getEra-0]]<<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-minus-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-minus-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-plus-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-plus-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-with-1]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-MinguoDate-with-2]]<<painless-api-reference-MinguoDate,MinguoDate>> link:{java8-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/MinguoDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/MinguoEra.asciidoc b/docs/reference/painless-api-reference/MinguoEra.asciidoc
new file mode 100644
index 00000000000..0b32509913d
--- /dev/null
+++ b/docs/reference/painless-api-reference/MinguoEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MinguoEra]]++MinguoEra++::
+** [[painless-api-reference-MinguoEra-BEFORE_ROC]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#BEFORE_ROC[BEFORE_ROC] (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#BEFORE_ROC[java 9])
+** [[painless-api-reference-MinguoEra-ROC]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#ROC[ROC] (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#ROC[java 9])
+* ++[[painless-api-reference-MinguoEra-of-1]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-valueOf-1]]static <<painless-api-reference-MinguoEra,MinguoEra>> link:{java8-javadoc}/java/time/chrono/MinguoEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-values-0]]static <<painless-api-reference-MinguoEra,MinguoEra>>[] link:{java8-javadoc}/java/time/chrono/MinguoEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-MinguoEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/MinguoEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/MinguoEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/MissingFormatArgumentException.asciidoc b/docs/reference/painless-api-reference/MissingFormatArgumentException.asciidoc
new file mode 100644
index 00000000000..7c09ad6ba5c
--- /dev/null
+++ b/docs/reference/painless-api-reference/MissingFormatArgumentException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingFormatArgumentException]]++MissingFormatArgumentException++::
+* ++[[painless-api-reference-MissingFormatArgumentException-MissingFormatArgumentException-1]]link:{java8-javadoc}/java/util/MissingFormatArgumentException.html#MissingFormatArgumentException%2Djava.lang.String%2D[MissingFormatArgumentException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingFormatArgumentException.html#MissingFormatArgumentException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingFormatArgumentException-getFormatSpecifier-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingFormatArgumentException.html#getFormatSpecifier%2D%2D[getFormatSpecifier]()++ (link:{java9-javadoc}/java/util/MissingFormatArgumentException.html#getFormatSpecifier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MissingFormatWidthException.asciidoc b/docs/reference/painless-api-reference/MissingFormatWidthException.asciidoc
new file mode 100644
index 00000000000..155637b48dd
--- /dev/null
+++ b/docs/reference/painless-api-reference/MissingFormatWidthException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingFormatWidthException]]++MissingFormatWidthException++::
+* ++[[painless-api-reference-MissingFormatWidthException-MissingFormatWidthException-1]]link:{java8-javadoc}/java/util/MissingFormatWidthException.html#MissingFormatWidthException%2Djava.lang.String%2D[MissingFormatWidthException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingFormatWidthException.html#MissingFormatWidthException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingFormatWidthException-getFormatSpecifier-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingFormatWidthException.html#getFormatSpecifier%2D%2D[getFormatSpecifier]()++ (link:{java9-javadoc}/java/util/MissingFormatWidthException.html#getFormatSpecifier%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/MissingResourceException.asciidoc b/docs/reference/painless-api-reference/MissingResourceException.asciidoc
new file mode 100644
index 00000000000..a415708a728
--- /dev/null
+++ b/docs/reference/painless-api-reference/MissingResourceException.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MissingResourceException]]++MissingResourceException++::
+* ++[[painless-api-reference-MissingResourceException-MissingResourceException-3]]link:{java8-javadoc}/java/util/MissingResourceException.html#MissingResourceException%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[MissingResourceException](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/MissingResourceException.html#MissingResourceException%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-MissingResourceException-getClassName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingResourceException.html#getClassName%2D%2D[getClassName]()++ (link:{java9-javadoc}/java/util/MissingResourceException.html#getClassName%2D%2D[java 9])
+* ++[[painless-api-reference-MissingResourceException-getKey-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/MissingResourceException.html#getKey%2D%2D[getKey]()++ (link:{java9-javadoc}/java/util/MissingResourceException.html#getKey%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Month.asciidoc b/docs/reference/painless-api-reference/Month.asciidoc
new file mode 100644
index 00000000000..f4eaa9cc129
--- /dev/null
+++ b/docs/reference/painless-api-reference/Month.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Month]]++Month++::
+** [[painless-api-reference-Month-APRIL]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#APRIL[APRIL] (link:{java9-javadoc}/java/time/Month.html#APRIL[java 9])
+** [[painless-api-reference-Month-AUGUST]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#AUGUST[AUGUST] (link:{java9-javadoc}/java/time/Month.html#AUGUST[java 9])
+** [[painless-api-reference-Month-DECEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#DECEMBER[DECEMBER] (link:{java9-javadoc}/java/time/Month.html#DECEMBER[java 9])
+** [[painless-api-reference-Month-FEBRUARY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#FEBRUARY[FEBRUARY] (link:{java9-javadoc}/java/time/Month.html#FEBRUARY[java 9])
+** [[painless-api-reference-Month-JANUARY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JANUARY[JANUARY] (link:{java9-javadoc}/java/time/Month.html#JANUARY[java 9])
+** [[painless-api-reference-Month-JULY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JULY[JULY] (link:{java9-javadoc}/java/time/Month.html#JULY[java 9])
+** [[painless-api-reference-Month-JUNE]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#JUNE[JUNE] (link:{java9-javadoc}/java/time/Month.html#JUNE[java 9])
+** [[painless-api-reference-Month-MARCH]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#MARCH[MARCH] (link:{java9-javadoc}/java/time/Month.html#MARCH[java 9])
+** [[painless-api-reference-Month-MAY]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#MAY[MAY] (link:{java9-javadoc}/java/time/Month.html#MAY[java 9])
+** [[painless-api-reference-Month-NOVEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#NOVEMBER[NOVEMBER] (link:{java9-javadoc}/java/time/Month.html#NOVEMBER[java 9])
+** [[painless-api-reference-Month-OCTOBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#OCTOBER[OCTOBER] (link:{java9-javadoc}/java/time/Month.html#OCTOBER[java 9])
+** [[painless-api-reference-Month-SEPTEMBER]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#SEPTEMBER[SEPTEMBER] (link:{java9-javadoc}/java/time/Month.html#SEPTEMBER[java 9])
+* ++[[painless-api-reference-Month-from-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Month.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Month-of-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/Month.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Month-valueOf-1]]static <<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/Month.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Month-values-0]]static <<painless-api-reference-Month,Month>>[] link:{java8-javadoc}/java/time/Month.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/Month.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-Month-firstDayOfYear-1]]int link:{java8-javadoc}/java/time/Month.html#firstDayOfYear%2Dboolean%2D[firstDayOfYear](boolean)++ (link:{java9-javadoc}/java/time/Month.html#firstDayOfYear%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Month-firstMonthOfQuarter-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#firstMonthOfQuarter%2D%2D[firstMonthOfQuarter]()++ (link:{java9-javadoc}/java/time/Month.html#firstMonthOfQuarter%2D%2D[java 9])
+* ++[[painless-api-reference-Month-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/Month.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/Month.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-Month-getValue-0]]int link:{java8-javadoc}/java/time/Month.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/Month.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Month-length-1]]int link:{java8-javadoc}/java/time/Month.html#length%2Dboolean%2D[length](boolean)++ (link:{java9-javadoc}/java/time/Month.html#length%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-Month-maxLength-0]]int link:{java8-javadoc}/java/time/Month.html#maxLength%2D%2D[maxLength]()++ (link:{java9-javadoc}/java/time/Month.html#maxLength%2D%2D[java 9])
+* ++[[painless-api-reference-Month-minLength-0]]int link:{java8-javadoc}/java/time/Month.html#minLength%2D%2D[minLength]()++ (link:{java9-javadoc}/java/time/Month.html#minLength%2D%2D[java 9])
+* ++[[painless-api-reference-Month-minus-1]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#minus%2Dlong%2D[minus](long)++ (link:{java9-javadoc}/java/time/Month.html#minus%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Month-plus-1]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/Month.html#plus%2Dlong%2D[plus](long)++ (link:{java9-javadoc}/java/time/Month.html#plus%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/MonthDay.asciidoc b/docs/reference/painless-api-reference/MonthDay.asciidoc
new file mode 100644
index 00000000000..4bb7c675d70
--- /dev/null
+++ b/docs/reference/painless-api-reference/MonthDay.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-MonthDay]]++MonthDay++::
+* ++[[painless-api-reference-MonthDay-from-1]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-MonthDay-of-2]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/MonthDay.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-parse-1]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-MonthDay-parse-2]]static <<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-MonthDay-atYear-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/MonthDay.html#atYear%2Dint%2D[atYear](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#atYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-compareTo-1]]int link:{java8-javadoc}/java/time/MonthDay.html#compareTo%2Djava.time.MonthDay%2D[compareTo](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#compareTo%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/MonthDay.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/MonthDay.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/MonthDay.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-getMonthValue-0]]int link:{java8-javadoc}/java/time/MonthDay.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/MonthDay.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isAfter-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isAfter%2Djava.time.MonthDay%2D[isAfter](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#isAfter%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isBefore-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isBefore%2Djava.time.MonthDay%2D[isBefore](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#isBefore%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-MonthDay-isValidYear-1]]boolean link:{java8-javadoc}/java/time/MonthDay.html#isValidYear%2Dint%2D[isValidYear](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#isValidYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-with-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#with%2Djava.time.Month%2D[with](<<painless-api-reference-Month,Month>>)++ (link:{java9-javadoc}/java/time/MonthDay.html#with%2Djava.time.Month%2D[java 9])
+* ++[[painless-api-reference-MonthDay-withDayOfMonth-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-MonthDay-withMonth-1]]<<painless-api-reference-MonthDay,MonthDay>> link:{java8-javadoc}/java/time/MonthDay.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/MonthDay.html#withMonth%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/NavigableMap.asciidoc b/docs/reference/painless-api-reference/NavigableMap.asciidoc
new file mode 100644
index 00000000000..507ab8acd61
--- /dev/null
+++ b/docs/reference/painless-api-reference/NavigableMap.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NavigableMap]]++NavigableMap++::
+* ++[[painless-api-reference-NavigableMap-ceilingEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#ceilingEntry%2Djava.lang.Object%2D[ceilingEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#ceilingEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-ceilingKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#ceilingKey%2Djava.lang.Object%2D[ceilingKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#ceilingKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-descendingKeySet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableMap.html#descendingKeySet%2D%2D[descendingKeySet]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#descendingKeySet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-descendingMap-0]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#descendingMap%2D%2D[descendingMap]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#descendingMap%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-firstEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#firstEntry%2D%2D[firstEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#firstEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-floorEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#floorEntry%2Djava.lang.Object%2D[floorEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#floorEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-floorKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#floorKey%2Djava.lang.Object%2D[floorKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#floorKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-headMap-2]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#headMap%2Djava.lang.Object%2Dboolean%2D[headMap](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#headMap%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-higherEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#higherEntry%2Djava.lang.Object%2D[higherEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#higherEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-higherKey-1]]def link:{java8-javadoc}/java/util/NavigableMap.html#higherKey%2Djava.lang.Object%2D[higherKey](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#higherKey%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-lastEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#lastEntry%2D%2D[lastEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#lastEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-lowerEntry-1]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#lowerEntry%2Djava.lang.Object%2D[lowerEntry](def)++ (link:{java9-javadoc}/java/util/NavigableMap.html#lowerEntry%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-navigableKeySet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableMap.html#navigableKeySet%2D%2D[navigableKeySet]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#navigableKeySet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-pollFirstEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#pollFirstEntry%2D%2D[pollFirstEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#pollFirstEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-pollLastEntry-0]]<<painless-api-reference-Map-Entry,Map.Entry>> link:{java8-javadoc}/java/util/NavigableMap.html#pollLastEntry%2D%2D[pollLastEntry]()++ (link:{java9-javadoc}/java/util/NavigableMap.html#pollLastEntry%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-subMap-4]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#subMap%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[subMap](def, boolean, def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#subMap%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableMap-tailMap-2]]<<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/util/NavigableMap.html#tailMap%2Djava.lang.Object%2Dboolean%2D[tailMap](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableMap.html#tailMap%2Djava.lang.Object%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-SortedMap,SortedMap>>++
diff --git a/docs/reference/painless-api-reference/NavigableSet.asciidoc b/docs/reference/painless-api-reference/NavigableSet.asciidoc
new file mode 100644
index 00000000000..15fff25f44d
--- /dev/null
+++ b/docs/reference/painless-api-reference/NavigableSet.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NavigableSet]]++NavigableSet++::
+* ++[[painless-api-reference-NavigableSet-ceiling-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#ceiling%2Djava.lang.Object%2D[ceiling](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#ceiling%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-descendingIterator-0]]<<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/NavigableSet.html#descendingIterator%2D%2D[descendingIterator]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#descendingIterator%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-descendingSet-0]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#descendingSet%2D%2D[descendingSet]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#descendingSet%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-floor-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#floor%2Djava.lang.Object%2D[floor](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#floor%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-headSet-2]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#headSet%2Djava.lang.Object%2Dboolean%2D[headSet](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#headSet%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-higher-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#higher%2Djava.lang.Object%2D[higher](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#higher%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-lower-1]]def link:{java8-javadoc}/java/util/NavigableSet.html#lower%2Djava.lang.Object%2D[lower](def)++ (link:{java9-javadoc}/java/util/NavigableSet.html#lower%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-pollFirst-0]]def link:{java8-javadoc}/java/util/NavigableSet.html#pollFirst%2D%2D[pollFirst]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#pollFirst%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-pollLast-0]]def link:{java8-javadoc}/java/util/NavigableSet.html#pollLast%2D%2D[pollLast]()++ (link:{java9-javadoc}/java/util/NavigableSet.html#pollLast%2D%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-subSet-4]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#subSet%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[subSet](def, boolean, def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#subSet%2Djava.lang.Object%2Dboolean%2Djava.lang.Object%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NavigableSet-tailSet-2]]<<painless-api-reference-NavigableSet,NavigableSet>> link:{java8-javadoc}/java/util/NavigableSet.html#tailSet%2Djava.lang.Object%2Dboolean%2D[tailSet](def, boolean)++ (link:{java9-javadoc}/java/util/NavigableSet.html#tailSet%2Djava.lang.Object%2Dboolean%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++, ++<<painless-api-reference-SortedSet,SortedSet>>++
diff --git a/docs/reference/painless-api-reference/NegativeArraySizeException.asciidoc b/docs/reference/painless-api-reference/NegativeArraySizeException.asciidoc
new file mode 100644
index 00000000000..78ee0006eca
--- /dev/null
+++ b/docs/reference/painless-api-reference/NegativeArraySizeException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NegativeArraySizeException]]++NegativeArraySizeException++::
+* ++[[painless-api-reference-NegativeArraySizeException-NegativeArraySizeException-0]]link:{java8-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2D%2D[NegativeArraySizeException]()++ (link:{java9-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2D%2D[java 9])
+* ++[[painless-api-reference-NegativeArraySizeException-NegativeArraySizeException-1]]link:{java8-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2Djava.lang.String%2D[NegativeArraySizeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NegativeArraySizeException.html#NegativeArraySizeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NoSuchElementException.asciidoc b/docs/reference/painless-api-reference/NoSuchElementException.asciidoc
new file mode 100644
index 00000000000..7a28bdfb5dd
--- /dev/null
+++ b/docs/reference/painless-api-reference/NoSuchElementException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchElementException]]++NoSuchElementException++::
+* ++[[painless-api-reference-NoSuchElementException-NoSuchElementException-0]]link:{java8-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2D%2D[NoSuchElementException]()++ (link:{java9-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchElementException-NoSuchElementException-1]]link:{java8-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2Djava.lang.String%2D[NoSuchElementException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/NoSuchElementException.html#NoSuchElementException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NoSuchFieldException.asciidoc b/docs/reference/painless-api-reference/NoSuchFieldException.asciidoc
new file mode 100644
index 00000000000..6f3e12d16d7
--- /dev/null
+++ b/docs/reference/painless-api-reference/NoSuchFieldException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchFieldException]]++NoSuchFieldException++::
+* ++[[painless-api-reference-NoSuchFieldException-NoSuchFieldException-0]]link:{java8-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2D%2D[NoSuchFieldException]()++ (link:{java9-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchFieldException-NoSuchFieldException-1]]link:{java8-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2Djava.lang.String%2D[NoSuchFieldException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NoSuchFieldException.html#NoSuchFieldException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NoSuchMethodException.asciidoc b/docs/reference/painless-api-reference/NoSuchMethodException.asciidoc
new file mode 100644
index 00000000000..0fec067e9d2
--- /dev/null
+++ b/docs/reference/painless-api-reference/NoSuchMethodException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NoSuchMethodException]]++NoSuchMethodException++::
+* ++[[painless-api-reference-NoSuchMethodException-NoSuchMethodException-0]]link:{java8-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2D%2D[NoSuchMethodException]()++ (link:{java9-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2D%2D[java 9])
+* ++[[painless-api-reference-NoSuchMethodException-NoSuchMethodException-1]]link:{java8-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2Djava.lang.String%2D[NoSuchMethodException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NoSuchMethodException.html#NoSuchMethodException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Normalizer.Form.asciidoc b/docs/reference/painless-api-reference/Normalizer.Form.asciidoc
new file mode 100644
index 00000000000..f74d4d243db
--- /dev/null
+++ b/docs/reference/painless-api-reference/Normalizer.Form.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Normalizer-Form]]++Normalizer.Form++::
+** [[painless-api-reference-Normalizer-Form-NFC]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer$Form.html#NFC[NFC] (link:{java9-javadoc}/java/text/Normalizer$Form.html#NFC[java 9])
+** [[painless-api-reference-Normalizer-Form-NFD]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer$Form.html#NFD[NFD] (link:{java9-javadoc}/java/text/Normalizer$Form.html#NFD[java 9])
+** [[painless-api-reference-Normalizer-Form-NFKC]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer$Form.html#NFKC[NFKC] (link:{java9-javadoc}/java/text/Normalizer$Form.html#NFKC[java 9])
+** [[painless-api-reference-Normalizer-Form-NFKD]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer$Form.html#NFKD[NFKD] (link:{java9-javadoc}/java/text/Normalizer$Form.html#NFKD[java 9])
+* ++[[painless-api-reference-Normalizer-Form-valueOf-1]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>> link:{java8-javadoc}/java/text/Normalizer$Form.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/Normalizer$Form.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Normalizer-Form-values-0]]static <<painless-api-reference-Normalizer-Form,Normalizer.Form>>[] link:{java8-javadoc}/java/text/Normalizer$Form.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/text/Normalizer$Form.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Normalizer.asciidoc b/docs/reference/painless-api-reference/Normalizer.asciidoc
new file mode 100644
index 00000000000..ecfe03c7fb4
--- /dev/null
+++ b/docs/reference/painless-api-reference/Normalizer.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Normalizer]]++Normalizer++::
+* ++[[painless-api-reference-Normalizer-isNormalized-2]]static boolean link:{java8-javadoc}/java/text/Normalizer.html#isNormalized%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[isNormalized](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Normalizer-Form,Normalizer.Form>>)++ (link:{java9-javadoc}/java/text/Normalizer.html#isNormalized%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[java 9])
+* ++[[painless-api-reference-Normalizer-normalize-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/Normalizer.html#normalize%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[normalize](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Normalizer-Form,Normalizer.Form>>)++ (link:{java9-javadoc}/java/text/Normalizer.html#normalize%2Djava.lang.CharSequence%2Djava.text.Normalizer$Form%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NullPointerException.asciidoc b/docs/reference/painless-api-reference/NullPointerException.asciidoc
new file mode 100644
index 00000000000..72795c73990
--- /dev/null
+++ b/docs/reference/painless-api-reference/NullPointerException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NullPointerException]]++NullPointerException++::
+* ++[[painless-api-reference-NullPointerException-NullPointerException-0]]link:{java8-javadoc}/java/lang/NullPointerException.html#NullPointerException%2D%2D[NullPointerException]()++ (link:{java9-javadoc}/java/lang/NullPointerException.html#NullPointerException%2D%2D[java 9])
+* ++[[painless-api-reference-NullPointerException-NullPointerException-1]]link:{java8-javadoc}/java/lang/NullPointerException.html#NullPointerException%2Djava.lang.String%2D[NullPointerException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NullPointerException.html#NullPointerException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Number.asciidoc b/docs/reference/painless-api-reference/Number.asciidoc
new file mode 100644
index 00000000000..a8c588f3370
--- /dev/null
+++ b/docs/reference/painless-api-reference/Number.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Number]]++Number++::
+* ++[[painless-api-reference-Number-byteValue-0]]byte link:{java8-javadoc}/java/lang/Number.html#byteValue%2D%2D[byteValue]()++ (link:{java9-javadoc}/java/lang/Number.html#byteValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-doubleValue-0]]double link:{java8-javadoc}/java/lang/Number.html#doubleValue%2D%2D[doubleValue]()++ (link:{java9-javadoc}/java/lang/Number.html#doubleValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-floatValue-0]]float link:{java8-javadoc}/java/lang/Number.html#floatValue%2D%2D[floatValue]()++ (link:{java9-javadoc}/java/lang/Number.html#floatValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-intValue-0]]int link:{java8-javadoc}/java/lang/Number.html#intValue%2D%2D[intValue]()++ (link:{java9-javadoc}/java/lang/Number.html#intValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-longValue-0]]long link:{java8-javadoc}/java/lang/Number.html#longValue%2D%2D[longValue]()++ (link:{java9-javadoc}/java/lang/Number.html#longValue%2D%2D[java 9])
+* ++[[painless-api-reference-Number-shortValue-0]]short link:{java8-javadoc}/java/lang/Number.html#shortValue%2D%2D[shortValue]()++ (link:{java9-javadoc}/java/lang/Number.html#shortValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NumberFormat.Field.asciidoc b/docs/reference/painless-api-reference/NumberFormat.Field.asciidoc
new file mode 100644
index 00000000000..aec4fc06357
--- /dev/null
+++ b/docs/reference/painless-api-reference/NumberFormat.Field.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormat-Field]]++NumberFormat.Field++::
+** [[painless-api-reference-NumberFormat-Field-CURRENCY]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#CURRENCY[CURRENCY] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#CURRENCY[java 9])
+** [[painless-api-reference-NumberFormat-Field-DECIMAL_SEPARATOR]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#DECIMAL_SEPARATOR[DECIMAL_SEPARATOR] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#DECIMAL_SEPARATOR[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#EXPONENT[EXPONENT] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#EXPONENT[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT_SIGN]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#EXPONENT_SIGN[EXPONENT_SIGN] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#EXPONENT_SIGN[java 9])
+** [[painless-api-reference-NumberFormat-Field-EXPONENT_SYMBOL]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#EXPONENT_SYMBOL[EXPONENT_SYMBOL] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#EXPONENT_SYMBOL[java 9])
+** [[painless-api-reference-NumberFormat-Field-FRACTION]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#FRACTION[FRACTION] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#FRACTION[java 9])
+** [[painless-api-reference-NumberFormat-Field-GROUPING_SEPARATOR]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#GROUPING_SEPARATOR[GROUPING_SEPARATOR] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#GROUPING_SEPARATOR[java 9])
+** [[painless-api-reference-NumberFormat-Field-INTEGER]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#INTEGER[INTEGER] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#INTEGER[java 9])
+** [[painless-api-reference-NumberFormat-Field-PERCENT]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#PERCENT[PERCENT] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#PERCENT[java 9])
+** [[painless-api-reference-NumberFormat-Field-PERMILLE]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#PERMILLE[PERMILLE] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#PERMILLE[java 9])
+** [[painless-api-reference-NumberFormat-Field-SIGN]]static <<painless-api-reference-NumberFormat-Field,NumberFormat.Field>> link:{java8-javadoc}/java/text/NumberFormat$Field.html#SIGN[SIGN] (link:{java9-javadoc}/java/text/NumberFormat$Field.html#SIGN[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NumberFormat.asciidoc b/docs/reference/painless-api-reference/NumberFormat.asciidoc
new file mode 100644
index 00000000000..ddca8ccd682
--- /dev/null
+++ b/docs/reference/painless-api-reference/NumberFormat.asciidoc
@@ -0,0 +1,38 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormat]]++NumberFormat++::
+** [[painless-api-reference-NumberFormat-FRACTION_FIELD]]static int link:{java8-javadoc}/java/text/NumberFormat.html#FRACTION_FIELD[FRACTION_FIELD] (link:{java9-javadoc}/java/text/NumberFormat.html#FRACTION_FIELD[java 9])
+** [[painless-api-reference-NumberFormat-INTEGER_FIELD]]static int link:{java8-javadoc}/java/text/NumberFormat.html#INTEGER_FIELD[INTEGER_FIELD] (link:{java9-javadoc}/java/text/NumberFormat.html#INTEGER_FIELD[java 9])
+* ++[[painless-api-reference-NumberFormat-getAvailableLocales-0]]static <<painless-api-reference-Locale,Locale>>[] link:{java8-javadoc}/java/text/NumberFormat.html#getAvailableLocales%2D%2D[getAvailableLocales]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getAvailableLocales%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrencyInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2D%2D[getCurrencyInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrencyInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2Djava.util.Locale%2D[getCurrencyInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrencyInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getInstance%2D%2D[getInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getInstance%2Djava.util.Locale%2D[getInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getIntegerInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2D%2D[getIntegerInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getIntegerInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2Djava.util.Locale%2D[getIntegerInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getIntegerInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getNumberInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getNumberInstance%2D%2D[getNumberInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getNumberInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getNumberInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getNumberInstance%2Djava.util.Locale%2D[getNumberInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getNumberInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getPercentInstance-0]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getPercentInstance%2D%2D[getPercentInstance]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getPercentInstance%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getPercentInstance-1]]static <<painless-api-reference-NumberFormat,NumberFormat>> link:{java8-javadoc}/java/text/NumberFormat.html#getPercentInstance%2Djava.util.Locale%2D[getPercentInstance](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#getPercentInstance%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getCurrency-0]]<<painless-api-reference-Currency,Currency>> link:{java8-javadoc}/java/text/NumberFormat.html#getCurrency%2D%2D[getCurrency]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getCurrency%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMaximumFractionDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMaximumFractionDigits%2D%2D[getMaximumFractionDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMaximumFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMaximumIntegerDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMaximumIntegerDigits%2D%2D[getMaximumIntegerDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMaximumIntegerDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMinimumFractionDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMinimumFractionDigits%2D%2D[getMinimumFractionDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMinimumFractionDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getMinimumIntegerDigits-0]]int link:{java8-javadoc}/java/text/NumberFormat.html#getMinimumIntegerDigits%2D%2D[getMinimumIntegerDigits]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getMinimumIntegerDigits%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-getRoundingMode-0]]<<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/text/NumberFormat.html#getRoundingMode%2D%2D[getRoundingMode]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#getRoundingMode%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-isGroupingUsed-0]]boolean link:{java8-javadoc}/java/text/NumberFormat.html#isGroupingUsed%2D%2D[isGroupingUsed]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#isGroupingUsed%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-isParseIntegerOnly-0]]boolean link:{java8-javadoc}/java/text/NumberFormat.html#isParseIntegerOnly%2D%2D[isParseIntegerOnly]()++ (link:{java9-javadoc}/java/text/NumberFormat.html#isParseIntegerOnly%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-parse-1]]<<painless-api-reference-Number,Number>> link:{java8-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2D[parse](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-parse-2]]<<painless-api-reference-Number,Number>> link:{java8-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[parse](<<painless-api-reference-String,String>>, <<painless-api-reference-ParsePosition,ParsePosition>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#parse%2Djava.lang.String%2Djava.text.ParsePosition%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setCurrency-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setCurrency%2Djava.util.Currency%2D[setCurrency](<<painless-api-reference-Currency,Currency>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setCurrency%2Djava.util.Currency%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setGroupingUsed-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setGroupingUsed%2Dboolean%2D[setGroupingUsed](boolean)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setGroupingUsed%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMaximumFractionDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMaximumFractionDigits%2Dint%2D[setMaximumFractionDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMaximumFractionDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMaximumIntegerDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMaximumIntegerDigits%2Dint%2D[setMaximumIntegerDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMaximumIntegerDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMinimumFractionDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMinimumFractionDigits%2Dint%2D[setMinimumFractionDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMinimumFractionDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setMinimumIntegerDigits-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setMinimumIntegerDigits%2Dint%2D[setMinimumIntegerDigits](int)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setMinimumIntegerDigits%2Dint%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setParseIntegerOnly-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setParseIntegerOnly%2Dboolean%2D[setParseIntegerOnly](boolean)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setParseIntegerOnly%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-NumberFormat-setRoundingMode-1]]void link:{java8-javadoc}/java/text/NumberFormat.html#setRoundingMode%2Djava.math.RoundingMode%2D[setRoundingMode](<<painless-api-reference-RoundingMode,RoundingMode>>)++ (link:{java9-javadoc}/java/text/NumberFormat.html#setRoundingMode%2Djava.math.RoundingMode%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/NumberFormatException.asciidoc b/docs/reference/painless-api-reference/NumberFormatException.asciidoc
new file mode 100644
index 00000000000..4066147650c
--- /dev/null
+++ b/docs/reference/painless-api-reference/NumberFormatException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-NumberFormatException]]++NumberFormatException++::
+* ++[[painless-api-reference-NumberFormatException-NumberFormatException-0]]link:{java8-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2D%2D[NumberFormatException]()++ (link:{java9-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2D%2D[java 9])
+* ++[[painless-api-reference-NumberFormatException-NumberFormatException-1]]link:{java8-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2Djava.lang.String%2D[NumberFormatException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/NumberFormatException.html#NumberFormatException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ObjDoubleConsumer.asciidoc b/docs/reference/painless-api-reference/ObjDoubleConsumer.asciidoc
new file mode 100644
index 00000000000..2fc41f9ce0f
--- /dev/null
+++ b/docs/reference/painless-api-reference/ObjDoubleConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjDoubleConsumer]]++ObjDoubleConsumer++::
+* ++[[painless-api-reference-ObjDoubleConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjDoubleConsumer.html#accept%2Djava.lang.Object%2Ddouble%2D[accept](def, double)++ (link:{java9-javadoc}/java/util/function/ObjDoubleConsumer.html#accept%2Djava.lang.Object%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ObjIntConsumer.asciidoc b/docs/reference/painless-api-reference/ObjIntConsumer.asciidoc
new file mode 100644
index 00000000000..8779b333df9
--- /dev/null
+++ b/docs/reference/painless-api-reference/ObjIntConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjIntConsumer]]++ObjIntConsumer++::
+* ++[[painless-api-reference-ObjIntConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjIntConsumer.html#accept%2Djava.lang.Object%2Dint%2D[accept](def, int)++ (link:{java9-javadoc}/java/util/function/ObjIntConsumer.html#accept%2Djava.lang.Object%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ObjLongConsumer.asciidoc b/docs/reference/painless-api-reference/ObjLongConsumer.asciidoc
new file mode 100644
index 00000000000..cdb0ed6546c
--- /dev/null
+++ b/docs/reference/painless-api-reference/ObjLongConsumer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ObjLongConsumer]]++ObjLongConsumer++::
+* ++[[painless-api-reference-ObjLongConsumer-accept-2]]void link:{java8-javadoc}/java/util/function/ObjLongConsumer.html#accept%2Djava.lang.Object%2Dlong%2D[accept](def, long)++ (link:{java9-javadoc}/java/util/function/ObjLongConsumer.html#accept%2Djava.lang.Object%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Object.asciidoc b/docs/reference/painless-api-reference/Object.asciidoc
new file mode 100644
index 00000000000..99c4ae5cffd
--- /dev/null
+++ b/docs/reference/painless-api-reference/Object.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Object]]++Object++::
+* ++[[painless-api-reference-Object-equals-1]]boolean link:{java8-javadoc}/java/lang/Object.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/lang/Object.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Object-hashCode-0]]int link:{java8-javadoc}/java/lang/Object.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/lang/Object.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Object-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Object.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/lang/Object.html#toString%2D%2D[java 9])
diff --git a/docs/reference/painless-api-reference/Objects.asciidoc b/docs/reference/painless-api-reference/Objects.asciidoc
new file mode 100644
index 00000000000..9fdbad2e5b7
--- /dev/null
+++ b/docs/reference/painless-api-reference/Objects.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Objects]]++Objects++::
+* ++[[painless-api-reference-Objects-compare-3]]static int link:{java8-javadoc}/java/util/Objects.html#compare%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.Comparator%2D[compare](def, def, <<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/Objects.html#compare%2Djava.lang.Object%2Djava.lang.Object%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Objects-deepEquals-2]]static boolean link:{java8-javadoc}/java/util/Objects.html#deepEquals%2Djava.lang.Object%2Djava.lang.Object%2D[deepEquals](<<painless-api-reference-Object,Object>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#deepEquals%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-equals-2]]static boolean link:{java8-javadoc}/java/util/Objects.html#equals%2Djava.lang.Object%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#equals%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-hash-1]]static int link:{java8-javadoc}/java/util/Objects.html#hash%2Djava.lang.Object:A%2D[hash](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Objects.html#hash%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Objects-hashCode-1]]static int link:{java8-javadoc}/java/util/Objects.html#hashCode%2Djava.lang.Object%2D[hashCode](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#hashCode%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-isNull-1]]static boolean link:{java8-javadoc}/java/util/Objects.html#isNull%2Djava.lang.Object%2D[isNull](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#isNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-nonNull-1]]static boolean link:{java8-javadoc}/java/util/Objects.html#nonNull%2Djava.lang.Object%2D[nonNull](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#nonNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-requireNonNull-1]]static def link:{java8-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2D[requireNonNull](def)++ (link:{java9-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-requireNonNull-2]]static def link:{java8-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2Djava.lang.String%2D[requireNonNull](def, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Objects.html#requireNonNull%2Djava.lang.Object%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Objects-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2D[toString](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Objects-toString-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2Djava.lang.String%2D[toString](<<painless-api-reference-Object,Object>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/Objects.html#toString%2Djava.lang.Object%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Observable.asciidoc b/docs/reference/painless-api-reference/Observable.asciidoc
new file mode 100644
index 00000000000..65d8fae45ae
--- /dev/null
+++ b/docs/reference/painless-api-reference/Observable.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Observable]]++Observable++::
+* ++[[painless-api-reference-Observable-Observable-0]]link:{java8-javadoc}/java/util/Observable.html#Observable%2D%2D[Observable]()++ (link:{java9-javadoc}/java/util/Observable.html#Observable%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-addObserver-1]]void link:{java8-javadoc}/java/util/Observable.html#addObserver%2Djava.util.Observer%2D[addObserver](<<painless-api-reference-Observer,Observer>>)++ (link:{java9-javadoc}/java/util/Observable.html#addObserver%2Djava.util.Observer%2D[java 9])
+* ++[[painless-api-reference-Observable-countObservers-0]]int link:{java8-javadoc}/java/util/Observable.html#countObservers%2D%2D[countObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#countObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-deleteObserver-1]]void link:{java8-javadoc}/java/util/Observable.html#deleteObserver%2Djava.util.Observer%2D[deleteObserver](<<painless-api-reference-Observer,Observer>>)++ (link:{java9-javadoc}/java/util/Observable.html#deleteObserver%2Djava.util.Observer%2D[java 9])
+* ++[[painless-api-reference-Observable-deleteObservers-0]]void link:{java8-javadoc}/java/util/Observable.html#deleteObservers%2D%2D[deleteObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#deleteObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-hasChanged-0]]boolean link:{java8-javadoc}/java/util/Observable.html#hasChanged%2D%2D[hasChanged]()++ (link:{java9-javadoc}/java/util/Observable.html#hasChanged%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-notifyObservers-0]]void link:{java8-javadoc}/java/util/Observable.html#notifyObservers%2D%2D[notifyObservers]()++ (link:{java9-javadoc}/java/util/Observable.html#notifyObservers%2D%2D[java 9])
+* ++[[painless-api-reference-Observable-notifyObservers-1]]void link:{java8-javadoc}/java/util/Observable.html#notifyObservers%2Djava.lang.Object%2D[notifyObservers](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Observable.html#notifyObservers%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Observer.asciidoc b/docs/reference/painless-api-reference/Observer.asciidoc
new file mode 100644
index 00000000000..9277663dd62
--- /dev/null
+++ b/docs/reference/painless-api-reference/Observer.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Observer]]++Observer++::
+* ++[[painless-api-reference-Observer-update-2]]void link:{java8-javadoc}/java/util/Observer.html#update%2Djava.util.Observable%2Djava.lang.Object%2D[update](<<painless-api-reference-Observable,Observable>>, <<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Observer.html#update%2Djava.util.Observable%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/OffsetDateTime.asciidoc b/docs/reference/painless-api-reference/OffsetDateTime.asciidoc
new file mode 100644
index 00000000000..d476c181c55
--- /dev/null
+++ b/docs/reference/painless-api-reference/OffsetDateTime.asciidoc
@@ -0,0 +1,75 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OffsetDateTime]]++OffsetDateTime++::
+** [[painless-api-reference-OffsetDateTime-MAX]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/OffsetDateTime.html#MAX[java 9])
+** [[painless-api-reference-OffsetDateTime-MIN]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/OffsetDateTime.html#MIN[java 9])
+* ++[[painless-api-reference-OffsetDateTime-from-1]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-3]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-of-8]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[of](int, int, int, int, int, int, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-ofInstant-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-parse-1]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-parse-2]]static <<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-timeLineOrder-0]]static <<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/time/OffsetDateTime.html#timeLineOrder%2D%2D[timeLineOrder]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#timeLineOrder%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-atZoneSameInstant-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#atZoneSameInstant%2Djava.time.ZoneId%2D[atZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#atZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-atZoneSimilarLocal-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#atZoneSimilarLocal%2Djava.time.ZoneId%2D[atZoneSimilarLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#atZoneSimilarLocal%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-compareTo-1]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#compareTo%2Djava.time.OffsetDateTime%2D[compareTo](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#compareTo%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/OffsetDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getHour-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getNano-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/OffsetDateTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-getYear-0]]int link:{java8-javadoc}/java/time/OffsetDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isAfter%2Djava.time.OffsetDateTime%2D[isAfter](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isAfter%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isBefore%2Djava.time.OffsetDateTime%2D[isBefore](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isBefore%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/OffsetDateTime.html#isEqual%2Djava.time.OffsetDateTime%2D[isEqual](<<painless-api-reference-OffsetDateTime,OffsetDateTime>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#isEqual%2Djava.time.OffsetDateTime%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minus-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minus-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusDays-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusHours-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusMinutes-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusMonths-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusNanos-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusSeconds-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusWeeks-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-minusYears-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plus-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plus-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusDays-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusHours-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusMinutes-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusMonths-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusNanos-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusSeconds-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusWeeks-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-plusYears-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toEpochSecond-0]]long link:{java8-javadoc}/java/time/OffsetDateTime.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toEpochSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toInstant%2D%2D[toInstant]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toInstant%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalDateTime-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toOffsetTime-0]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toOffsetTime%2D%2D[toOffsetTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toOffsetTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-toZonedDateTime-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#toZonedDateTime%2D%2D[toZonedDateTime]()++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#toZonedDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-truncatedTo-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-with-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-with-2]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withDayOfMonth-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withDayOfYear-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withHour-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withMinute-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withMonth-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withNano-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withOffsetSameInstant-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[withOffsetSameInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withOffsetSameLocal-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[withOffsetSameLocal](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withSecond-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetDateTime-withYear-1]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/OffsetDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/OffsetDateTime.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/OffsetTime.asciidoc b/docs/reference/painless-api-reference/OffsetTime.asciidoc
new file mode 100644
index 00000000000..b23e9bb9577
--- /dev/null
+++ b/docs/reference/painless-api-reference/OffsetTime.asciidoc
@@ -0,0 +1,47 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OffsetTime]]++OffsetTime++::
+** [[painless-api-reference-OffsetTime-MAX]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#MAX[MAX] (link:{java9-javadoc}/java/time/OffsetTime.html#MAX[java 9])
+** [[painless-api-reference-OffsetTime-MIN]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#MIN[MIN] (link:{java9-javadoc}/java/time/OffsetTime.html#MIN[java 9])
+* ++[[painless-api-reference-OffsetTime-from-1]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-of-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#of%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#of%2Djava.time.LocalTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-of-5]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#of%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[of](int, int, int, int, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#of%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-ofInstant-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-parse-1]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-parse-2]]static <<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-compareTo-1]]int link:{java8-javadoc}/java/time/OffsetTime.html#compareTo%2Djava.time.OffsetTime%2D[compareTo](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#compareTo%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/OffsetTime.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getHour-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getMinute-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getNano-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/OffsetTime.html#getOffset%2D%2D[getOffset]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getOffset%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-getSecond-0]]int link:{java8-javadoc}/java/time/OffsetTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isAfter-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isAfter%2Djava.time.OffsetTime%2D[isAfter](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isAfter%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isBefore-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isBefore%2Djava.time.OffsetTime%2D[isBefore](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isBefore%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-isEqual-1]]boolean link:{java8-javadoc}/java/time/OffsetTime.html#isEqual%2Djava.time.OffsetTime%2D[isEqual](<<painless-api-reference-OffsetTime,OffsetTime>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#isEqual%2Djava.time.OffsetTime%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minus-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minus-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusHours-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusMinutes-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusNanos-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-minusSeconds-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plus-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plus-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusHours-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusMinutes-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusNanos-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-plusSeconds-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-toLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/OffsetTime.html#toLocalTime%2D%2D[toLocalTime]()++ (link:{java9-javadoc}/java/time/OffsetTime.html#toLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-truncatedTo-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-with-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-with-2]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/OffsetTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withHour-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withMinute-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withNano-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withOffsetSameInstant-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[withOffsetSameInstant](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withOffsetSameInstant%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withOffsetSameLocal-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[withOffsetSameLocal](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withOffsetSameLocal%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-OffsetTime-withSecond-1]]<<painless-api-reference-OffsetTime,OffsetTime>> link:{java8-javadoc}/java/time/OffsetTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/OffsetTime.html#withSecond%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/Optional.asciidoc b/docs/reference/painless-api-reference/Optional.asciidoc
new file mode 100644
index 00000000000..a67616f006f
--- /dev/null
+++ b/docs/reference/painless-api-reference/Optional.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Optional]]++Optional++::
+* ++[[painless-api-reference-Optional-empty-0]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/Optional.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-of-1]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#of%2Djava.lang.Object%2D[of](def)++ (link:{java9-javadoc}/java/util/Optional.html#of%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-ofNullable-1]]static <<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#ofNullable%2Djava.lang.Object%2D[ofNullable](def)++ (link:{java9-javadoc}/java/util/Optional.html#ofNullable%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-filter-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#filter%2Djava.util.function.Predicate%2D[filter](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/Optional.html#filter%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Optional-flatMap-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#flatMap%2Djava.util.function.Function%2D[flatMap](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Optional.html#flatMap%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Optional-get-0]]def link:{java8-javadoc}/java/util/Optional.html#get%2D%2D[get]()++ (link:{java9-javadoc}/java/util/Optional.html#get%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-ifPresent-1]]void link:{java8-javadoc}/java/util/Optional.html#ifPresent%2Djava.util.function.Consumer%2D[ifPresent](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Optional.html#ifPresent%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Optional-isPresent-0]]boolean link:{java8-javadoc}/java/util/Optional.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/Optional.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-Optional-map-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/Optional.html#map%2Djava.util.function.Function%2D[map](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/Optional.html#map%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Optional-orElse-1]]def link:{java8-javadoc}/java/util/Optional.html#orElse%2Djava.lang.Object%2D[orElse](def)++ (link:{java9-javadoc}/java/util/Optional.html#orElse%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Optional-orElseGet-1]]def link:{java8-javadoc}/java/util/Optional.html#orElseGet%2Djava.util.function.Supplier%2D[orElseGet](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/Optional.html#orElseGet%2Djava.util.function.Supplier%2D[java 9])
+* ++[[painless-api-reference-Optional-orElseThrow-1]]def link:{java8-javadoc}/java/util/Optional.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/Optional.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/OptionalDouble.asciidoc b/docs/reference/painless-api-reference/OptionalDouble.asciidoc
new file mode 100644
index 00000000000..56ac37deaa3
--- /dev/null
+++ b/docs/reference/painless-api-reference/OptionalDouble.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalDouble]]++OptionalDouble++::
+* ++[[painless-api-reference-OptionalDouble-empty-0]]static <<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/OptionalDouble.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-of-1]]static <<painless-api-reference-OptionalDouble,OptionalDouble>> link:{java8-javadoc}/java/util/OptionalDouble.html#of%2Ddouble%2D[of](double)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#of%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-getAsDouble-0]]double link:{java8-javadoc}/java/util/OptionalDouble.html#getAsDouble%2D%2D[getAsDouble]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#getAsDouble%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalDouble.html#ifPresent%2Djava.util.function.DoubleConsumer%2D[ifPresent](<<painless-api-reference-DoubleConsumer,DoubleConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#ifPresent%2Djava.util.function.DoubleConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalDouble.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalDouble.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElse-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElse%2Ddouble%2D[orElse](double)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElse%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElseGet-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElseGet%2Djava.util.function.DoubleSupplier%2D[orElseGet](<<painless-api-reference-DoubleSupplier,DoubleSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElseGet%2Djava.util.function.DoubleSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalDouble-orElseThrow-1]]double link:{java8-javadoc}/java/util/OptionalDouble.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalDouble.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/OptionalInt.asciidoc b/docs/reference/painless-api-reference/OptionalInt.asciidoc
new file mode 100644
index 00000000000..69f0059b85f
--- /dev/null
+++ b/docs/reference/painless-api-reference/OptionalInt.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalInt]]++OptionalInt++::
+* ++[[painless-api-reference-OptionalInt-empty-0]]static <<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/OptionalInt.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-of-1]]static <<painless-api-reference-OptionalInt,OptionalInt>> link:{java8-javadoc}/java/util/OptionalInt.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/util/OptionalInt.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-getAsInt-0]]int link:{java8-javadoc}/java/util/OptionalInt.html#getAsInt%2D%2D[getAsInt]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#getAsInt%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalInt.html#ifPresent%2Djava.util.function.IntConsumer%2D[ifPresent](<<painless-api-reference-IntConsumer,IntConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#ifPresent%2Djava.util.function.IntConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalInt.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalInt.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElse-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElse%2Dint%2D[orElse](int)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElse%2Dint%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElseGet-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElseGet%2Djava.util.function.IntSupplier%2D[orElseGet](<<painless-api-reference-IntSupplier,IntSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElseGet%2Djava.util.function.IntSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalInt-orElseThrow-1]]int link:{java8-javadoc}/java/util/OptionalInt.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalInt.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/OptionalLong.asciidoc b/docs/reference/painless-api-reference/OptionalLong.asciidoc
new file mode 100644
index 00000000000..a4b135e5511
--- /dev/null
+++ b/docs/reference/painless-api-reference/OptionalLong.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-OptionalLong]]++OptionalLong++::
+* ++[[painless-api-reference-OptionalLong-empty-0]]static <<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/OptionalLong.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-of-1]]static <<painless-api-reference-OptionalLong,OptionalLong>> link:{java8-javadoc}/java/util/OptionalLong.html#of%2Dlong%2D[of](long)++ (link:{java9-javadoc}/java/util/OptionalLong.html#of%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-getAsLong-0]]long link:{java8-javadoc}/java/util/OptionalLong.html#getAsLong%2D%2D[getAsLong]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#getAsLong%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-ifPresent-1]]void link:{java8-javadoc}/java/util/OptionalLong.html#ifPresent%2Djava.util.function.LongConsumer%2D[ifPresent](<<painless-api-reference-LongConsumer,LongConsumer>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#ifPresent%2Djava.util.function.LongConsumer%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-isPresent-0]]boolean link:{java8-javadoc}/java/util/OptionalLong.html#isPresent%2D%2D[isPresent]()++ (link:{java9-javadoc}/java/util/OptionalLong.html#isPresent%2D%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElse-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElse%2Dlong%2D[orElse](long)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElse%2Dlong%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElseGet-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElseGet%2Djava.util.function.LongSupplier%2D[orElseGet](<<painless-api-reference-LongSupplier,LongSupplier>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElseGet%2Djava.util.function.LongSupplier%2D[java 9])
+* ++[[painless-api-reference-OptionalLong-orElseThrow-1]]long link:{java8-javadoc}/java/util/OptionalLong.html#orElseThrow%2Djava.util.function.Supplier%2D[orElseThrow](<<painless-api-reference-Supplier,Supplier>>)++ (link:{java9-javadoc}/java/util/OptionalLong.html#orElseThrow%2Djava.util.function.Supplier%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ParseException.asciidoc b/docs/reference/painless-api-reference/ParseException.asciidoc
new file mode 100644
index 00000000000..086bcd0b7aa
--- /dev/null
+++ b/docs/reference/painless-api-reference/ParseException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ParseException]]++ParseException++::
+* ++[[painless-api-reference-ParseException-ParseException-2]]link:{java8-javadoc}/java/text/ParseException.html#ParseException%2Djava.lang.String%2Dint%2D[ParseException](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/ParseException.html#ParseException%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParseException-getErrorOffset-0]]int link:{java8-javadoc}/java/text/ParseException.html#getErrorOffset%2D%2D[getErrorOffset]()++ (link:{java9-javadoc}/java/text/ParseException.html#getErrorOffset%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ParsePosition.asciidoc b/docs/reference/painless-api-reference/ParsePosition.asciidoc
new file mode 100644
index 00000000000..88fe6369657
--- /dev/null
+++ b/docs/reference/painless-api-reference/ParsePosition.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ParsePosition]]++ParsePosition++::
+* ++[[painless-api-reference-ParsePosition-ParsePosition-1]]link:{java8-javadoc}/java/text/ParsePosition.html#ParsePosition%2Dint%2D[ParsePosition](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#ParsePosition%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-getErrorIndex-0]]int link:{java8-javadoc}/java/text/ParsePosition.html#getErrorIndex%2D%2D[getErrorIndex]()++ (link:{java9-javadoc}/java/text/ParsePosition.html#getErrorIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-getIndex-0]]int link:{java8-javadoc}/java/text/ParsePosition.html#getIndex%2D%2D[getIndex]()++ (link:{java9-javadoc}/java/text/ParsePosition.html#getIndex%2D%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-setErrorIndex-1]]void link:{java8-javadoc}/java/text/ParsePosition.html#setErrorIndex%2Dint%2D[setErrorIndex](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#setErrorIndex%2Dint%2D[java 9])
+* ++[[painless-api-reference-ParsePosition-setIndex-1]]void link:{java8-javadoc}/java/text/ParsePosition.html#setIndex%2Dint%2D[setIndex](int)++ (link:{java9-javadoc}/java/text/ParsePosition.html#setIndex%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Pattern.asciidoc b/docs/reference/painless-api-reference/Pattern.asciidoc
new file mode 100644
index 00000000000..ad7b3603036
--- /dev/null
+++ b/docs/reference/painless-api-reference/Pattern.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Pattern]]++Pattern++::
+* ++[[painless-api-reference-Pattern-quote-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Pattern.html#quote%2Djava.lang.String%2D[quote](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#quote%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Pattern-asPredicate-0]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/regex/Pattern.html#asPredicate%2D%2D[asPredicate]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#asPredicate%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-flags-0]]int link:{java8-javadoc}/java/util/regex/Pattern.html#flags%2D%2D[flags]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#flags%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-matcher-1]]<<painless-api-reference-Matcher,Matcher>> link:{java8-javadoc}/java/util/regex/Pattern.html#matcher%2Djava.lang.CharSequence%2D[matcher](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#matcher%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Pattern-pattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/regex/Pattern.html#pattern%2D%2D[pattern]()++ (link:{java9-javadoc}/java/util/regex/Pattern.html#pattern%2D%2D[java 9])
+* ++[[painless-api-reference-Pattern-split-1]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2D[split](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Pattern-split-2]]<<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2Dint%2D[split](<<painless-api-reference-CharSequence,CharSequence>>, int)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#split%2Djava.lang.CharSequence%2Dint%2D[java 9])
+* ++[[painless-api-reference-Pattern-splitAsStream-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/regex/Pattern.html#splitAsStream%2Djava.lang.CharSequence%2D[splitAsStream](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/regex/Pattern.html#splitAsStream%2Djava.lang.CharSequence%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Period.asciidoc b/docs/reference/painless-api-reference/Period.asciidoc
new file mode 100644
index 00000000000..bb20dddae60
--- /dev/null
+++ b/docs/reference/painless-api-reference/Period.asciidoc
@@ -0,0 +1,35 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Period]]++Period++::
+** [[painless-api-reference-Period-ZERO]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ZERO[ZERO] (link:{java9-javadoc}/java/time/Period.html#ZERO[java 9])
+* ++[[painless-api-reference-Period-between-2]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#between%2Djava.time.LocalDate%2Djava.time.LocalDate%2D[between](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalDate,LocalDate>>)++ (link:{java9-javadoc}/java/time/Period.html#between%2Djava.time.LocalDate%2Djava.time.LocalDate%2D[java 9])
+* ++[[painless-api-reference-Period-from-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#from%2Djava.time.temporal.TemporalAmount%2D[from](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#from%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-of-3]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/Period.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofDays-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofDays%2Dint%2D[ofDays](int)++ (link:{java9-javadoc}/java/time/Period.html#ofDays%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofMonths-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofMonths%2Dint%2D[ofMonths](int)++ (link:{java9-javadoc}/java/time/Period.html#ofMonths%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofWeeks-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofWeeks%2Dint%2D[ofWeeks](int)++ (link:{java9-javadoc}/java/time/Period.html#ofWeeks%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-ofYears-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#ofYears%2Dint%2D[ofYears](int)++ (link:{java9-javadoc}/java/time/Period.html#ofYears%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-parse-1]]static <<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Period.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Period-getChronology-0]]<<painless-api-reference-IsoChronology,IsoChronology>> link:{java8-javadoc}/java/time/Period.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/Period.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getDays-0]]int link:{java8-javadoc}/java/time/Period.html#getDays%2D%2D[getDays]()++ (link:{java9-javadoc}/java/time/Period.html#getDays%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getMonths-0]]int link:{java8-javadoc}/java/time/Period.html#getMonths%2D%2D[getMonths]()++ (link:{java9-javadoc}/java/time/Period.html#getMonths%2D%2D[java 9])
+* ++[[painless-api-reference-Period-getYears-0]]int link:{java8-javadoc}/java/time/Period.html#getYears%2D%2D[getYears]()++ (link:{java9-javadoc}/java/time/Period.html#getYears%2D%2D[java 9])
+* ++[[painless-api-reference-Period-minus-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-minusDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/Period.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-minusMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/Period.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-minusYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/Period.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-multipliedBy-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#multipliedBy%2Dint%2D[multipliedBy](int)++ (link:{java9-javadoc}/java/time/Period.html#multipliedBy%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-negated-0]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#negated%2D%2D[negated]()++ (link:{java9-javadoc}/java/time/Period.html#negated%2D%2D[java 9])
+* ++[[painless-api-reference-Period-normalized-0]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/Period.html#normalized%2D%2D[java 9])
+* ++[[painless-api-reference-Period-plus-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Period.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Period-plusDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/Period.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-plusMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/Period.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-plusYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/Period.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Period-toTotalMonths-0]]long link:{java8-javadoc}/java/time/Period.html#toTotalMonths%2D%2D[toTotalMonths]()++ (link:{java9-javadoc}/java/time/Period.html#toTotalMonths%2D%2D[java 9])
+* ++[[painless-api-reference-Period-withDays-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withDays%2Dint%2D[withDays](int)++ (link:{java9-javadoc}/java/time/Period.html#withDays%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-withMonths-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withMonths%2Dint%2D[withMonths](int)++ (link:{java9-javadoc}/java/time/Period.html#withMonths%2Dint%2D[java 9])
+* ++[[painless-api-reference-Period-withYears-1]]<<painless-api-reference-Period,Period>> link:{java8-javadoc}/java/time/Period.html#withYears%2Dint%2D[withYears](int)++ (link:{java9-javadoc}/java/time/Period.html#withYears%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoPeriod,ChronoPeriod>>++, ++<<painless-api-reference-TemporalAmount,TemporalAmount>>++
diff --git a/docs/reference/painless-api-reference/Predicate.asciidoc b/docs/reference/painless-api-reference/Predicate.asciidoc
new file mode 100644
index 00000000000..6eef93ea27c
--- /dev/null
+++ b/docs/reference/painless-api-reference/Predicate.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Predicate]]++Predicate++::
+* ++[[painless-api-reference-Predicate-isEqual-1]]static <<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#isEqual%2Djava.lang.Object%2D[isEqual](def)++ (link:{java9-javadoc}/java/util/function/Predicate.html#isEqual%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Predicate-and-1]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#and%2Djava.util.function.Predicate%2D[and](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/function/Predicate.html#and%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Predicate-negate-0]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#negate%2D%2D[negate]()++ (link:{java9-javadoc}/java/util/function/Predicate.html#negate%2D%2D[java 9])
+* ++[[painless-api-reference-Predicate-or-1]]<<painless-api-reference-Predicate,Predicate>> link:{java8-javadoc}/java/util/function/Predicate.html#or%2Djava.util.function.Predicate%2D[or](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/function/Predicate.html#or%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Predicate-test-1]]boolean link:{java8-javadoc}/java/util/function/Predicate.html#test%2Djava.lang.Object%2D[test](def)++ (link:{java9-javadoc}/java/util/function/Predicate.html#test%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc b/docs/reference/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc
new file mode 100644
index 00000000000..bb26cf3b514
--- /dev/null
+++ b/docs/reference/painless-api-reference/PrimitiveIterator.OfDouble.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfDouble]]++PrimitiveIterator.OfDouble++::
+* ++[[painless-api-reference-PrimitiveIterator-OfDouble-next-0]]<<painless-api-reference-Double,Double>> link:{java8-javadoc}/java/util/PrimitiveIterator$OfDouble.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfDouble.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfDouble-nextDouble-0]]double link:{java8-javadoc}/java/util/PrimitiveIterator$OfDouble.html#nextDouble%2D%2D[nextDouble]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfDouble.html#nextDouble%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/reference/painless-api-reference/PrimitiveIterator.OfInt.asciidoc b/docs/reference/painless-api-reference/PrimitiveIterator.OfInt.asciidoc
new file mode 100644
index 00000000000..1cd90b8e13f
--- /dev/null
+++ b/docs/reference/painless-api-reference/PrimitiveIterator.OfInt.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfInt]]++PrimitiveIterator.OfInt++::
+* ++[[painless-api-reference-PrimitiveIterator-OfInt-next-0]]<<painless-api-reference-Integer,Integer>> link:{java8-javadoc}/java/util/PrimitiveIterator$OfInt.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfInt.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfInt-nextInt-0]]int link:{java8-javadoc}/java/util/PrimitiveIterator$OfInt.html#nextInt%2D%2D[nextInt]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfInt.html#nextInt%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/reference/painless-api-reference/PrimitiveIterator.OfLong.asciidoc b/docs/reference/painless-api-reference/PrimitiveIterator.OfLong.asciidoc
new file mode 100644
index 00000000000..bcfff0e7edc
--- /dev/null
+++ b/docs/reference/painless-api-reference/PrimitiveIterator.OfLong.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator-OfLong]]++PrimitiveIterator.OfLong++::
+* ++[[painless-api-reference-PrimitiveIterator-OfLong-next-0]]<<painless-api-reference-Long,Long>> link:{java8-javadoc}/java/util/PrimitiveIterator$OfLong.html#next%2D%2D[next]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfLong.html#next%2D%2D[java 9])
+* ++[[painless-api-reference-PrimitiveIterator-OfLong-nextLong-0]]long link:{java8-javadoc}/java/util/PrimitiveIterator$OfLong.html#nextLong%2D%2D[nextLong]()++ (link:{java9-javadoc}/java/util/PrimitiveIterator$OfLong.html#nextLong%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-PrimitiveIterator,PrimitiveIterator>>++
diff --git a/docs/reference/painless-api-reference/PrimitiveIterator.asciidoc b/docs/reference/painless-api-reference/PrimitiveIterator.asciidoc
new file mode 100644
index 00000000000..42742f79d3d
--- /dev/null
+++ b/docs/reference/painless-api-reference/PrimitiveIterator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PrimitiveIterator]]++PrimitiveIterator++::
+* ++[[painless-api-reference-PrimitiveIterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/PrimitiveIterator.html#forEachRemaining%2Djava.lang.Object%2D[forEachRemaining](def)++ (link:{java9-javadoc}/java/util/PrimitiveIterator.html#forEachRemaining%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Iterator,Iterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/PriorityQueue.asciidoc b/docs/reference/painless-api-reference/PriorityQueue.asciidoc
new file mode 100644
index 00000000000..88334cd8b9c
--- /dev/null
+++ b/docs/reference/painless-api-reference/PriorityQueue.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-PriorityQueue]]++PriorityQueue++::
+* ++[[painless-api-reference-PriorityQueue-PriorityQueue-0]]link:{java8-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2D%2D[PriorityQueue]()++ (link:{java9-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2D%2D[java 9])
+* ++[[painless-api-reference-PriorityQueue-PriorityQueue-1]]link:{java8-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2Djava.util.Comparator%2D[PriorityQueue](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/PriorityQueue.html#PriorityQueue%2Djava.util.Comparator%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Queue,Queue>>++
diff --git a/docs/reference/painless-api-reference/Queue.asciidoc b/docs/reference/painless-api-reference/Queue.asciidoc
new file mode 100644
index 00000000000..7d323457591
--- /dev/null
+++ b/docs/reference/painless-api-reference/Queue.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Queue]]++Queue++::
+* ++[[painless-api-reference-Queue-element-0]]def link:{java8-javadoc}/java/util/Queue.html#element%2D%2D[element]()++ (link:{java9-javadoc}/java/util/Queue.html#element%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-offer-1]]boolean link:{java8-javadoc}/java/util/Queue.html#offer%2Djava.lang.Object%2D[offer](def)++ (link:{java9-javadoc}/java/util/Queue.html#offer%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Queue-peek-0]]def link:{java8-javadoc}/java/util/Queue.html#peek%2D%2D[peek]()++ (link:{java9-javadoc}/java/util/Queue.html#peek%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-poll-0]]def link:{java8-javadoc}/java/util/Queue.html#poll%2D%2D[poll]()++ (link:{java9-javadoc}/java/util/Queue.html#poll%2D%2D[java 9])
+* ++[[painless-api-reference-Queue-remove-0]]def link:{java8-javadoc}/java/util/Queue.html#remove%2D%2D[remove]()++ (link:{java9-javadoc}/java/util/Queue.html#remove%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Random.asciidoc b/docs/reference/painless-api-reference/Random.asciidoc
new file mode 100644
index 00000000000..4288c330850
--- /dev/null
+++ b/docs/reference/painless-api-reference/Random.asciidoc
@@ -0,0 +1,24 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Random]]++Random++::
+* ++[[painless-api-reference-Random-Random-0]]link:{java8-javadoc}/java/util/Random.html#Random%2D%2D[Random]()++ (link:{java9-javadoc}/java/util/Random.html#Random%2D%2D[java 9])
+* ++[[painless-api-reference-Random-Random-1]]link:{java8-javadoc}/java/util/Random.html#Random%2Dlong%2D[Random](long)++ (link:{java9-javadoc}/java/util/Random.html#Random%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-doubles-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/Random.html#doubles%2Dlong%2D[doubles](long)++ (link:{java9-javadoc}/java/util/Random.html#doubles%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-doubles-3]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/Random.html#doubles%2Dlong%2Ddouble%2Ddouble%2D[doubles](long, double, double)++ (link:{java9-javadoc}/java/util/Random.html#doubles%2Dlong%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-Random-ints-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/Random.html#ints%2Dlong%2D[ints](long)++ (link:{java9-javadoc}/java/util/Random.html#ints%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-ints-3]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/Random.html#ints%2Dlong%2Dint%2Dint%2D[ints](long, int, int)++ (link:{java9-javadoc}/java/util/Random.html#ints%2Dlong%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-Random-longs-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/Random.html#longs%2Dlong%2D[longs](long)++ (link:{java9-javadoc}/java/util/Random.html#longs%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-longs-3]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/Random.html#longs%2Dlong%2Dlong%2Dlong%2D[longs](long, long, long)++ (link:{java9-javadoc}/java/util/Random.html#longs%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Random-nextBoolean-0]]boolean link:{java8-javadoc}/java/util/Random.html#nextBoolean%2D%2D[nextBoolean]()++ (link:{java9-javadoc}/java/util/Random.html#nextBoolean%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextBytes-1]]void link:{java8-javadoc}/java/util/Random.html#nextBytes%2Dbyte:A%2D[nextBytes](byte[])++ (link:{java9-javadoc}/java/util/Random.html#nextBytes%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-Random-nextDouble-0]]double link:{java8-javadoc}/java/util/Random.html#nextDouble%2D%2D[nextDouble]()++ (link:{java9-javadoc}/java/util/Random.html#nextDouble%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextFloat-0]]float link:{java8-javadoc}/java/util/Random.html#nextFloat%2D%2D[nextFloat]()++ (link:{java9-javadoc}/java/util/Random.html#nextFloat%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextGaussian-0]]double link:{java8-javadoc}/java/util/Random.html#nextGaussian%2D%2D[nextGaussian]()++ (link:{java9-javadoc}/java/util/Random.html#nextGaussian%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextInt-0]]int link:{java8-javadoc}/java/util/Random.html#nextInt%2D%2D[nextInt]()++ (link:{java9-javadoc}/java/util/Random.html#nextInt%2D%2D[java 9])
+* ++[[painless-api-reference-Random-nextInt-1]]int link:{java8-javadoc}/java/util/Random.html#nextInt%2Dint%2D[nextInt](int)++ (link:{java9-javadoc}/java/util/Random.html#nextInt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Random-nextLong-0]]long link:{java8-javadoc}/java/util/Random.html#nextLong%2D%2D[nextLong]()++ (link:{java9-javadoc}/java/util/Random.html#nextLong%2D%2D[java 9])
+* ++[[painless-api-reference-Random-setSeed-1]]void link:{java8-javadoc}/java/util/Random.html#setSeed%2Dlong%2D[setSeed](long)++ (link:{java9-javadoc}/java/util/Random.html#setSeed%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/RandomAccess.asciidoc b/docs/reference/painless-api-reference/RandomAccess.asciidoc
new file mode 100644
index 00000000000..b7afb205f0c
--- /dev/null
+++ b/docs/reference/painless-api-reference/RandomAccess.asciidoc
@@ -0,0 +1,7 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RandomAccess]]++RandomAccess++::
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ReflectiveOperationException.asciidoc b/docs/reference/painless-api-reference/ReflectiveOperationException.asciidoc
new file mode 100644
index 00000000000..eee134d373a
--- /dev/null
+++ b/docs/reference/painless-api-reference/ReflectiveOperationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ReflectiveOperationException]]++ReflectiveOperationException++::
+* ++[[painless-api-reference-ReflectiveOperationException-ReflectiveOperationException-0]]link:{java8-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2D%2D[ReflectiveOperationException]()++ (link:{java9-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2D%2D[java 9])
+* ++[[painless-api-reference-ReflectiveOperationException-ReflectiveOperationException-1]]link:{java8-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2Djava.lang.String%2D[ReflectiveOperationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/ReflectiveOperationException.html#ReflectiveOperationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ResolverStyle.asciidoc b/docs/reference/painless-api-reference/ResolverStyle.asciidoc
new file mode 100644
index 00000000000..00576a2d93a
--- /dev/null
+++ b/docs/reference/painless-api-reference/ResolverStyle.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ResolverStyle]]++ResolverStyle++::
+** [[painless-api-reference-ResolverStyle-LENIENT]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#LENIENT[LENIENT] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#LENIENT[java 9])
+** [[painless-api-reference-ResolverStyle-SMART]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#SMART[SMART] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#SMART[java 9])
+** [[painless-api-reference-ResolverStyle-STRICT]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#STRICT[STRICT] (link:{java9-javadoc}/java/time/format/ResolverStyle.html#STRICT[java 9])
+* ++[[painless-api-reference-ResolverStyle-valueOf-1]]static <<painless-api-reference-ResolverStyle,ResolverStyle>> link:{java8-javadoc}/java/time/format/ResolverStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/ResolverStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ResolverStyle-values-0]]static <<painless-api-reference-ResolverStyle,ResolverStyle>>[] link:{java8-javadoc}/java/time/format/ResolverStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/ResolverStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/RoundingMode.asciidoc b/docs/reference/painless-api-reference/RoundingMode.asciidoc
new file mode 100644
index 00000000000..e0b4d579835
--- /dev/null
+++ b/docs/reference/painless-api-reference/RoundingMode.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RoundingMode]]++RoundingMode++::
+** [[painless-api-reference-RoundingMode-CEILING]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#CEILING[CEILING] (link:{java9-javadoc}/java/math/RoundingMode.html#CEILING[java 9])
+** [[painless-api-reference-RoundingMode-DOWN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#DOWN[DOWN] (link:{java9-javadoc}/java/math/RoundingMode.html#DOWN[java 9])
+** [[painless-api-reference-RoundingMode-FLOOR]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#FLOOR[FLOOR] (link:{java9-javadoc}/java/math/RoundingMode.html#FLOOR[java 9])
+** [[painless-api-reference-RoundingMode-HALF_DOWN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_DOWN[HALF_DOWN] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_DOWN[java 9])
+** [[painless-api-reference-RoundingMode-HALF_EVEN]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_EVEN[HALF_EVEN] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_EVEN[java 9])
+** [[painless-api-reference-RoundingMode-HALF_UP]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#HALF_UP[HALF_UP] (link:{java9-javadoc}/java/math/RoundingMode.html#HALF_UP[java 9])
+** [[painless-api-reference-RoundingMode-UNNECESSARY]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#UNNECESSARY[UNNECESSARY] (link:{java9-javadoc}/java/math/RoundingMode.html#UNNECESSARY[java 9])
+** [[painless-api-reference-RoundingMode-UP]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#UP[UP] (link:{java9-javadoc}/java/math/RoundingMode.html#UP[java 9])
+* ++[[painless-api-reference-RoundingMode-valueOf-1]]static <<painless-api-reference-RoundingMode,RoundingMode>> link:{java8-javadoc}/java/math/RoundingMode.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/math/RoundingMode.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RoundingMode-values-0]]static <<painless-api-reference-RoundingMode,RoundingMode>>[] link:{java8-javadoc}/java/math/RoundingMode.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/math/RoundingMode.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/RuleBasedCollator.asciidoc b/docs/reference/painless-api-reference/RuleBasedCollator.asciidoc
new file mode 100644
index 00000000000..5581768e2f3
--- /dev/null
+++ b/docs/reference/painless-api-reference/RuleBasedCollator.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RuleBasedCollator]]++RuleBasedCollator++::
+* ++[[painless-api-reference-RuleBasedCollator-RuleBasedCollator-1]]link:{java8-javadoc}/java/text/RuleBasedCollator.html#RuleBasedCollator%2Djava.lang.String%2D[RuleBasedCollator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#RuleBasedCollator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RuleBasedCollator-getCollationElementIterator-1]]<<painless-api-reference-CollationElementIterator,CollationElementIterator>> link:{java8-javadoc}/java/text/RuleBasedCollator.html#getCollationElementIterator%2Djava.lang.String%2D[getCollationElementIterator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#getCollationElementIterator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-RuleBasedCollator-getRules-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/RuleBasedCollator.html#getRules%2D%2D[getRules]()++ (link:{java9-javadoc}/java/text/RuleBasedCollator.html#getRules%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collator,Collator>>++, ++<<painless-api-reference-Comparator,Comparator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/RuntimeException.asciidoc b/docs/reference/painless-api-reference/RuntimeException.asciidoc
new file mode 100644
index 00000000000..fc8523169e3
--- /dev/null
+++ b/docs/reference/painless-api-reference/RuntimeException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-RuntimeException]]++RuntimeException++::
+* ++[[painless-api-reference-RuntimeException-RuntimeException-0]]link:{java8-javadoc}/java/lang/RuntimeException.html#RuntimeException%2D%2D[RuntimeException]()++ (link:{java9-javadoc}/java/lang/RuntimeException.html#RuntimeException%2D%2D[java 9])
+* ++[[painless-api-reference-RuntimeException-RuntimeException-1]]link:{java8-javadoc}/java/lang/RuntimeException.html#RuntimeException%2Djava.lang.String%2D[RuntimeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/RuntimeException.html#RuntimeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/SecurityException.asciidoc b/docs/reference/painless-api-reference/SecurityException.asciidoc
new file mode 100644
index 00000000000..ec6ea157e55
--- /dev/null
+++ b/docs/reference/painless-api-reference/SecurityException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SecurityException]]++SecurityException++::
+* ++[[painless-api-reference-SecurityException-SecurityException-0]]link:{java8-javadoc}/java/lang/SecurityException.html#SecurityException%2D%2D[SecurityException]()++ (link:{java9-javadoc}/java/lang/SecurityException.html#SecurityException%2D%2D[java 9])
+* ++[[painless-api-reference-SecurityException-SecurityException-1]]link:{java8-javadoc}/java/lang/SecurityException.html#SecurityException%2Djava.lang.String%2D[SecurityException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/SecurityException.html#SecurityException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Set.asciidoc b/docs/reference/painless-api-reference/Set.asciidoc
new file mode 100644
index 00000000000..23d8a352f23
--- /dev/null
+++ b/docs/reference/painless-api-reference/Set.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Set]]++Set++::
+* ++[[painless-api-reference-Set-equals-1]]boolean link:{java8-javadoc}/java/util/Set.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++ (link:{java9-javadoc}/java/util/Set.html#equals%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Set-hashCode-0]]int link:{java8-javadoc}/java/util/Set.html#hashCode%2D%2D[hashCode]()++ (link:{java9-javadoc}/java/util/Set.html#hashCode%2D%2D[java 9])
+* ++[[painless-api-reference-Set-remove-1]]boolean link:{java8-javadoc}/java/util/Set.html#remove%2Djava.lang.Object%2D[remove](def)++ (link:{java9-javadoc}/java/util/Set.html#remove%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Short.asciidoc b/docs/reference/painless-api-reference/Short.asciidoc
new file mode 100644
index 00000000000..12f0336e8fd
--- /dev/null
+++ b/docs/reference/painless-api-reference/Short.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Short]]++Short++::
+** [[painless-api-reference-Short-BYTES]]static int link:{java8-javadoc}/java/lang/Short.html#BYTES[BYTES] (link:{java9-javadoc}/java/lang/Short.html#BYTES[java 9])
+** [[painless-api-reference-Short-MAX_VALUE]]static short link:{java8-javadoc}/java/lang/Short.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/lang/Short.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Short-MIN_VALUE]]static short link:{java8-javadoc}/java/lang/Short.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/lang/Short.html#MIN_VALUE[java 9])
+** [[painless-api-reference-Short-SIZE]]static int link:{java8-javadoc}/java/lang/Short.html#SIZE[SIZE] (link:{java9-javadoc}/java/lang/Short.html#SIZE[java 9])
+* ++[[painless-api-reference-Short-compare-2]]static int link:{java8-javadoc}/java/lang/Short.html#compare%2Dshort%2Dshort%2D[compare](short, short)++ (link:{java9-javadoc}/java/lang/Short.html#compare%2Dshort%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-decode-1]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#decode%2Djava.lang.String%2D[decode](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Short.html#decode%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Short-hashCode-1]]static int link:{java8-javadoc}/java/lang/Short.html#hashCode%2Dshort%2D[hashCode](short)++ (link:{java9-javadoc}/java/lang/Short.html#hashCode%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-parseShort-1]]static short link:{java8-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2D[parseShort](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-Short-parseShort-2]]static short link:{java8-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2Dint%2D[parseShort](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Short.html#parseShort%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Short-reverseBytes-1]]static short link:{java8-javadoc}/java/lang/Short.html#reverseBytes%2Dshort%2D[reverseBytes](short)++ (link:{java9-javadoc}/java/lang/Short.html#reverseBytes%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toString-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/Short.html#toString%2Dshort%2D[toString](short)++ (link:{java9-javadoc}/java/lang/Short.html#toString%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toUnsignedInt-1]]static int link:{java8-javadoc}/java/lang/Short.html#toUnsignedInt%2Dshort%2D[toUnsignedInt](short)++ (link:{java9-javadoc}/java/lang/Short.html#toUnsignedInt%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-toUnsignedLong-1]]static long link:{java8-javadoc}/java/lang/Short.html#toUnsignedLong%2Dshort%2D[toUnsignedLong](short)++ (link:{java9-javadoc}/java/lang/Short.html#toUnsignedLong%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-valueOf-1]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#valueOf%2Dshort%2D[valueOf](short)++ (link:{java9-javadoc}/java/lang/Short.html#valueOf%2Dshort%2D[java 9])
+* ++[[painless-api-reference-Short-valueOf-2]]static <<painless-api-reference-Short,Short>> link:{java8-javadoc}/java/lang/Short.html#valueOf%2Djava.lang.String%2Dint%2D[valueOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/Short.html#valueOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-Short-compareTo-1]]int link:{java8-javadoc}/java/lang/Short.html#compareTo%2Djava.lang.Short%2D[compareTo](<<painless-api-reference-Short,Short>>)++ (link:{java9-javadoc}/java/lang/Short.html#compareTo%2Djava.lang.Short%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Number,Number>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/SignStyle.asciidoc b/docs/reference/painless-api-reference/SignStyle.asciidoc
new file mode 100644
index 00000000000..a42ce94ebfa
--- /dev/null
+++ b/docs/reference/painless-api-reference/SignStyle.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SignStyle]]++SignStyle++::
+** [[painless-api-reference-SignStyle-ALWAYS]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#ALWAYS[ALWAYS] (link:{java9-javadoc}/java/time/format/SignStyle.html#ALWAYS[java 9])
+** [[painless-api-reference-SignStyle-EXCEEDS_PAD]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#EXCEEDS_PAD[EXCEEDS_PAD] (link:{java9-javadoc}/java/time/format/SignStyle.html#EXCEEDS_PAD[java 9])
+** [[painless-api-reference-SignStyle-NEVER]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NEVER[NEVER] (link:{java9-javadoc}/java/time/format/SignStyle.html#NEVER[java 9])
+** [[painless-api-reference-SignStyle-NORMAL]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NORMAL[NORMAL] (link:{java9-javadoc}/java/time/format/SignStyle.html#NORMAL[java 9])
+** [[painless-api-reference-SignStyle-NOT_NEGATIVE]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#NOT_NEGATIVE[NOT_NEGATIVE] (link:{java9-javadoc}/java/time/format/SignStyle.html#NOT_NEGATIVE[java 9])
+* ++[[painless-api-reference-SignStyle-valueOf-1]]static <<painless-api-reference-SignStyle,SignStyle>> link:{java8-javadoc}/java/time/format/SignStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/SignStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SignStyle-values-0]]static <<painless-api-reference-SignStyle,SignStyle>>[] link:{java8-javadoc}/java/time/format/SignStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/SignStyle.html#values%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/SimpleDateFormat.asciidoc b/docs/reference/painless-api-reference/SimpleDateFormat.asciidoc
new file mode 100644
index 00000000000..b7a8ffc191c
--- /dev/null
+++ b/docs/reference/painless-api-reference/SimpleDateFormat.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SimpleDateFormat]]++SimpleDateFormat++::
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-0]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2D%2D[SimpleDateFormat]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-1]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2D[SimpleDateFormat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-SimpleDateFormat-2]]link:{java8-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2Djava.util.Locale%2D[SimpleDateFormat](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#SimpleDateFormat%2Djava.lang.String%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-applyLocalizedPattern-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[applyLocalizedPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#applyLocalizedPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-applyPattern-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#applyPattern%2Djava.lang.String%2D[applyPattern](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#applyPattern%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-get2DigitYearStart-0]]<<painless-api-reference-Date,Date>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#get2DigitYearStart%2D%2D[get2DigitYearStart]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#get2DigitYearStart%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-getDateFormatSymbols-0]]<<painless-api-reference-DateFormatSymbols,DateFormatSymbols>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#getDateFormatSymbols%2D%2D[getDateFormatSymbols]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#getDateFormatSymbols%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-set2DigitYearStart-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#set2DigitYearStart%2Djava.util.Date%2D[set2DigitYearStart](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#set2DigitYearStart%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-setDateFormatSymbols-1]]void link:{java8-javadoc}/java/text/SimpleDateFormat.html#setDateFormatSymbols%2Djava.text.DateFormatSymbols%2D[setDateFormatSymbols](<<painless-api-reference-DateFormatSymbols,DateFormatSymbols>>)++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#setDateFormatSymbols%2Djava.text.DateFormatSymbols%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-toLocalizedPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#toLocalizedPattern%2D%2D[toLocalizedPattern]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#toLocalizedPattern%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleDateFormat-toPattern-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/text/SimpleDateFormat.html#toPattern%2D%2D[toPattern]()++ (link:{java9-javadoc}/java/text/SimpleDateFormat.html#toPattern%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-DateFormat,DateFormat>>++, ++<<painless-api-reference-Format,Format>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/SimpleTimeZone.asciidoc b/docs/reference/painless-api-reference/SimpleTimeZone.asciidoc
new file mode 100644
index 00000000000..eff23a40723
--- /dev/null
+++ b/docs/reference/painless-api-reference/SimpleTimeZone.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SimpleTimeZone]]++SimpleTimeZone++::
+** [[painless-api-reference-SimpleTimeZone-STANDARD_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#STANDARD_TIME[STANDARD_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#STANDARD_TIME[java 9])
+** [[painless-api-reference-SimpleTimeZone-UTC_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#UTC_TIME[UTC_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#UTC_TIME[java 9])
+** [[painless-api-reference-SimpleTimeZone-WALL_TIME]]static int link:{java8-javadoc}/java/util/SimpleTimeZone.html#WALL_TIME[WALL_TIME] (link:{java9-javadoc}/java/util/SimpleTimeZone.html#WALL_TIME[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-2]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-10]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-11]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-SimpleTimeZone-13]]link:{java8-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[SimpleTimeZone](int, <<painless-api-reference-String,String>>, int, int, int, int, int, int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#SimpleTimeZone%2Dint%2Djava.lang.String%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-getDSTSavings-0]]int link:{java8-javadoc}/java/util/SimpleTimeZone.html#getDSTSavings%2D%2D[getDSTSavings]()++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#getDSTSavings%2D%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setDSTSavings-1]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setDSTSavings%2Dint%2D[setDSTSavings](int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setDSTSavings%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-3]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2D[setEndRule](int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-4]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2D[setEndRule](int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setEndRule-5]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[setEndRule](int, int, int, int, boolean)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setEndRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-3]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2D[setStartRule](int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-4]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2D[setStartRule](int, int, int, int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartRule-5]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[setStartRule](int, int, int, int, boolean)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartRule%2Dint%2Dint%2Dint%2Dint%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-SimpleTimeZone-setStartYear-1]]void link:{java8-javadoc}/java/util/SimpleTimeZone.html#setStartYear%2Dint%2D[setStartYear](int)++ (link:{java9-javadoc}/java/util/SimpleTimeZone.html#setStartYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TimeZone,TimeZone>>++
diff --git a/docs/reference/painless-api-reference/SortedMap.asciidoc b/docs/reference/painless-api-reference/SortedMap.asciidoc
new file mode 100644
index 00000000000..d4e31e6c8fa
--- /dev/null
+++ b/docs/reference/painless-api-reference/SortedMap.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SortedMap]]++SortedMap++::
+* ++[[painless-api-reference-SortedMap-comparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/SortedMap.html#comparator%2D%2D[comparator]()++ (link:{java9-javadoc}/java/util/SortedMap.html#comparator%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-firstKey-0]]def link:{java8-javadoc}/java/util/SortedMap.html#firstKey%2D%2D[firstKey]()++ (link:{java9-javadoc}/java/util/SortedMap.html#firstKey%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-headMap-1]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#headMap%2Djava.lang.Object%2D[headMap](def)++ (link:{java9-javadoc}/java/util/SortedMap.html#headMap%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedMap-lastKey-0]]def link:{java8-javadoc}/java/util/SortedMap.html#lastKey%2D%2D[lastKey]()++ (link:{java9-javadoc}/java/util/SortedMap.html#lastKey%2D%2D[java 9])
+* ++[[painless-api-reference-SortedMap-subMap-2]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#subMap%2Djava.lang.Object%2Djava.lang.Object%2D[subMap](def, def)++ (link:{java9-javadoc}/java/util/SortedMap.html#subMap%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedMap-tailMap-1]]<<painless-api-reference-SortedMap,SortedMap>> link:{java8-javadoc}/java/util/SortedMap.html#tailMap%2Djava.lang.Object%2D[tailMap](def)++ (link:{java9-javadoc}/java/util/SortedMap.html#tailMap%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/SortedSet.asciidoc b/docs/reference/painless-api-reference/SortedSet.asciidoc
new file mode 100644
index 00000000000..2a7160aa2ba
--- /dev/null
+++ b/docs/reference/painless-api-reference/SortedSet.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-SortedSet]]++SortedSet++::
+* ++[[painless-api-reference-SortedSet-comparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/SortedSet.html#comparator%2D%2D[comparator]()++ (link:{java9-javadoc}/java/util/SortedSet.html#comparator%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-first-0]]def link:{java8-javadoc}/java/util/SortedSet.html#first%2D%2D[first]()++ (link:{java9-javadoc}/java/util/SortedSet.html#first%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-headSet-1]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#headSet%2Djava.lang.Object%2D[headSet](def)++ (link:{java9-javadoc}/java/util/SortedSet.html#headSet%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedSet-last-0]]def link:{java8-javadoc}/java/util/SortedSet.html#last%2D%2D[last]()++ (link:{java9-javadoc}/java/util/SortedSet.html#last%2D%2D[java 9])
+* ++[[painless-api-reference-SortedSet-subSet-2]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#subSet%2Djava.lang.Object%2Djava.lang.Object%2D[subSet](def, def)++ (link:{java9-javadoc}/java/util/SortedSet.html#subSet%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-SortedSet-tailSet-1]]<<painless-api-reference-SortedSet,SortedSet>> link:{java8-javadoc}/java/util/SortedSet.html#tailSet%2Djava.lang.Object%2D[tailSet](def)++ (link:{java9-javadoc}/java/util/SortedSet.html#tailSet%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++
diff --git a/docs/reference/painless-api-reference/Spliterator.OfDouble.asciidoc b/docs/reference/painless-api-reference/Spliterator.OfDouble.asciidoc
new file mode 100644
index 00000000000..7292d2bffca
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterator.OfDouble.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfDouble]]++Spliterator.OfDouble++::
+* ++[[painless-api-reference-Spliterator-OfDouble-trySplit-0]]<<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/Spliterator$OfDouble.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator$OfDouble.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/reference/painless-api-reference/Spliterator.OfInt.asciidoc b/docs/reference/painless-api-reference/Spliterator.OfInt.asciidoc
new file mode 100644
index 00000000000..1f19e7013aa
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterator.OfInt.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfInt]]++Spliterator.OfInt++::
+* ++[[painless-api-reference-Spliterator-OfInt-trySplit-0]]<<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/Spliterator$OfInt.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator$OfInt.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/reference/painless-api-reference/Spliterator.OfLong.asciidoc b/docs/reference/painless-api-reference/Spliterator.OfLong.asciidoc
new file mode 100644
index 00000000000..a1d56d5e1ee
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterator.OfLong.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfLong]]++Spliterator.OfLong++::
+* ++[[painless-api-reference-Spliterator-OfLong-trySplit-0]]<<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/Spliterator$OfLong.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator$OfLong.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++, ++<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>>++
diff --git a/docs/reference/painless-api-reference/Spliterator.OfPrimitive.asciidoc b/docs/reference/painless-api-reference/Spliterator.OfPrimitive.asciidoc
new file mode 100644
index 00000000000..00cd8b47261
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterator.OfPrimitive.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator-OfPrimitive]]++Spliterator.OfPrimitive++::
+* ++[[painless-api-reference-Spliterator-OfPrimitive-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Spliterator$OfPrimitive.html#forEachRemaining%2Djava.lang.Object%2D[forEachRemaining](def)++ (link:{java9-javadoc}/java/util/Spliterator$OfPrimitive.html#forEachRemaining%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Spliterator-OfPrimitive-tryAdvance-1]]boolean link:{java8-javadoc}/java/util/Spliterator$OfPrimitive.html#tryAdvance%2Djava.lang.Object%2D[tryAdvance](def)++ (link:{java9-javadoc}/java/util/Spliterator$OfPrimitive.html#tryAdvance%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Spliterator-OfPrimitive-trySplit-0]]<<painless-api-reference-Spliterator-OfPrimitive,Spliterator.OfPrimitive>> link:{java8-javadoc}/java/util/Spliterator$OfPrimitive.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator$OfPrimitive.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Spliterator,Spliterator>>++
diff --git a/docs/reference/painless-api-reference/Spliterator.asciidoc b/docs/reference/painless-api-reference/Spliterator.asciidoc
new file mode 100644
index 00000000000..fcf91940985
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterator.asciidoc
@@ -0,0 +1,23 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterator]]++Spliterator++::
+** [[painless-api-reference-Spliterator-CONCURRENT]]static int link:{java8-javadoc}/java/util/Spliterator.html#CONCURRENT[CONCURRENT] (link:{java9-javadoc}/java/util/Spliterator.html#CONCURRENT[java 9])
+** [[painless-api-reference-Spliterator-DISTINCT]]static int link:{java8-javadoc}/java/util/Spliterator.html#DISTINCT[DISTINCT] (link:{java9-javadoc}/java/util/Spliterator.html#DISTINCT[java 9])
+** [[painless-api-reference-Spliterator-IMMUTABLE]]static int link:{java8-javadoc}/java/util/Spliterator.html#IMMUTABLE[IMMUTABLE] (link:{java9-javadoc}/java/util/Spliterator.html#IMMUTABLE[java 9])
+** [[painless-api-reference-Spliterator-NONNULL]]static int link:{java8-javadoc}/java/util/Spliterator.html#NONNULL[NONNULL] (link:{java9-javadoc}/java/util/Spliterator.html#NONNULL[java 9])
+** [[painless-api-reference-Spliterator-ORDERED]]static int link:{java8-javadoc}/java/util/Spliterator.html#ORDERED[ORDERED] (link:{java9-javadoc}/java/util/Spliterator.html#ORDERED[java 9])
+** [[painless-api-reference-Spliterator-SIZED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SIZED[SIZED] (link:{java9-javadoc}/java/util/Spliterator.html#SIZED[java 9])
+** [[painless-api-reference-Spliterator-SORTED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SORTED[SORTED] (link:{java9-javadoc}/java/util/Spliterator.html#SORTED[java 9])
+** [[painless-api-reference-Spliterator-SUBSIZED]]static int link:{java8-javadoc}/java/util/Spliterator.html#SUBSIZED[SUBSIZED] (link:{java9-javadoc}/java/util/Spliterator.html#SUBSIZED[java 9])
+* ++[[painless-api-reference-Spliterator-characteristics-0]]int link:{java8-javadoc}/java/util/Spliterator.html#characteristics%2D%2D[characteristics]()++ (link:{java9-javadoc}/java/util/Spliterator.html#characteristics%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-estimateSize-0]]long link:{java8-javadoc}/java/util/Spliterator.html#estimateSize%2D%2D[estimateSize]()++ (link:{java9-javadoc}/java/util/Spliterator.html#estimateSize%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-forEachRemaining-1]]void link:{java8-javadoc}/java/util/Spliterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[forEachRemaining](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Spliterator.html#forEachRemaining%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Spliterator-getComparator-0]]<<painless-api-reference-Comparator,Comparator>> link:{java8-javadoc}/java/util/Spliterator.html#getComparator%2D%2D[getComparator]()++ (link:{java9-javadoc}/java/util/Spliterator.html#getComparator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-getExactSizeIfKnown-0]]long link:{java8-javadoc}/java/util/Spliterator.html#getExactSizeIfKnown%2D%2D[getExactSizeIfKnown]()++ (link:{java9-javadoc}/java/util/Spliterator.html#getExactSizeIfKnown%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterator-hasCharacteristics-1]]boolean link:{java8-javadoc}/java/util/Spliterator.html#hasCharacteristics%2Dint%2D[hasCharacteristics](int)++ (link:{java9-javadoc}/java/util/Spliterator.html#hasCharacteristics%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterator-tryAdvance-1]]boolean link:{java8-javadoc}/java/util/Spliterator.html#tryAdvance%2Djava.util.function.Consumer%2D[tryAdvance](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/Spliterator.html#tryAdvance%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Spliterator-trySplit-0]]<<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterator.html#trySplit%2D%2D[trySplit]()++ (link:{java9-javadoc}/java/util/Spliterator.html#trySplit%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Spliterators.asciidoc b/docs/reference/painless-api-reference/Spliterators.asciidoc
new file mode 100644
index 00000000000..6d8202a5674
--- /dev/null
+++ b/docs/reference/painless-api-reference/Spliterators.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Spliterators]]++Spliterators++::
+* ++[[painless-api-reference-Spliterators-emptyDoubleSpliterator-0]]static <<painless-api-reference-Spliterator-OfDouble,Spliterator.OfDouble>> link:{java8-javadoc}/java/util/Spliterators.html#emptyDoubleSpliterator%2D%2D[emptyDoubleSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyDoubleSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptyIntSpliterator-0]]static <<painless-api-reference-Spliterator-OfInt,Spliterator.OfInt>> link:{java8-javadoc}/java/util/Spliterators.html#emptyIntSpliterator%2D%2D[emptyIntSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyIntSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptyLongSpliterator-0]]static <<painless-api-reference-Spliterator-OfLong,Spliterator.OfLong>> link:{java8-javadoc}/java/util/Spliterators.html#emptyLongSpliterator%2D%2D[emptyLongSpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptyLongSpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-emptySpliterator-0]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#emptySpliterator%2D%2D[emptySpliterator]()++ (link:{java9-javadoc}/java/util/Spliterators.html#emptySpliterator%2D%2D[java 9])
+* ++[[painless-api-reference-Spliterators-iterator-1]]static <<painless-api-reference-Iterator,Iterator>> link:{java8-javadoc}/java/util/Spliterators.html#iterator%2Djava.util.Spliterator%2D[iterator](<<painless-api-reference-Spliterator,Spliterator>>)++ (link:{java9-javadoc}/java/util/Spliterators.html#iterator%2Djava.util.Spliterator%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliterator-2]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Collection%2Dint%2D[spliterator](<<painless-api-reference-Collection,Collection>>, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Collection%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliterator-3]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Iterator%2Dlong%2Dint%2D[spliterator](<<painless-api-reference-Iterator,Iterator>>, long, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliterator%2Djava.util.Iterator%2Dlong%2Dint%2D[java 9])
+* ++[[painless-api-reference-Spliterators-spliteratorUnknownSize-2]]static <<painless-api-reference-Spliterator,Spliterator>> link:{java8-javadoc}/java/util/Spliterators.html#spliteratorUnknownSize%2Djava.util.Iterator%2Dint%2D[spliteratorUnknownSize](<<painless-api-reference-Iterator,Iterator>>, int)++ (link:{java9-javadoc}/java/util/Spliterators.html#spliteratorUnknownSize%2Djava.util.Iterator%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Stack.asciidoc b/docs/reference/painless-api-reference/Stack.asciidoc
new file mode 100644
index 00000000000..ad1520d9d88
--- /dev/null
+++ b/docs/reference/painless-api-reference/Stack.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stack]]++Stack++::
+* ++[[painless-api-reference-Stack-Stack-0]]link:{java8-javadoc}/java/util/Stack.html#Stack%2D%2D[Stack]()++ (link:{java9-javadoc}/java/util/Stack.html#Stack%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-empty-0]]boolean link:{java8-javadoc}/java/util/Stack.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/Stack.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-peek-0]]def link:{java8-javadoc}/java/util/Stack.html#peek%2D%2D[peek]()++ (link:{java9-javadoc}/java/util/Stack.html#peek%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-pop-0]]def link:{java8-javadoc}/java/util/Stack.html#pop%2D%2D[pop]()++ (link:{java9-javadoc}/java/util/Stack.html#pop%2D%2D[java 9])
+* ++[[painless-api-reference-Stack-push-1]]def link:{java8-javadoc}/java/util/Stack.html#push%2Djava.lang.Object%2D[push](def)++ (link:{java9-javadoc}/java/util/Stack.html#push%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Stack-search-1]]int link:{java8-javadoc}/java/util/Stack.html#search%2Djava.lang.Object%2D[search](def)++ (link:{java9-javadoc}/java/util/Stack.html#search%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Vector,Vector>>++
diff --git a/docs/reference/painless-api-reference/StackTraceElement.asciidoc b/docs/reference/painless-api-reference/StackTraceElement.asciidoc
new file mode 100644
index 00000000000..046e56ef1f0
--- /dev/null
+++ b/docs/reference/painless-api-reference/StackTraceElement.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StackTraceElement]]++StackTraceElement++::
+* ++[[painless-api-reference-StackTraceElement-StackTraceElement-4]]link:{java8-javadoc}/java/lang/StackTraceElement.html#StackTraceElement%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2Dint%2D[StackTraceElement](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#StackTraceElement%2Djava.lang.String%2Djava.lang.String%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getClassName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getClassName%2D%2D[getClassName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getClassName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getFileName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getFileName%2D%2D[getFileName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getFileName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getLineNumber-0]]int link:{java8-javadoc}/java/lang/StackTraceElement.html#getLineNumber%2D%2D[getLineNumber]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getLineNumber%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-getMethodName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StackTraceElement.html#getMethodName%2D%2D[getMethodName]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#getMethodName%2D%2D[java 9])
+* ++[[painless-api-reference-StackTraceElement-isNativeMethod-0]]boolean link:{java8-javadoc}/java/lang/StackTraceElement.html#isNativeMethod%2D%2D[isNativeMethod]()++ (link:{java9-javadoc}/java/lang/StackTraceElement.html#isNativeMethod%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Stream.Builder.asciidoc b/docs/reference/painless-api-reference/Stream.Builder.asciidoc
new file mode 100644
index 00000000000..aa9146ce8ed
--- /dev/null
+++ b/docs/reference/painless-api-reference/Stream.Builder.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stream-Builder]]++Stream.Builder++::
+* ++[[painless-api-reference-Stream-Builder-add-1]]<<painless-api-reference-Stream-Builder,Stream.Builder>> link:{java8-javadoc}/java/util/stream/Stream$Builder.html#add%2Djava.lang.Object%2D[add](def)++ (link:{java9-javadoc}/java/util/stream/Stream$Builder.html#add%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Stream-Builder-build-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream$Builder.html#build%2D%2D[build]()++ (link:{java9-javadoc}/java/util/stream/Stream$Builder.html#build%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Consumer,Consumer>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Stream.asciidoc b/docs/reference/painless-api-reference/Stream.asciidoc
new file mode 100644
index 00000000000..0713ad1ce9d
--- /dev/null
+++ b/docs/reference/painless-api-reference/Stream.asciidoc
@@ -0,0 +1,43 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Stream]]++Stream++::
+* ++[[painless-api-reference-Stream-builder-0]]static <<painless-api-reference-Stream-Builder,Stream.Builder>> link:{java8-javadoc}/java/util/stream/Stream.html#builder%2D%2D[builder]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#builder%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-concat-2]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#concat%2Djava.util.stream.Stream%2Djava.util.stream.Stream%2D[concat](<<painless-api-reference-Stream,Stream>>, <<painless-api-reference-Stream,Stream>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#concat%2Djava.util.stream.Stream%2Djava.util.stream.Stream%2D[java 9])
+* ++[[painless-api-reference-Stream-empty-0]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#empty%2D%2D[empty]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#empty%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-of-1]]static <<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#of%2Djava.lang.Object:A%2D[of](def[])++ (link:{java9-javadoc}/java/util/stream/Stream.html#of%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Stream-allMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#allMatch%2Djava.util.function.Predicate%2D[allMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#allMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-anyMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#anyMatch%2Djava.util.function.Predicate%2D[anyMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#anyMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-collect-1]]def link:{java8-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.stream.Collector%2D[collect](<<painless-api-reference-Collector,Collector>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.stream.Collector%2D[java 9])
+* ++[[painless-api-reference-Stream-collect-3]]def link:{java8-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BiConsumer%2D[collect](<<painless-api-reference-Supplier,Supplier>>, <<painless-api-reference-BiConsumer,BiConsumer>>, <<painless-api-reference-BiConsumer,BiConsumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#collect%2Djava.util.function.Supplier%2Djava.util.function.BiConsumer%2Djava.util.function.BiConsumer%2D[java 9])
+* ++[[painless-api-reference-Stream-count-0]]long link:{java8-javadoc}/java/util/stream/Stream.html#count%2D%2D[count]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#count%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-distinct-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#distinct%2D%2D[distinct]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#distinct%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-filter-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#filter%2Djava.util.function.Predicate%2D[filter](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#filter%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-findAny-0]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#findAny%2D%2D[findAny]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#findAny%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-findFirst-0]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#findFirst%2D%2D[findFirst]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#findFirst%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMap-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMap%2Djava.util.function.Function%2D[flatMap](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMap%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToDouble%2Djava.util.function.Function%2D[flatMapToDouble](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToDouble%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToInt%2Djava.util.function.Function%2D[flatMapToInt](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToInt%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-flatMapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/Stream.html#flatMapToLong%2Djava.util.function.Function%2D[flatMapToLong](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#flatMapToLong%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-forEach-1]]void link:{java8-javadoc}/java/util/stream/Stream.html#forEach%2Djava.util.function.Consumer%2D[forEach](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#forEach%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-forEachOrdered-1]]void link:{java8-javadoc}/java/util/stream/Stream.html#forEachOrdered%2Djava.util.function.Consumer%2D[forEachOrdered](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#forEachOrdered%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-limit-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#limit%2Dlong%2D[limit](long)++ (link:{java9-javadoc}/java/util/stream/Stream.html#limit%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Stream-map-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#map%2Djava.util.function.Function%2D[map](<<painless-api-reference-Function,Function>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#map%2Djava.util.function.Function%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToDouble-1]]<<painless-api-reference-DoubleStream,DoubleStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToDouble%2Djava.util.function.ToDoubleFunction%2D[mapToDouble](<<painless-api-reference-ToDoubleFunction,ToDoubleFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToDouble%2Djava.util.function.ToDoubleFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToInt-1]]<<painless-api-reference-IntStream,IntStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToInt%2Djava.util.function.ToIntFunction%2D[mapToInt](<<painless-api-reference-ToIntFunction,ToIntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToInt%2Djava.util.function.ToIntFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-mapToLong-1]]<<painless-api-reference-LongStream,LongStream>> link:{java8-javadoc}/java/util/stream/Stream.html#mapToLong%2Djava.util.function.ToLongFunction%2D[mapToLong](<<painless-api-reference-ToLongFunction,ToLongFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#mapToLong%2Djava.util.function.ToLongFunction%2D[java 9])
+* ++[[painless-api-reference-Stream-max-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#max%2Djava.util.Comparator%2D[max](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#max%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-min-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#min%2Djava.util.Comparator%2D[min](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#min%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-noneMatch-1]]boolean link:{java8-javadoc}/java/util/stream/Stream.html#noneMatch%2Djava.util.function.Predicate%2D[noneMatch](<<painless-api-reference-Predicate,Predicate>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#noneMatch%2Djava.util.function.Predicate%2D[java 9])
+* ++[[painless-api-reference-Stream-peek-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#peek%2Djava.util.function.Consumer%2D[peek](<<painless-api-reference-Consumer,Consumer>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#peek%2Djava.util.function.Consumer%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-1]]<<painless-api-reference-Optional,Optional>> link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.util.function.BinaryOperator%2D[reduce](<<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-2]]def link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[reduce](def, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-reduce-3]]def link:{java8-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BiFunction%2Djava.util.function.BinaryOperator%2D[reduce](def, <<painless-api-reference-BiFunction,BiFunction>>, <<painless-api-reference-BinaryOperator,BinaryOperator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#reduce%2Djava.lang.Object%2Djava.util.function.BiFunction%2Djava.util.function.BinaryOperator%2D[java 9])
+* ++[[painless-api-reference-Stream-skip-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#skip%2Dlong%2D[skip](long)++ (link:{java9-javadoc}/java/util/stream/Stream.html#skip%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Stream-sorted-0]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#sorted%2D%2D[sorted]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#sorted%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-sorted-1]]<<painless-api-reference-Stream,Stream>> link:{java8-javadoc}/java/util/stream/Stream.html#sorted%2Djava.util.Comparator%2D[sorted](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#sorted%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-Stream-toArray-0]]def[] link:{java8-javadoc}/java/util/stream/Stream.html#toArray%2D%2D[toArray]()++ (link:{java9-javadoc}/java/util/stream/Stream.html#toArray%2D%2D[java 9])
+* ++[[painless-api-reference-Stream-toArray-1]]def[] link:{java8-javadoc}/java/util/stream/Stream.html#toArray%2Djava.util.function.IntFunction%2D[toArray](<<painless-api-reference-IntFunction,IntFunction>>)++ (link:{java9-javadoc}/java/util/stream/Stream.html#toArray%2Djava.util.function.IntFunction%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-BaseStream,BaseStream>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StrictMath.asciidoc b/docs/reference/painless-api-reference/StrictMath.asciidoc
new file mode 100644
index 00000000000..17346c33ba3
--- /dev/null
+++ b/docs/reference/painless-api-reference/StrictMath.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StrictMath]]++StrictMath++::
+** [[painless-api-reference-StrictMath-E]]static double link:{java8-javadoc}/java/lang/StrictMath.html#E[E] (link:{java9-javadoc}/java/lang/StrictMath.html#E[java 9])
+** [[painless-api-reference-StrictMath-PI]]static double link:{java8-javadoc}/java/lang/StrictMath.html#PI[PI] (link:{java9-javadoc}/java/lang/StrictMath.html#PI[java 9])
+* ++[[painless-api-reference-StrictMath-IEEEremainder-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#IEEEremainder%2Ddouble%2Ddouble%2D[IEEEremainder](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#IEEEremainder%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-abs-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#abs%2Ddouble%2D[abs](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#abs%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-acos-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#acos%2Ddouble%2D[acos](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#acos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-asin-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#asin%2Ddouble%2D[asin](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#asin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-atan-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#atan%2Ddouble%2D[atan](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#atan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-atan2-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#atan2%2Ddouble%2Ddouble%2D[atan2](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#atan2%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cbrt-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cbrt%2Ddouble%2D[cbrt](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cbrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-ceil-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#ceil%2Ddouble%2D[ceil](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#ceil%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-copySign-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#copySign%2Ddouble%2Ddouble%2D[copySign](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#copySign%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cos-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cos%2Ddouble%2D[cos](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cos%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-cosh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#cosh%2Ddouble%2D[cosh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#cosh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-exp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#exp%2Ddouble%2D[exp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#exp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-expm1-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#expm1%2Ddouble%2D[expm1](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#expm1%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-floor-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#floor%2Ddouble%2D[floor](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#floor%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-hypot-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#hypot%2Ddouble%2Ddouble%2D[hypot](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#hypot%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log%2Ddouble%2D[log](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log10-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log10%2Ddouble%2D[log10](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log10%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-log1p-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#log1p%2Ddouble%2D[log1p](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#log1p%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-max-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#max%2Ddouble%2Ddouble%2D[max](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#max%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-min-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#min%2Ddouble%2Ddouble%2D[min](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#min%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextAfter-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextAfter%2Ddouble%2Ddouble%2D[nextAfter](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextAfter%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextDown-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextDown%2Ddouble%2D[nextDown](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextDown%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-nextUp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#nextUp%2Ddouble%2D[nextUp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#nextUp%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-pow-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#pow%2Ddouble%2Ddouble%2D[pow](double, double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#pow%2Ddouble%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-random-0]]static double link:{java8-javadoc}/java/lang/StrictMath.html#random%2D%2D[random]()++ (link:{java9-javadoc}/java/lang/StrictMath.html#random%2D%2D[java 9])
+* ++[[painless-api-reference-StrictMath-rint-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#rint%2Ddouble%2D[rint](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#rint%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-round-1]]static long link:{java8-javadoc}/java/lang/StrictMath.html#round%2Ddouble%2D[round](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#round%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-scalb-2]]static double link:{java8-javadoc}/java/lang/StrictMath.html#scalb%2Ddouble%2Dint%2D[scalb](double, int)++ (link:{java9-javadoc}/java/lang/StrictMath.html#scalb%2Ddouble%2Dint%2D[java 9])
+* ++[[painless-api-reference-StrictMath-signum-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#signum%2Ddouble%2D[signum](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#signum%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sin-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sin%2Ddouble%2D[sin](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sin%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sinh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sinh%2Ddouble%2D[sinh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sinh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-sqrt-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#sqrt%2Ddouble%2D[sqrt](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#sqrt%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-tan-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#tan%2Ddouble%2D[tan](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#tan%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-tanh-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#tanh%2Ddouble%2D[tanh](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#tanh%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-toDegrees-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#toDegrees%2Ddouble%2D[toDegrees](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#toDegrees%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-toRadians-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#toRadians%2Ddouble%2D[toRadians](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#toRadians%2Ddouble%2D[java 9])
+* ++[[painless-api-reference-StrictMath-ulp-1]]static double link:{java8-javadoc}/java/lang/StrictMath.html#ulp%2Ddouble%2D[ulp](double)++ (link:{java9-javadoc}/java/lang/StrictMath.html#ulp%2Ddouble%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/String.asciidoc b/docs/reference/painless-api-reference/String.asciidoc
new file mode 100644
index 00000000000..a40e1811183
--- /dev/null
+++ b/docs/reference/painless-api-reference/String.asciidoc
@@ -0,0 +1,46 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-String]]++String++::
+* ++[[painless-api-reference-String-copyValueOf-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2D[copyValueOf](char[])++ (link:{java9-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2D[java 9])
+* ++[[painless-api-reference-String-copyValueOf-3]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2Dint%2Dint%2D[copyValueOf](char[], int, int)++ (link:{java9-javadoc}/java/lang/String.html#copyValueOf%2Dchar:A%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-format-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/lang/String.html#format%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-String-format-3]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[format](<<painless-api-reference-Locale,Locale>>, <<painless-api-reference-String,String>>, def[])++ (link:{java9-javadoc}/java/lang/String.html#format%2Djava.util.Locale%2Djava.lang.String%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-String-join-2]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#join%2Djava.lang.CharSequence%2Djava.lang.Iterable%2D[join](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-Iterable,Iterable>>)++ (link:{java9-javadoc}/java/lang/String.html#join%2Djava.lang.CharSequence%2Djava.lang.Iterable%2D[java 9])
+* ++[[painless-api-reference-String-valueOf-1]]static <<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#valueOf%2Djava.lang.Object%2D[valueOf](def)++ (link:{java9-javadoc}/java/lang/String.html#valueOf%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-String-String-0]]link:{java8-javadoc}/java/lang/String.html#String%2D%2D[String]()++ (link:{java9-javadoc}/java/lang/String.html#String%2D%2D[java 9])
+* ++[[painless-api-reference-String-codePointAt-1]]int link:{java8-javadoc}/java/lang/String.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/String.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-codePointBefore-1]]int link:{java8-javadoc}/java/lang/String.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/String.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-codePointCount-2]]int link:{java8-javadoc}/java/lang/String.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/String.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-compareTo-1]]int link:{java8-javadoc}/java/lang/String.html#compareTo%2Djava.lang.String%2D[compareTo](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#compareTo%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-compareToIgnoreCase-1]]int link:{java8-javadoc}/java/lang/String.html#compareToIgnoreCase%2Djava.lang.String%2D[compareToIgnoreCase](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#compareToIgnoreCase%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-concat-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#concat%2Djava.lang.String%2D[concat](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#concat%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-contains-1]]boolean link:{java8-javadoc}/java/lang/String.html#contains%2Djava.lang.CharSequence%2D[contains](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#contains%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-contentEquals-1]]boolean link:{java8-javadoc}/java/lang/String.html#contentEquals%2Djava.lang.CharSequence%2D[contentEquals](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#contentEquals%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-decodeBase64-0]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#decodeBase64%2Djava.lang.String%2D[decodeBase64]()++
+* ++[[painless-api-reference-String-encodeBase64-0]]<<painless-api-reference-String,String>> link:{painless-javadoc}/org/elasticsearch/painless/api/Augmentation.html#encodeBase64%2Djava.lang.String%2D[encodeBase64]()++
+* ++[[painless-api-reference-String-endsWith-1]]boolean link:{java8-javadoc}/java/lang/String.html#endsWith%2Djava.lang.String%2D[endsWith](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#endsWith%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-equalsIgnoreCase-1]]boolean link:{java8-javadoc}/java/lang/String.html#equalsIgnoreCase%2Djava.lang.String%2D[equalsIgnoreCase](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#equalsIgnoreCase%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-getChars-4]]void link:{java8-javadoc}/java/lang/String.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/String.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-indexOf-1]]int link:{java8-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-indexOf-2]]int link:{java8-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-isEmpty-0]]boolean link:{java8-javadoc}/java/lang/String.html#isEmpty%2D%2D[isEmpty]()++ (link:{java9-javadoc}/java/lang/String.html#isEmpty%2D%2D[java 9])
+* ++[[painless-api-reference-String-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/String.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/String.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-regionMatches-4]]boolean link:{java8-javadoc}/java/lang/String.html#regionMatches%2Dint%2Djava.lang.String%2Dint%2Dint%2D[regionMatches](int, <<painless-api-reference-String,String>>, int, int)++ (link:{java9-javadoc}/java/lang/String.html#regionMatches%2Dint%2Djava.lang.String%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-regionMatches-5]]boolean link:{java8-javadoc}/java/lang/String.html#regionMatches%2Dboolean%2Dint%2Djava.lang.String%2Dint%2Dint%2D[regionMatches](boolean, int, <<painless-api-reference-String,String>>, int, int)++ (link:{java9-javadoc}/java/lang/String.html#regionMatches%2Dboolean%2Dint%2Djava.lang.String%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-replace-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#replace%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[replace](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/String.html#replace%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-String-startsWith-1]]boolean link:{java8-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2D[startsWith](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-String-startsWith-2]]boolean link:{java8-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2Dint%2D[startsWith](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/String.html#startsWith%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/String.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/String.html#substring%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-String-toCharArray-0]]char[] link:{java8-javadoc}/java/lang/String.html#toCharArray%2D%2D[toCharArray]()++ (link:{java9-javadoc}/java/lang/String.html#toCharArray%2D%2D[java 9])
+* ++[[painless-api-reference-String-toLowerCase-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toLowerCase%2D%2D[toLowerCase]()++ (link:{java9-javadoc}/java/lang/String.html#toLowerCase%2D%2D[java 9])
+* ++[[painless-api-reference-String-toLowerCase-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toLowerCase%2Djava.util.Locale%2D[toLowerCase](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/lang/String.html#toLowerCase%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-String-toUpperCase-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toUpperCase%2D%2D[toUpperCase]()++ (link:{java9-javadoc}/java/lang/String.html#toUpperCase%2D%2D[java 9])
+* ++[[painless-api-reference-String-toUpperCase-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#toUpperCase%2Djava.util.Locale%2D[toUpperCase](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/lang/String.html#toUpperCase%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-String-trim-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/String.html#trim%2D%2D[trim]()++ (link:{java9-javadoc}/java/lang/String.html#trim%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringBuffer.asciidoc b/docs/reference/painless-api-reference/StringBuffer.asciidoc
new file mode 100644
index 00000000000..5993832a983
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringBuffer.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringBuffer]]++StringBuffer++::
+* ++[[painless-api-reference-StringBuffer-StringBuffer-0]]link:{java8-javadoc}/java/lang/StringBuffer.html#StringBuffer%2D%2D[StringBuffer]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#StringBuffer%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-StringBuffer-1]]link:{java8-javadoc}/java/lang/StringBuffer.html#StringBuffer%2Djava.lang.CharSequence%2D[StringBuffer](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#StringBuffer%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-append-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.Object%2D[append](def)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-append-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-appendCodePoint-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#appendCodePoint%2Dint%2D[appendCodePoint](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#appendCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-capacity-0]]int link:{java8-javadoc}/java/lang/StringBuffer.html#capacity%2D%2D[capacity]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#capacity%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointAt-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointBefore-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-codePointCount-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-delete-2]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#delete%2Dint%2Dint%2D[delete](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#delete%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-deleteCharAt-1]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#deleteCharAt%2Dint%2D[deleteCharAt](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#deleteCharAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-getChars-4]]void link:{java8-javadoc}/java/lang/StringBuffer.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-indexOf-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-indexOf-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-insert-2]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#insert%2Dint%2Djava.lang.Object%2D[insert](int, def)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#insert%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/StringBuffer.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-replace-3]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#replace%2Dint%2Dint%2Djava.lang.String%2D[replace](int, int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#replace%2Dint%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-reverse-0]]<<painless-api-reference-StringBuffer,StringBuffer>> link:{java8-javadoc}/java/lang/StringBuffer.html#reverse%2D%2D[reverse]()++ (link:{java9-javadoc}/java/lang/StringBuffer.html#reverse%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-setCharAt-2]]void link:{java8-javadoc}/java/lang/StringBuffer.html#setCharAt%2Dint%2Dchar%2D[setCharAt](int, char)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#setCharAt%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-setLength-1]]void link:{java8-javadoc}/java/lang/StringBuffer.html#setLength%2Dint%2D[setLength](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#setLength%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuffer-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/StringBuffer.html#substring%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringBuilder.asciidoc b/docs/reference/painless-api-reference/StringBuilder.asciidoc
new file mode 100644
index 00000000000..fd9c0c097cc
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringBuilder.asciidoc
@@ -0,0 +1,31 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringBuilder]]++StringBuilder++::
+* ++[[painless-api-reference-StringBuilder-StringBuilder-0]]link:{java8-javadoc}/java/lang/StringBuilder.html#StringBuilder%2D%2D[StringBuilder]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#StringBuilder%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-StringBuilder-1]]link:{java8-javadoc}/java/lang/StringBuilder.html#StringBuilder%2Djava.lang.CharSequence%2D[StringBuilder](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#StringBuilder%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-append-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.Object%2D[append](def)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-append-3]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[append](<<painless-api-reference-CharSequence,CharSequence>>, int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#append%2Djava.lang.CharSequence%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-appendCodePoint-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#appendCodePoint%2Dint%2D[appendCodePoint](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#appendCodePoint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-capacity-0]]int link:{java8-javadoc}/java/lang/StringBuilder.html#capacity%2D%2D[capacity]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#capacity%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointAt-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointAt%2Dint%2D[codePointAt](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointBefore-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointBefore%2Dint%2D[codePointBefore](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointBefore%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-codePointCount-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#codePointCount%2Dint%2Dint%2D[codePointCount](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#codePointCount%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-delete-2]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#delete%2Dint%2Dint%2D[delete](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#delete%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-deleteCharAt-1]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#deleteCharAt%2Dint%2D[deleteCharAt](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#deleteCharAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-getChars-4]]void link:{java8-javadoc}/java/lang/StringBuilder.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[getChars](int, int, char[], int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#getChars%2Dint%2Dint%2Dchar:A%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-indexOf-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2D[indexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-indexOf-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2Dint%2D[indexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#indexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-insert-2]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#insert%2Dint%2Djava.lang.Object%2D[insert](int, def)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#insert%2Dint%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-lastIndexOf-1]]int link:{java8-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2D[lastIndexOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-lastIndexOf-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2Dint%2D[lastIndexOf](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#lastIndexOf%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-offsetByCodePoints-2]]int link:{java8-javadoc}/java/lang/StringBuilder.html#offsetByCodePoints%2Dint%2Dint%2D[offsetByCodePoints](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#offsetByCodePoints%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-replace-3]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#replace%2Dint%2Dint%2Djava.lang.String%2D[replace](int, int, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#replace%2Dint%2Dint%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-reverse-0]]<<painless-api-reference-StringBuilder,StringBuilder>> link:{java8-javadoc}/java/lang/StringBuilder.html#reverse%2D%2D[reverse]()++ (link:{java9-javadoc}/java/lang/StringBuilder.html#reverse%2D%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-setCharAt-2]]void link:{java8-javadoc}/java/lang/StringBuilder.html#setCharAt%2Dint%2Dchar%2D[setCharAt](int, char)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#setCharAt%2Dint%2Dchar%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-setLength-1]]void link:{java8-javadoc}/java/lang/StringBuilder.html#setLength%2Dint%2D[setLength](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#setLength%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-substring-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2D[substring](int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringBuilder-substring-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2Dint%2D[substring](int, int)++ (link:{java9-javadoc}/java/lang/StringBuilder.html#substring%2Dint%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharSequence,CharSequence>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringCharacterIterator.asciidoc b/docs/reference/painless-api-reference/StringCharacterIterator.asciidoc
new file mode 100644
index 00000000000..8f65f2b6c5e
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringCharacterIterator.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringCharacterIterator]]++StringCharacterIterator++::
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-1]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2D[StringCharacterIterator](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-2]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2D[StringCharacterIterator](<<painless-api-reference-String,String>>, int)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-StringCharacterIterator-4]]link:{java8-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2Dint%2Dint%2D[StringCharacterIterator](<<painless-api-reference-String,String>>, int, int, int)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#StringCharacterIterator%2Djava.lang.String%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-StringCharacterIterator-setText-1]]void link:{java8-javadoc}/java/text/StringCharacterIterator.html#setText%2Djava.lang.String%2D[setText](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/text/StringCharacterIterator.html#setText%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-CharacterIterator,CharacterIterator>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringIndexOutOfBoundsException.asciidoc b/docs/reference/painless-api-reference/StringIndexOutOfBoundsException.asciidoc
new file mode 100644
index 00000000000..a29cae3f857
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringIndexOutOfBoundsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringIndexOutOfBoundsException]]++StringIndexOutOfBoundsException++::
+* ++[[painless-api-reference-StringIndexOutOfBoundsException-StringIndexOutOfBoundsException-0]]link:{java8-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2D%2D[StringIndexOutOfBoundsException]()++ (link:{java9-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2D%2D[java 9])
+* ++[[painless-api-reference-StringIndexOutOfBoundsException-StringIndexOutOfBoundsException-1]]link:{java8-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2Djava.lang.String%2D[StringIndexOutOfBoundsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/StringIndexOutOfBoundsException.html#StringIndexOutOfBoundsException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringJoiner.asciidoc b/docs/reference/painless-api-reference/StringJoiner.asciidoc
new file mode 100644
index 00000000000..7bd9baa4460
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringJoiner.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringJoiner]]++StringJoiner++::
+* ++[[painless-api-reference-StringJoiner-StringJoiner-1]]link:{java8-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2D[StringJoiner](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-StringJoiner-3]]link:{java8-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[StringJoiner](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#StringJoiner%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-add-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#add%2Djava.lang.CharSequence%2D[add](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#add%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-length-0]]int link:{java8-javadoc}/java/util/StringJoiner.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/util/StringJoiner.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-merge-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#merge%2Djava.util.StringJoiner%2D[merge](<<painless-api-reference-StringJoiner,StringJoiner>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#merge%2Djava.util.StringJoiner%2D[java 9])
+* ++[[painless-api-reference-StringJoiner-setEmptyValue-1]]<<painless-api-reference-StringJoiner,StringJoiner>> link:{java8-javadoc}/java/util/StringJoiner.html#setEmptyValue%2Djava.lang.CharSequence%2D[setEmptyValue](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/util/StringJoiner.html#setEmptyValue%2Djava.lang.CharSequence%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/StringTokenizer.asciidoc b/docs/reference/painless-api-reference/StringTokenizer.asciidoc
new file mode 100644
index 00000000000..9462cecc33b
--- /dev/null
+++ b/docs/reference/painless-api-reference/StringTokenizer.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-StringTokenizer]]++StringTokenizer++::
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-1]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2D[StringTokenizer](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-2]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2D[StringTokenizer](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-StringTokenizer-3]]link:{java8-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2Dboolean%2D[StringTokenizer](<<painless-api-reference-String,String>>, <<painless-api-reference-String,String>>, boolean)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#StringTokenizer%2Djava.lang.String%2Djava.lang.String%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-countTokens-0]]int link:{java8-javadoc}/java/util/StringTokenizer.html#countTokens%2D%2D[countTokens]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#countTokens%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-hasMoreTokens-0]]boolean link:{java8-javadoc}/java/util/StringTokenizer.html#hasMoreTokens%2D%2D[hasMoreTokens]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#hasMoreTokens%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-nextToken-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/StringTokenizer.html#nextToken%2D%2D[nextToken]()++ (link:{java9-javadoc}/java/util/StringTokenizer.html#nextToken%2D%2D[java 9])
+* ++[[painless-api-reference-StringTokenizer-nextToken-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/StringTokenizer.html#nextToken%2Djava.lang.String%2D[nextToken](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/StringTokenizer.html#nextToken%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enumeration,Enumeration>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Supplier.asciidoc b/docs/reference/painless-api-reference/Supplier.asciidoc
new file mode 100644
index 00000000000..ba3cd9da951
--- /dev/null
+++ b/docs/reference/painless-api-reference/Supplier.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Supplier]]++Supplier++::
+* ++[[painless-api-reference-Supplier-get-0]]def link:{java8-javadoc}/java/util/function/Supplier.html#get%2D%2D[get]()++ (link:{java9-javadoc}/java/util/function/Supplier.html#get%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/System.asciidoc b/docs/reference/painless-api-reference/System.asciidoc
new file mode 100644
index 00000000000..a791d5dc159
--- /dev/null
+++ b/docs/reference/painless-api-reference/System.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-System]]++System++::
+* ++[[painless-api-reference-System-arraycopy-5]]static void link:{java8-javadoc}/java/lang/System.html#arraycopy%2Djava.lang.Object%2Dint%2Djava.lang.Object%2Dint%2Dint%2D[arraycopy](<<painless-api-reference-Object,Object>>, int, <<painless-api-reference-Object,Object>>, int, int)++ (link:{java9-javadoc}/java/lang/System.html#arraycopy%2Djava.lang.Object%2Dint%2Djava.lang.Object%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-System-currentTimeMillis-0]]static long link:{java8-javadoc}/java/lang/System.html#currentTimeMillis%2D%2D[currentTimeMillis]()++ (link:{java9-javadoc}/java/lang/System.html#currentTimeMillis%2D%2D[java 9])
+* ++[[painless-api-reference-System-nanoTime-0]]static long link:{java8-javadoc}/java/lang/System.html#nanoTime%2D%2D[nanoTime]()++ (link:{java9-javadoc}/java/lang/System.html#nanoTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Temporal.asciidoc b/docs/reference/painless-api-reference/Temporal.asciidoc
new file mode 100644
index 00000000000..ce693cfde51
--- /dev/null
+++ b/docs/reference/painless-api-reference/Temporal.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Temporal]]++Temporal++::
+* ++[[painless-api-reference-Temporal-minus-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Temporal-minus-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-plus-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Temporal-plus-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-until-2]]long link:{java8-javadoc}/java/time/temporal/Temporal.html#until%2Djava.time.temporal.Temporal%2Djava.time.temporal.TemporalUnit%2D[until](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#until%2Djava.time.temporal.Temporal%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Temporal-with-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Temporal-with-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/temporal/Temporal.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/reference/painless-api-reference/TemporalAccessor.asciidoc b/docs/reference/painless-api-reference/TemporalAccessor.asciidoc
new file mode 100644
index 00000000000..7583b2180d9
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalAccessor.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAccessor]]++TemporalAccessor++::
+* ++[[painless-api-reference-TemporalAccessor-get-1]]int link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#get%2Djava.time.temporal.TemporalField%2D[get](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#get%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-getLong-1]]long link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#getLong%2Djava.time.temporal.TemporalField%2D[getLong](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#getLong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-isSupported-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#isSupported%2Djava.time.temporal.TemporalField%2D[isSupported](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#isSupported%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-query-1]]def link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#query%2Djava.time.temporal.TemporalQuery%2D[query](<<painless-api-reference-TemporalQuery,TemporalQuery>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#query%2Djava.time.temporal.TemporalQuery%2D[java 9])
+* ++[[painless-api-reference-TemporalAccessor-range-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalAccessor.html#range%2Djava.time.temporal.TemporalField%2D[range](<<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAccessor.html#range%2Djava.time.temporal.TemporalField%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalAdjuster.asciidoc b/docs/reference/painless-api-reference/TemporalAdjuster.asciidoc
new file mode 100644
index 00000000000..b80f0d7afbf
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalAdjuster.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAdjuster]]++TemporalAdjuster++::
+* ++[[painless-api-reference-TemporalAdjuster-adjustInto-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAdjuster.html#adjustInto%2Djava.time.temporal.Temporal%2D[adjustInto](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjuster.html#adjustInto%2Djava.time.temporal.Temporal%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalAdjusters.asciidoc b/docs/reference/painless-api-reference/TemporalAdjusters.asciidoc
new file mode 100644
index 00000000000..7299ae9256a
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalAdjusters.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAdjusters]]++TemporalAdjusters++::
+* ++[[painless-api-reference-TemporalAdjusters-dayOfWeekInMonth-2]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#dayOfWeekInMonth%2Dint%2Djava.time.DayOfWeek%2D[dayOfWeekInMonth](int, <<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#dayOfWeekInMonth%2Dint%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfMonth%2D%2D[firstDayOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfNextMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextMonth%2D%2D[firstDayOfNextMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfNextYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextYear%2D%2D[firstDayOfNextYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfNextYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstDayOfYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfYear%2D%2D[firstDayOfYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-firstInMonth-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#firstInMonth%2Djava.time.DayOfWeek%2D[firstInMonth](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#firstInMonth%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastDayOfMonth-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfMonth%2D%2D[lastDayOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastDayOfYear-0]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfYear%2D%2D[lastDayOfYear]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-lastInMonth-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#lastInMonth%2Djava.time.DayOfWeek%2D[lastInMonth](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#lastInMonth%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-next-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#next%2Djava.time.DayOfWeek%2D[next](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#next%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-nextOrSame-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#nextOrSame%2Djava.time.DayOfWeek%2D[nextOrSame](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#nextOrSame%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-ofDateAdjuster-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#ofDateAdjuster%2Djava.util.function.UnaryOperator%2D[ofDateAdjuster](<<painless-api-reference-UnaryOperator,UnaryOperator>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#ofDateAdjuster%2Djava.util.function.UnaryOperator%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-previous-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#previous%2Djava.time.DayOfWeek%2D[previous](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#previous%2Djava.time.DayOfWeek%2D[java 9])
+* ++[[painless-api-reference-TemporalAdjusters-previousOrSame-1]]static <<painless-api-reference-TemporalAdjuster,TemporalAdjuster>> link:{java8-javadoc}/java/time/temporal/TemporalAdjusters.html#previousOrSame%2Djava.time.DayOfWeek%2D[previousOrSame](<<painless-api-reference-DayOfWeek,DayOfWeek>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAdjusters.html#previousOrSame%2Djava.time.DayOfWeek%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalAmount.asciidoc b/docs/reference/painless-api-reference/TemporalAmount.asciidoc
new file mode 100644
index 00000000000..e27954b4db3
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalAmount.asciidoc
@@ -0,0 +1,11 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalAmount]]++TemporalAmount++::
+* ++[[painless-api-reference-TemporalAmount-addTo-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#addTo%2Djava.time.temporal.Temporal%2D[addTo](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#addTo%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-get-1]]long link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#get%2Djava.time.temporal.TemporalUnit%2D[get](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#get%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-getUnits-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#getUnits%2D%2D[getUnits]()++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#getUnits%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalAmount-subtractFrom-1]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalAmount.html#subtractFrom%2Djava.time.temporal.Temporal%2D[subtractFrom](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalAmount.html#subtractFrom%2Djava.time.temporal.Temporal%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalField.asciidoc b/docs/reference/painless-api-reference/TemporalField.asciidoc
new file mode 100644
index 00000000000..dd38544d46e
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalField.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalField]]++TemporalField++::
+* ++[[painless-api-reference-TemporalField-adjustInto-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#adjustInto%2Djava.time.temporal.Temporal%2Dlong%2D[adjustInto](<<painless-api-reference-Temporal,Temporal>>, long)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#adjustInto%2Djava.time.temporal.Temporal%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getBaseUnit-0]]<<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getBaseUnit%2D%2D[getBaseUnit]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getBaseUnit%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getFrom-1]]long link:{java8-javadoc}/java/time/temporal/TemporalField.html#getFrom%2Djava.time.temporal.TemporalAccessor%2D[getFrom](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getFrom%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-getRangeUnit-0]]<<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#getRangeUnit%2D%2D[getRangeUnit]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#getRangeUnit%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isDateBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isDateBased%2D%2D[isDateBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isDateBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isSupportedBy-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isSupportedBy%2Djava.time.temporal.TemporalAccessor%2D[isSupportedBy](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isSupportedBy%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-isTimeBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalField.html#isTimeBased%2D%2D[isTimeBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#isTimeBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-range-0]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#range%2D%2D[range]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#range%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalField-rangeRefinedBy-1]]<<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#rangeRefinedBy%2Djava.time.temporal.TemporalAccessor%2D[rangeRefinedBy](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#rangeRefinedBy%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-TemporalField-resolve-3]]<<painless-api-reference-TemporalAccessor,TemporalAccessor>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#resolve%2Djava.util.Map%2Djava.time.temporal.TemporalAccessor%2Djava.time.format.ResolverStyle%2D[resolve](<<painless-api-reference-Map,Map>>, <<painless-api-reference-TemporalAccessor,TemporalAccessor>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#resolve%2Djava.util.Map%2Djava.time.temporal.TemporalAccessor%2Djava.time.format.ResolverStyle%2D[java 9])
+* ++[[painless-api-reference-TemporalField-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalField.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/temporal/TemporalField.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalQueries.asciidoc b/docs/reference/painless-api-reference/TemporalQueries.asciidoc
new file mode 100644
index 00000000000..bd01ba6c2e0
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalQueries.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalQueries]]++TemporalQueries++::
+* ++[[painless-api-reference-TemporalQueries-chronology-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#chronology%2D%2D[chronology]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#chronology%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-localDate-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#localDate%2D%2D[localDate]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#localDate%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-localTime-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#localTime%2D%2D[localTime]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#localTime%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-offset-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#offset%2D%2D[offset]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#offset%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-precision-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#precision%2D%2D[precision]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#precision%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-zone-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#zone%2D%2D[zone]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#zone%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalQueries-zoneId-0]]static <<painless-api-reference-TemporalQuery,TemporalQuery>> link:{java8-javadoc}/java/time/temporal/TemporalQueries.html#zoneId%2D%2D[zoneId]()++ (link:{java9-javadoc}/java/time/temporal/TemporalQueries.html#zoneId%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalQuery.asciidoc b/docs/reference/painless-api-reference/TemporalQuery.asciidoc
new file mode 100644
index 00000000000..ca09eff368d
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalQuery.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalQuery]]++TemporalQuery++::
+* ++[[painless-api-reference-TemporalQuery-queryFrom-1]]def link:{java8-javadoc}/java/time/temporal/TemporalQuery.html#queryFrom%2Djava.time.temporal.TemporalAccessor%2D[queryFrom](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalQuery.html#queryFrom%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TemporalUnit.asciidoc b/docs/reference/painless-api-reference/TemporalUnit.asciidoc
new file mode 100644
index 00000000000..cc940d838ef
--- /dev/null
+++ b/docs/reference/painless-api-reference/TemporalUnit.asciidoc
@@ -0,0 +1,15 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TemporalUnit]]++TemporalUnit++::
+* ++[[painless-api-reference-TemporalUnit-addTo-2]]<<painless-api-reference-Temporal,Temporal>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#addTo%2Djava.time.temporal.Temporal%2Dlong%2D[addTo](<<painless-api-reference-Temporal,Temporal>>, long)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#addTo%2Djava.time.temporal.Temporal%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-between-2]]long link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[between](<<painless-api-reference-Temporal,Temporal>>, <<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#between%2Djava.time.temporal.Temporal%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-getDuration-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#getDuration%2D%2D[getDuration]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#getDuration%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isDateBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isDateBased%2D%2D[isDateBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isDateBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isDurationEstimated-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isDurationEstimated%2D%2D[isDurationEstimated]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isDurationEstimated%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isSupportedBy-1]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isSupportedBy%2Djava.time.temporal.Temporal%2D[isSupportedBy](<<painless-api-reference-Temporal,Temporal>>)++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isSupportedBy%2Djava.time.temporal.Temporal%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-isTimeBased-0]]boolean link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#isTimeBased%2D%2D[isTimeBased]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#isTimeBased%2D%2D[java 9])
+* ++[[painless-api-reference-TemporalUnit-toString-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/temporal/TemporalUnit.html#toString%2D%2D[toString]()++ (link:{java9-javadoc}/java/time/temporal/TemporalUnit.html#toString%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TextStyle.asciidoc b/docs/reference/painless-api-reference/TextStyle.asciidoc
new file mode 100644
index 00000000000..c7b122f9345
--- /dev/null
+++ b/docs/reference/painless-api-reference/TextStyle.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TextStyle]]++TextStyle++::
+** [[painless-api-reference-TextStyle-FULL]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#FULL[FULL] (link:{java9-javadoc}/java/time/format/TextStyle.html#FULL[java 9])
+** [[painless-api-reference-TextStyle-FULL_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#FULL_STANDALONE[FULL_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#FULL_STANDALONE[java 9])
+** [[painless-api-reference-TextStyle-NARROW]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#NARROW[NARROW] (link:{java9-javadoc}/java/time/format/TextStyle.html#NARROW[java 9])
+** [[painless-api-reference-TextStyle-NARROW_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#NARROW_STANDALONE[NARROW_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#NARROW_STANDALONE[java 9])
+** [[painless-api-reference-TextStyle-SHORT]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#SHORT[SHORT] (link:{java9-javadoc}/java/time/format/TextStyle.html#SHORT[java 9])
+** [[painless-api-reference-TextStyle-SHORT_STANDALONE]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#SHORT_STANDALONE[SHORT_STANDALONE] (link:{java9-javadoc}/java/time/format/TextStyle.html#SHORT_STANDALONE[java 9])
+* ++[[painless-api-reference-TextStyle-valueOf-1]]static <<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/format/TextStyle.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-TextStyle-values-0]]static <<painless-api-reference-TextStyle,TextStyle>>[] link:{java8-javadoc}/java/time/format/TextStyle.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-asNormal-0]]<<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#asNormal%2D%2D[asNormal]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#asNormal%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-asStandalone-0]]<<painless-api-reference-TextStyle,TextStyle>> link:{java8-javadoc}/java/time/format/TextStyle.html#asStandalone%2D%2D[asStandalone]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#asStandalone%2D%2D[java 9])
+* ++[[painless-api-reference-TextStyle-isStandalone-0]]boolean link:{java8-javadoc}/java/time/format/TextStyle.html#isStandalone%2D%2D[isStandalone]()++ (link:{java9-javadoc}/java/time/format/TextStyle.html#isStandalone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ThaiBuddhistChronology.asciidoc b/docs/reference/painless-api-reference/ThaiBuddhistChronology.asciidoc
new file mode 100644
index 00000000000..86c721fda7a
--- /dev/null
+++ b/docs/reference/painless-api-reference/ThaiBuddhistChronology.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistChronology]]++ThaiBuddhistChronology++::
+** [[painless-api-reference-ThaiBuddhistChronology-INSTANCE]]static <<painless-api-reference-ThaiBuddhistChronology,ThaiBuddhistChronology>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#INSTANCE[INSTANCE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#INSTANCE[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[date](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-3]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Dint%2Dint%2Dint%2D[date](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-date-4]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[date](<<painless-api-reference-Era,Era>>, int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#date%2Djava.time.chrono.Era%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateEpochDay-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateEpochDay%2Dlong%2D[dateEpochDay](long)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateEpochDay%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateYearDay-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Dint%2Dint%2D[dateYearDay](int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-dateYearDay-3]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[dateYearDay](<<painless-api-reference-Era,Era>>, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#dateYearDay%2Djava.time.chrono.Era%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-eraOf-1]]<<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#eraOf%2Dint%2D[eraOf](int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#eraOf%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistChronology-resolveDate-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[resolveDate](<<painless-api-reference-Map,Map>>, <<painless-api-reference-ResolverStyle,ResolverStyle>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistChronology.html#resolveDate%2Djava.util.Map%2Djava.time.format.ResolverStyle%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Chronology,Chronology>>++
diff --git a/docs/reference/painless-api-reference/ThaiBuddhistDate.asciidoc b/docs/reference/painless-api-reference/ThaiBuddhistDate.asciidoc
new file mode 100644
index 00000000000..94d620182df
--- /dev/null
+++ b/docs/reference/painless-api-reference/ThaiBuddhistDate.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistDate]]++ThaiBuddhistDate++::
+* ++[[painless-api-reference-ThaiBuddhistDate-from-1]]static <<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-of-3]]static <<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#of%2Dint%2Dint%2Dint%2D[of](int, int, int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#of%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-getChronology-0]]<<painless-api-reference-ThaiBuddhistChronology,ThaiBuddhistChronology>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getChronology%2D%2D[getChronology]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getChronology%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-getEra-0]]<<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getEra%2D%2D[getEra]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#getEra%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-minus-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-minus-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-plus-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-plus-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-with-1]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistDate-with-2]]<<painless-api-reference-ThaiBuddhistDate,ThaiBuddhistDate>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistDate.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoLocalDate,ChronoLocalDate>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/ThaiBuddhistEra.asciidoc b/docs/reference/painless-api-reference/ThaiBuddhistEra.asciidoc
new file mode 100644
index 00000000000..b6c070503a9
--- /dev/null
+++ b/docs/reference/painless-api-reference/ThaiBuddhistEra.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ThaiBuddhistEra]]++ThaiBuddhistEra++::
+** [[painless-api-reference-ThaiBuddhistEra-BE]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BE[BE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BE[java 9])
+** [[painless-api-reference-ThaiBuddhistEra-BEFORE_BE]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BEFORE_BE[BEFORE_BE] (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#BEFORE_BE[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-of-1]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-valueOf-1]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>> link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-values-0]]static <<painless-api-reference-ThaiBuddhistEra,ThaiBuddhistEra>>[] link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ThaiBuddhistEra-getValue-0]]int link:{java8-javadoc}/java/time/chrono/ThaiBuddhistEra.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/chrono/ThaiBuddhistEra.html#getValue%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Era,Era>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/TimeZone.asciidoc b/docs/reference/painless-api-reference/TimeZone.asciidoc
new file mode 100644
index 00000000000..5618e388a44
--- /dev/null
+++ b/docs/reference/painless-api-reference/TimeZone.asciidoc
@@ -0,0 +1,29 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TimeZone]]++TimeZone++::
+** [[painless-api-reference-TimeZone-LONG]]static int link:{java8-javadoc}/java/util/TimeZone.html#LONG[LONG] (link:{java9-javadoc}/java/util/TimeZone.html#LONG[java 9])
+** [[painless-api-reference-TimeZone-SHORT]]static int link:{java8-javadoc}/java/util/TimeZone.html#SHORT[SHORT] (link:{java9-javadoc}/java/util/TimeZone.html#SHORT[java 9])
+* ++[[painless-api-reference-TimeZone-getAvailableIDs-0]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/TimeZone.html#getAvailableIDs%2D%2D[getAvailableIDs]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getAvailableIDs%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getAvailableIDs-1]]static <<painless-api-reference-String,String>>[] link:{java8-javadoc}/java/util/TimeZone.html#getAvailableIDs%2Dint%2D[getAvailableIDs](int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getAvailableIDs%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDefault-0]]static <<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/TimeZone.html#getDefault%2D%2D[getDefault]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDefault%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getTimeZone-1]]static <<painless-api-reference-TimeZone,TimeZone>> link:{java8-javadoc}/java/util/TimeZone.html#getTimeZone%2Djava.lang.String%2D[getTimeZone](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getTimeZone%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-TimeZone-clone-0]]def link:{java8-javadoc}/java/util/TimeZone.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TimeZone.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDSTSavings-0]]int link:{java8-javadoc}/java/util/TimeZone.html#getDSTSavings%2D%2D[getDSTSavings]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDSTSavings%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2D%2D[getDisplayName]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2D[getDisplayName](boolean, int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getDisplayName-3]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2Djava.util.Locale%2D[getDisplayName](boolean, int, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#getDisplayName%2Dboolean%2Dint%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getID-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/TimeZone.html#getID%2D%2D[getID]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getID%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getOffset-1]]int link:{java8-javadoc}/java/util/TimeZone.html#getOffset%2Dlong%2D[getOffset](long)++ (link:{java9-javadoc}/java/util/TimeZone.html#getOffset%2Dlong%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getOffset-6]]int link:{java8-javadoc}/java/util/TimeZone.html#getOffset%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[getOffset](int, int, int, int, int, int)++ (link:{java9-javadoc}/java/util/TimeZone.html#getOffset%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-getRawOffset-0]]int link:{java8-javadoc}/java/util/TimeZone.html#getRawOffset%2D%2D[getRawOffset]()++ (link:{java9-javadoc}/java/util/TimeZone.html#getRawOffset%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-hasSameRules-1]]boolean link:{java8-javadoc}/java/util/TimeZone.html#hasSameRules%2Djava.util.TimeZone%2D[hasSameRules](<<painless-api-reference-TimeZone,TimeZone>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#hasSameRules%2Djava.util.TimeZone%2D[java 9])
+* ++[[painless-api-reference-TimeZone-inDaylightTime-1]]boolean link:{java8-javadoc}/java/util/TimeZone.html#inDaylightTime%2Djava.util.Date%2D[inDaylightTime](<<painless-api-reference-Date,Date>>)++ (link:{java9-javadoc}/java/util/TimeZone.html#inDaylightTime%2Djava.util.Date%2D[java 9])
+* ++[[painless-api-reference-TimeZone-observesDaylightTime-0]]boolean link:{java8-javadoc}/java/util/TimeZone.html#observesDaylightTime%2D%2D[observesDaylightTime]()++ (link:{java9-javadoc}/java/util/TimeZone.html#observesDaylightTime%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-setRawOffset-1]]void link:{java8-javadoc}/java/util/TimeZone.html#setRawOffset%2Dint%2D[setRawOffset](int)++ (link:{java9-javadoc}/java/util/TimeZone.html#setRawOffset%2Dint%2D[java 9])
+* ++[[painless-api-reference-TimeZone-toZoneId-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/util/TimeZone.html#toZoneId%2D%2D[toZoneId]()++ (link:{java9-javadoc}/java/util/TimeZone.html#toZoneId%2D%2D[java 9])
+* ++[[painless-api-reference-TimeZone-useDaylightTime-0]]boolean link:{java8-javadoc}/java/util/TimeZone.html#useDaylightTime%2D%2D[useDaylightTime]()++ (link:{java9-javadoc}/java/util/TimeZone.html#useDaylightTime%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToDoubleBiFunction.asciidoc b/docs/reference/painless-api-reference/ToDoubleBiFunction.asciidoc
new file mode 100644
index 00000000000..8ef0873fb5e
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToDoubleBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToDoubleBiFunction]]++ToDoubleBiFunction++::
+* ++[[painless-api-reference-ToDoubleBiFunction-applyAsDouble-2]]double link:{java8-javadoc}/java/util/function/ToDoubleBiFunction.html#applyAsDouble%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsDouble](def, def)++ (link:{java9-javadoc}/java/util/function/ToDoubleBiFunction.html#applyAsDouble%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToDoubleFunction.asciidoc b/docs/reference/painless-api-reference/ToDoubleFunction.asciidoc
new file mode 100644
index 00000000000..b0d66be9dc0
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToDoubleFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToDoubleFunction]]++ToDoubleFunction++::
+* ++[[painless-api-reference-ToDoubleFunction-applyAsDouble-1]]double link:{java8-javadoc}/java/util/function/ToDoubleFunction.html#applyAsDouble%2Djava.lang.Object%2D[applyAsDouble](def)++ (link:{java9-javadoc}/java/util/function/ToDoubleFunction.html#applyAsDouble%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToIntBiFunction.asciidoc b/docs/reference/painless-api-reference/ToIntBiFunction.asciidoc
new file mode 100644
index 00000000000..42cc396a1a9
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToIntBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToIntBiFunction]]++ToIntBiFunction++::
+* ++[[painless-api-reference-ToIntBiFunction-applyAsInt-2]]int link:{java8-javadoc}/java/util/function/ToIntBiFunction.html#applyAsInt%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsInt](def, def)++ (link:{java9-javadoc}/java/util/function/ToIntBiFunction.html#applyAsInt%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToIntFunction.asciidoc b/docs/reference/painless-api-reference/ToIntFunction.asciidoc
new file mode 100644
index 00000000000..c471de6b77d
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToIntFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToIntFunction]]++ToIntFunction++::
+* ++[[painless-api-reference-ToIntFunction-applyAsInt-1]]int link:{java8-javadoc}/java/util/function/ToIntFunction.html#applyAsInt%2Djava.lang.Object%2D[applyAsInt](def)++ (link:{java9-javadoc}/java/util/function/ToIntFunction.html#applyAsInt%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToLongBiFunction.asciidoc b/docs/reference/painless-api-reference/ToLongBiFunction.asciidoc
new file mode 100644
index 00000000000..ffda1353210
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToLongBiFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToLongBiFunction]]++ToLongBiFunction++::
+* ++[[painless-api-reference-ToLongBiFunction-applyAsLong-2]]long link:{java8-javadoc}/java/util/function/ToLongBiFunction.html#applyAsLong%2Djava.lang.Object%2Djava.lang.Object%2D[applyAsLong](def, def)++ (link:{java9-javadoc}/java/util/function/ToLongBiFunction.html#applyAsLong%2Djava.lang.Object%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ToLongFunction.asciidoc b/docs/reference/painless-api-reference/ToLongFunction.asciidoc
new file mode 100644
index 00000000000..785c414f8c9
--- /dev/null
+++ b/docs/reference/painless-api-reference/ToLongFunction.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ToLongFunction]]++ToLongFunction++::
+* ++[[painless-api-reference-ToLongFunction-applyAsLong-1]]long link:{java8-javadoc}/java/util/function/ToLongFunction.html#applyAsLong%2Djava.lang.Object%2D[applyAsLong](def)++ (link:{java9-javadoc}/java/util/function/ToLongFunction.html#applyAsLong%2Djava.lang.Object%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TooManyListenersException.asciidoc b/docs/reference/painless-api-reference/TooManyListenersException.asciidoc
new file mode 100644
index 00000000000..5a2d3d57b31
--- /dev/null
+++ b/docs/reference/painless-api-reference/TooManyListenersException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TooManyListenersException]]++TooManyListenersException++::
+* ++[[painless-api-reference-TooManyListenersException-TooManyListenersException-0]]link:{java8-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2D%2D[TooManyListenersException]()++ (link:{java9-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2D%2D[java 9])
+* ++[[painless-api-reference-TooManyListenersException-TooManyListenersException-1]]link:{java8-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2Djava.lang.String%2D[TooManyListenersException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/TooManyListenersException.html#TooManyListenersException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/TreeMap.asciidoc b/docs/reference/painless-api-reference/TreeMap.asciidoc
new file mode 100644
index 00000000000..f91bae69162
--- /dev/null
+++ b/docs/reference/painless-api-reference/TreeMap.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TreeMap]]++TreeMap++::
+* ++[[painless-api-reference-TreeMap-TreeMap-0]]link:{java8-javadoc}/java/util/TreeMap.html#TreeMap%2D%2D[TreeMap]()++ (link:{java9-javadoc}/java/util/TreeMap.html#TreeMap%2D%2D[java 9])
+* ++[[painless-api-reference-TreeMap-TreeMap-1]]link:{java8-javadoc}/java/util/TreeMap.html#TreeMap%2Djava.util.Comparator%2D[TreeMap](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/TreeMap.html#TreeMap%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-TreeMap-clone-0]]def link:{java8-javadoc}/java/util/TreeMap.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TreeMap.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Map,Map>>++, ++<<painless-api-reference-NavigableMap,NavigableMap>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-SortedMap,SortedMap>>++
diff --git a/docs/reference/painless-api-reference/TreeSet.asciidoc b/docs/reference/painless-api-reference/TreeSet.asciidoc
new file mode 100644
index 00000000000..84e8a339fe3
--- /dev/null
+++ b/docs/reference/painless-api-reference/TreeSet.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TreeSet]]++TreeSet++::
+* ++[[painless-api-reference-TreeSet-TreeSet-0]]link:{java8-javadoc}/java/util/TreeSet.html#TreeSet%2D%2D[TreeSet]()++ (link:{java9-javadoc}/java/util/TreeSet.html#TreeSet%2D%2D[java 9])
+* ++[[painless-api-reference-TreeSet-TreeSet-1]]link:{java8-javadoc}/java/util/TreeSet.html#TreeSet%2Djava.util.Comparator%2D[TreeSet](<<painless-api-reference-Comparator,Comparator>>)++ (link:{java9-javadoc}/java/util/TreeSet.html#TreeSet%2Djava.util.Comparator%2D[java 9])
+* ++[[painless-api-reference-TreeSet-clone-0]]def link:{java8-javadoc}/java/util/TreeSet.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/TreeSet.html#clone%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-NavigableSet,NavigableSet>>++, ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Set,Set>>++, ++<<painless-api-reference-SortedSet,SortedSet>>++
diff --git a/docs/reference/painless-api-reference/TypeNotPresentException.asciidoc b/docs/reference/painless-api-reference/TypeNotPresentException.asciidoc
new file mode 100644
index 00000000000..cecb588d83b
--- /dev/null
+++ b/docs/reference/painless-api-reference/TypeNotPresentException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-TypeNotPresentException]]++TypeNotPresentException++::
+* ++[[painless-api-reference-TypeNotPresentException-typeName-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/lang/TypeNotPresentException.html#typeName%2D%2D[typeName]()++ (link:{java9-javadoc}/java/lang/TypeNotPresentException.html#typeName%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UUID.asciidoc b/docs/reference/painless-api-reference/UUID.asciidoc
new file mode 100644
index 00000000000..a47620f9bc1
--- /dev/null
+++ b/docs/reference/painless-api-reference/UUID.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UUID]]++UUID++::
+* ++[[painless-api-reference-UUID-fromString-1]]static <<painless-api-reference-UUID,UUID>> link:{java8-javadoc}/java/util/UUID.html#fromString%2Djava.lang.String%2D[fromString](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UUID.html#fromString%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UUID-nameUUIDFromBytes-1]]static <<painless-api-reference-UUID,UUID>> link:{java8-javadoc}/java/util/UUID.html#nameUUIDFromBytes%2Dbyte:A%2D[nameUUIDFromBytes](byte[])++ (link:{java9-javadoc}/java/util/UUID.html#nameUUIDFromBytes%2Dbyte:A%2D[java 9])
+* ++[[painless-api-reference-UUID-UUID-2]]link:{java8-javadoc}/java/util/UUID.html#UUID%2Dlong%2Dlong%2D[UUID](long, long)++ (link:{java9-javadoc}/java/util/UUID.html#UUID%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-UUID-clockSequence-0]]int link:{java8-javadoc}/java/util/UUID.html#clockSequence%2D%2D[clockSequence]()++ (link:{java9-javadoc}/java/util/UUID.html#clockSequence%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-compareTo-1]]int link:{java8-javadoc}/java/util/UUID.html#compareTo%2Djava.util.UUID%2D[compareTo](<<painless-api-reference-UUID,UUID>>)++ (link:{java9-javadoc}/java/util/UUID.html#compareTo%2Djava.util.UUID%2D[java 9])
+* ++[[painless-api-reference-UUID-getLeastSignificantBits-0]]long link:{java8-javadoc}/java/util/UUID.html#getLeastSignificantBits%2D%2D[getLeastSignificantBits]()++ (link:{java9-javadoc}/java/util/UUID.html#getLeastSignificantBits%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-getMostSignificantBits-0]]long link:{java8-javadoc}/java/util/UUID.html#getMostSignificantBits%2D%2D[getMostSignificantBits]()++ (link:{java9-javadoc}/java/util/UUID.html#getMostSignificantBits%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-node-0]]long link:{java8-javadoc}/java/util/UUID.html#node%2D%2D[node]()++ (link:{java9-javadoc}/java/util/UUID.html#node%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-timestamp-0]]long link:{java8-javadoc}/java/util/UUID.html#timestamp%2D%2D[timestamp]()++ (link:{java9-javadoc}/java/util/UUID.html#timestamp%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-variant-0]]int link:{java8-javadoc}/java/util/UUID.html#variant%2D%2D[variant]()++ (link:{java9-javadoc}/java/util/UUID.html#variant%2D%2D[java 9])
+* ++[[painless-api-reference-UUID-version-0]]int link:{java8-javadoc}/java/util/UUID.html#version%2D%2D[version]()++ (link:{java9-javadoc}/java/util/UUID.html#version%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UnaryOperator.asciidoc b/docs/reference/painless-api-reference/UnaryOperator.asciidoc
new file mode 100644
index 00000000000..fb7c6ed70d2
--- /dev/null
+++ b/docs/reference/painless-api-reference/UnaryOperator.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnaryOperator]]++UnaryOperator++::
+* ++[[painless-api-reference-UnaryOperator-identity-0]]static <<painless-api-reference-UnaryOperator,UnaryOperator>> link:{java8-javadoc}/java/util/function/UnaryOperator.html#identity%2D%2D[identity]()++ (link:{java9-javadoc}/java/util/function/UnaryOperator.html#identity%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Function,Function>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UnknownFormatConversionException.asciidoc b/docs/reference/painless-api-reference/UnknownFormatConversionException.asciidoc
new file mode 100644
index 00000000000..95dd348f422
--- /dev/null
+++ b/docs/reference/painless-api-reference/UnknownFormatConversionException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnknownFormatConversionException]]++UnknownFormatConversionException++::
+* ++[[painless-api-reference-UnknownFormatConversionException-UnknownFormatConversionException-1]]link:{java8-javadoc}/java/util/UnknownFormatConversionException.html#UnknownFormatConversionException%2Djava.lang.String%2D[UnknownFormatConversionException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UnknownFormatConversionException.html#UnknownFormatConversionException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UnknownFormatConversionException-getConversion-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/UnknownFormatConversionException.html#getConversion%2D%2D[getConversion]()++ (link:{java9-javadoc}/java/util/UnknownFormatConversionException.html#getConversion%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UnknownFormatFlagsException.asciidoc b/docs/reference/painless-api-reference/UnknownFormatFlagsException.asciidoc
new file mode 100644
index 00000000000..f14d986c05b
--- /dev/null
+++ b/docs/reference/painless-api-reference/UnknownFormatFlagsException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnknownFormatFlagsException]]++UnknownFormatFlagsException++::
+* ++[[painless-api-reference-UnknownFormatFlagsException-UnknownFormatFlagsException-1]]link:{java8-javadoc}/java/util/UnknownFormatFlagsException.html#UnknownFormatFlagsException%2Djava.lang.String%2D[UnknownFormatFlagsException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/util/UnknownFormatFlagsException.html#UnknownFormatFlagsException%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-UnknownFormatFlagsException-getFlags-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/util/UnknownFormatFlagsException.html#getFlags%2D%2D[getFlags]()++ (link:{java9-javadoc}/java/util/UnknownFormatFlagsException.html#getFlags%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UnsupportedOperationException.asciidoc b/docs/reference/painless-api-reference/UnsupportedOperationException.asciidoc
new file mode 100644
index 00000000000..ad9cbdb1478
--- /dev/null
+++ b/docs/reference/painless-api-reference/UnsupportedOperationException.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnsupportedOperationException]]++UnsupportedOperationException++::
+* ++[[painless-api-reference-UnsupportedOperationException-UnsupportedOperationException-0]]link:{java8-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2D%2D[UnsupportedOperationException]()++ (link:{java9-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2D%2D[java 9])
+* ++[[painless-api-reference-UnsupportedOperationException-UnsupportedOperationException-1]]link:{java8-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2Djava.lang.String%2D[UnsupportedOperationException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/lang/UnsupportedOperationException.html#UnsupportedOperationException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/UnsupportedTemporalTypeException.asciidoc b/docs/reference/painless-api-reference/UnsupportedTemporalTypeException.asciidoc
new file mode 100644
index 00000000000..96e4c1d8402
--- /dev/null
+++ b/docs/reference/painless-api-reference/UnsupportedTemporalTypeException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-UnsupportedTemporalTypeException]]++UnsupportedTemporalTypeException++::
+* ++[[painless-api-reference-UnsupportedTemporalTypeException-UnsupportedTemporalTypeException-1]]link:{java8-javadoc}/java/time/temporal/UnsupportedTemporalTypeException.html#UnsupportedTemporalTypeException%2Djava.lang.String%2D[UnsupportedTemporalTypeException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/temporal/UnsupportedTemporalTypeException.html#UnsupportedTemporalTypeException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ValueRange.asciidoc b/docs/reference/painless-api-reference/ValueRange.asciidoc
new file mode 100644
index 00000000000..409d155f954
--- /dev/null
+++ b/docs/reference/painless-api-reference/ValueRange.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ValueRange]]++ValueRange++::
+* ++[[painless-api-reference-ValueRange-of-2]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2D[of](long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-of-3]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2D[of](long, long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-of-4]]static <<painless-api-reference-ValueRange,ValueRange>> link:{java8-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2Dlong%2D[of](long, long, long, long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#of%2Dlong%2Dlong%2Dlong%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-checkValidIntValue-2]]int link:{java8-javadoc}/java/time/temporal/ValueRange.html#checkValidIntValue%2Dlong%2Djava.time.temporal.TemporalField%2D[checkValidIntValue](long, <<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#checkValidIntValue%2Dlong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-ValueRange-checkValidValue-2]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#checkValidValue%2Dlong%2Djava.time.temporal.TemporalField%2D[checkValidValue](long, <<painless-api-reference-TemporalField,TemporalField>>)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#checkValidValue%2Dlong%2Djava.time.temporal.TemporalField%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getLargestMinimum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getLargestMinimum%2D%2D[getLargestMinimum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getLargestMinimum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getMaximum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getMaximum%2D%2D[getMaximum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getMaximum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getMinimum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getMinimum%2D%2D[getMinimum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getMinimum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-getSmallestMaximum-0]]long link:{java8-javadoc}/java/time/temporal/ValueRange.html#getSmallestMaximum%2D%2D[getSmallestMaximum]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#getSmallestMaximum%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isFixed-0]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isFixed%2D%2D[isFixed]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isFixed%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isIntValue-0]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isIntValue%2D%2D[isIntValue]()++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isIntValue%2D%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isValidIntValue-1]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isValidIntValue%2Dlong%2D[isValidIntValue](long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isValidIntValue%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ValueRange-isValidValue-1]]boolean link:{java8-javadoc}/java/time/temporal/ValueRange.html#isValidValue%2Dlong%2D[isValidValue](long)++ (link:{java9-javadoc}/java/time/temporal/ValueRange.html#isValidValue%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Vector.asciidoc b/docs/reference/painless-api-reference/Vector.asciidoc
new file mode 100644
index 00000000000..b1d9ed88b6f
--- /dev/null
+++ b/docs/reference/painless-api-reference/Vector.asciidoc
@@ -0,0 +1,22 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Vector]]++Vector++::
+* ++[[painless-api-reference-Vector-Vector-0]]link:{java8-javadoc}/java/util/Vector.html#Vector%2D%2D[Vector]()++ (link:{java9-javadoc}/java/util/Vector.html#Vector%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-Vector-1]]link:{java8-javadoc}/java/util/Vector.html#Vector%2Djava.util.Collection%2D[Vector](<<painless-api-reference-Collection,Collection>>)++ (link:{java9-javadoc}/java/util/Vector.html#Vector%2Djava.util.Collection%2D[java 9])
+* ++[[painless-api-reference-Vector-addElement-1]]void link:{java8-javadoc}/java/util/Vector.html#addElement%2Djava.lang.Object%2D[addElement](def)++ (link:{java9-javadoc}/java/util/Vector.html#addElement%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Vector-clone-0]]def link:{java8-javadoc}/java/util/Vector.html#clone%2D%2D[clone]()++ (link:{java9-javadoc}/java/util/Vector.html#clone%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-copyInto-1]]void link:{java8-javadoc}/java/util/Vector.html#copyInto%2Djava.lang.Object:A%2D[copyInto](<<painless-api-reference-Object,Object>>[])++ (link:{java9-javadoc}/java/util/Vector.html#copyInto%2Djava.lang.Object:A%2D[java 9])
+* ++[[painless-api-reference-Vector-elementAt-1]]def link:{java8-javadoc}/java/util/Vector.html#elementAt%2Dint%2D[elementAt](int)++ (link:{java9-javadoc}/java/util/Vector.html#elementAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-elements-0]]<<painless-api-reference-Enumeration,Enumeration>> link:{java8-javadoc}/java/util/Vector.html#elements%2D%2D[elements]()++ (link:{java9-javadoc}/java/util/Vector.html#elements%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-firstElement-0]]def link:{java8-javadoc}/java/util/Vector.html#firstElement%2D%2D[firstElement]()++ (link:{java9-javadoc}/java/util/Vector.html#firstElement%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-insertElementAt-2]]void link:{java8-javadoc}/java/util/Vector.html#insertElementAt%2Djava.lang.Object%2Dint%2D[insertElementAt](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#insertElementAt%2Djava.lang.Object%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-lastElement-0]]def link:{java8-javadoc}/java/util/Vector.html#lastElement%2D%2D[lastElement]()++ (link:{java9-javadoc}/java/util/Vector.html#lastElement%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-lastIndexOf-2]]int link:{java8-javadoc}/java/util/Vector.html#lastIndexOf%2Djava.lang.Object%2Dint%2D[lastIndexOf](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#lastIndexOf%2Djava.lang.Object%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-removeAllElements-0]]void link:{java8-javadoc}/java/util/Vector.html#removeAllElements%2D%2D[removeAllElements]()++ (link:{java9-javadoc}/java/util/Vector.html#removeAllElements%2D%2D[java 9])
+* ++[[painless-api-reference-Vector-removeElement-1]]boolean link:{java8-javadoc}/java/util/Vector.html#removeElement%2Djava.lang.Object%2D[removeElement](def)++ (link:{java9-javadoc}/java/util/Vector.html#removeElement%2Djava.lang.Object%2D[java 9])
+* ++[[painless-api-reference-Vector-removeElementAt-1]]void link:{java8-javadoc}/java/util/Vector.html#removeElementAt%2Dint%2D[removeElementAt](int)++ (link:{java9-javadoc}/java/util/Vector.html#removeElementAt%2Dint%2D[java 9])
+* ++[[painless-api-reference-Vector-setElementAt-2]]void link:{java8-javadoc}/java/util/Vector.html#setElementAt%2Djava.lang.Object%2Dint%2D[setElementAt](def, int)++ (link:{java9-javadoc}/java/util/Vector.html#setElementAt%2Djava.lang.Object%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/WeekFields.asciidoc b/docs/reference/painless-api-reference/WeekFields.asciidoc
new file mode 100644
index 00000000000..330b2e98d2e
--- /dev/null
+++ b/docs/reference/painless-api-reference/WeekFields.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-WeekFields]]++WeekFields++::
+** [[painless-api-reference-WeekFields-ISO]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#ISO[ISO] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#ISO[java 9])
+** [[painless-api-reference-WeekFields-SUNDAY_START]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#SUNDAY_START[SUNDAY_START] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#SUNDAY_START[java 9])
+** [[painless-api-reference-WeekFields-WEEK_BASED_YEARS]]static <<painless-api-reference-TemporalUnit,TemporalUnit>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#WEEK_BASED_YEARS[WEEK_BASED_YEARS] (link:{java9-javadoc}/java/time/temporal/WeekFields.html#WEEK_BASED_YEARS[java 9])
+* ++[[painless-api-reference-WeekFields-of-1]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.util.Locale%2D[of](<<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-WeekFields-of-2]]static <<painless-api-reference-WeekFields,WeekFields>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.time.DayOfWeek%2Dint%2D[of](<<painless-api-reference-DayOfWeek,DayOfWeek>>, int)++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#of%2Djava.time.DayOfWeek%2Dint%2D[java 9])
+* ++[[painless-api-reference-WeekFields-dayOfWeek-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#dayOfWeek%2D%2D[dayOfWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#dayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-getFirstDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#getFirstDayOfWeek%2D%2D[getFirstDayOfWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#getFirstDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-getMinimalDaysInFirstWeek-0]]int link:{java8-javadoc}/java/time/temporal/WeekFields.html#getMinimalDaysInFirstWeek%2D%2D[getMinimalDaysInFirstWeek]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#getMinimalDaysInFirstWeek%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekBasedYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekBasedYear%2D%2D[weekBasedYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekBasedYear%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfMonth-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfMonth%2D%2D[weekOfMonth]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfWeekBasedYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfWeekBasedYear%2D%2D[weekOfWeekBasedYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfWeekBasedYear%2D%2D[java 9])
+* ++[[painless-api-reference-WeekFields-weekOfYear-0]]<<painless-api-reference-TemporalField,TemporalField>> link:{java8-javadoc}/java/time/temporal/WeekFields.html#weekOfYear%2D%2D[weekOfYear]()++ (link:{java9-javadoc}/java/time/temporal/WeekFields.html#weekOfYear%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/Year.asciidoc b/docs/reference/painless-api-reference/Year.asciidoc
new file mode 100644
index 00000000000..e3800991039
--- /dev/null
+++ b/docs/reference/painless-api-reference/Year.asciidoc
@@ -0,0 +1,32 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-Year]]++Year++::
+** [[painless-api-reference-Year-MAX_VALUE]]static int link:{java8-javadoc}/java/time/Year.html#MAX_VALUE[MAX_VALUE] (link:{java9-javadoc}/java/time/Year.html#MAX_VALUE[java 9])
+** [[painless-api-reference-Year-MIN_VALUE]]static int link:{java8-javadoc}/java/time/Year.html#MIN_VALUE[MIN_VALUE] (link:{java9-javadoc}/java/time/Year.html#MIN_VALUE[java 9])
+* ++[[painless-api-reference-Year-from-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/Year.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-Year-isLeap-1]]static boolean link:{java8-javadoc}/java/time/Year.html#isLeap%2Dlong%2D[isLeap](long)++ (link:{java9-javadoc}/java/time/Year.html#isLeap%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-of-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#of%2Dint%2D[of](int)++ (link:{java9-javadoc}/java/time/Year.html#of%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-parse-1]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-Year-parse-2]]static <<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/Year.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-Year-atDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/Year.html#atDay%2Dint%2D[atDay](int)++ (link:{java9-javadoc}/java/time/Year.html#atDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-atMonth-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/Year.html#atMonth%2Dint%2D[atMonth](int)++ (link:{java9-javadoc}/java/time/Year.html#atMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-Year-atMonthDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/Year.html#atMonthDay%2Djava.time.MonthDay%2D[atMonthDay](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/Year.html#atMonthDay%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-Year-compareTo-1]]int link:{java8-javadoc}/java/time/Year.html#compareTo%2Djava.time.Year%2D[compareTo](<<painless-api-reference-Year,Year>>)++ (link:{java9-javadoc}/java/time/Year.html#compareTo%2Djava.time.Year%2D[java 9])
+* ++[[painless-api-reference-Year-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/Year.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/Year.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-Year-getValue-0]]int link:{java8-javadoc}/java/time/Year.html#getValue%2D%2D[getValue]()++ (link:{java9-javadoc}/java/time/Year.html#getValue%2D%2D[java 9])
+* ++[[painless-api-reference-Year-isAfter-1]]boolean link:{java8-javadoc}/java/time/Year.html#isAfter%2Djava.time.Year%2D[isAfter](<<painless-api-reference-Year,Year>>)++ (link:{java9-javadoc}/java/time/Year.html#isAfter%2Djava.time.Year%2D[java 9])
+* ++[[painless-api-reference-Year-isLeap-0]]boolean link:{java8-javadoc}/java/time/Year.html#isLeap%2D%2D[isLeap]()++ (link:{java9-javadoc}/java/time/Year.html#isLeap%2D%2D[java 9])
+* ++[[painless-api-reference-Year-isValidMonthDay-1]]boolean link:{java8-javadoc}/java/time/Year.html#isValidMonthDay%2Djava.time.MonthDay%2D[isValidMonthDay](<<painless-api-reference-MonthDay,MonthDay>>)++ (link:{java9-javadoc}/java/time/Year.html#isValidMonthDay%2Djava.time.MonthDay%2D[java 9])
+* ++[[painless-api-reference-Year-length-0]]int link:{java8-javadoc}/java/time/Year.html#length%2D%2D[length]()++ (link:{java9-javadoc}/java/time/Year.html#length%2D%2D[java 9])
+* ++[[painless-api-reference-Year-minus-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Year.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Year-minus-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Year.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Year-minusYears-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/Year.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-plus-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/Year.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-Year-plus-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/Year.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-Year-plusYears-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/Year.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-Year-with-1]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-Year-with-2]]<<painless-api-reference-Year,Year>> link:{java8-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/Year.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/YearMonth.asciidoc b/docs/reference/painless-api-reference/YearMonth.asciidoc
new file mode 100644
index 00000000000..5573b16c064
--- /dev/null
+++ b/docs/reference/painless-api-reference/YearMonth.asciidoc
@@ -0,0 +1,36 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-YearMonth]]++YearMonth++::
+* ++[[painless-api-reference-YearMonth-from-1]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-YearMonth-of-2]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#of%2Dint%2Dint%2D[of](int, int)++ (link:{java9-javadoc}/java/time/YearMonth.html#of%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-parse-1]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-YearMonth-parse-2]]static <<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-YearMonth-atDay-1]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/YearMonth.html#atDay%2Dint%2D[atDay](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#atDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-atEndOfMonth-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/YearMonth.html#atEndOfMonth%2D%2D[atEndOfMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#atEndOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-compareTo-1]]int link:{java8-javadoc}/java/time/YearMonth.html#compareTo%2Djava.time.YearMonth%2D[compareTo](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#compareTo%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-format-1]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/YearMonth.html#format%2Djava.time.format.DateTimeFormatter%2D[format](<<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#format%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/YearMonth.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getMonthValue-0]]int link:{java8-javadoc}/java/time/YearMonth.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-getYear-0]]int link:{java8-javadoc}/java/time/YearMonth.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isAfter-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isAfter%2Djava.time.YearMonth%2D[isAfter](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#isAfter%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isBefore-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isBefore%2Djava.time.YearMonth%2D[isBefore](<<painless-api-reference-YearMonth,YearMonth>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#isBefore%2Djava.time.YearMonth%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isLeapYear-0]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isLeapYear%2D%2D[isLeapYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#isLeapYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-isValidDay-1]]boolean link:{java8-javadoc}/java/time/YearMonth.html#isValidDay%2Dint%2D[isValidDay](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#isValidDay%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-lengthOfMonth-0]]int link:{java8-javadoc}/java/time/YearMonth.html#lengthOfMonth%2D%2D[lengthOfMonth]()++ (link:{java9-javadoc}/java/time/YearMonth.html#lengthOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-lengthOfYear-0]]int link:{java8-javadoc}/java/time/YearMonth.html#lengthOfYear%2D%2D[lengthOfYear]()++ (link:{java9-javadoc}/java/time/YearMonth.html#lengthOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minus-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minus-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minusMonths-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-minusYears-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plus-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plus-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plusMonths-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-plusYears-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/YearMonth.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-with-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-YearMonth-with-2]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/YearMonth.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-YearMonth-withMonth-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-YearMonth-withYear-1]]<<painless-api-reference-YearMonth,YearMonth>> link:{java8-javadoc}/java/time/YearMonth.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/YearMonth.html#withYear%2Dint%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++, ++<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>++
diff --git a/docs/reference/painless-api-reference/ZoneId.asciidoc b/docs/reference/painless-api-reference/ZoneId.asciidoc
new file mode 100644
index 00000000000..fe31c9df00b
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneId.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneId]]++ZoneId++::
+** [[painless-api-reference-ZoneId-SHORT_IDS]]static <<painless-api-reference-Map,Map>> link:{java8-javadoc}/java/time/ZoneId.html#SHORT_IDS[SHORT_IDS] (link:{java9-javadoc}/java/time/ZoneId.html#SHORT_IDS[java 9])
+* ++[[painless-api-reference-ZoneId-from-1]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getAvailableZoneIds-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/ZoneId.html#getAvailableZoneIds%2D%2D[getAvailableZoneIds]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getAvailableZoneIds%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-of-1]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneId-of-2]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2Djava.util.Map%2D[of](<<painless-api-reference-String,String>>, <<painless-api-reference-Map,Map>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#of%2Djava.lang.String%2Djava.util.Map%2D[java 9])
+* ++[[painless-api-reference-ZoneId-ofOffset-2]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#ofOffset%2Djava.lang.String%2Djava.time.ZoneOffset%2D[ofOffset](<<painless-api-reference-String,String>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#ofOffset%2Djava.lang.String%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneId-systemDefault-0]]static <<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#systemDefault%2D%2D[systemDefault]()++ (link:{java9-javadoc}/java/time/ZoneId.html#systemDefault%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getDisplayName-2]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/ZoneId.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[getDisplayName](<<painless-api-reference-TextStyle,TextStyle>>, <<painless-api-reference-Locale,Locale>>)++ (link:{java9-javadoc}/java/time/ZoneId.html#getDisplayName%2Djava.time.format.TextStyle%2Djava.util.Locale%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getId-0]]<<painless-api-reference-String,String>> link:{java8-javadoc}/java/time/ZoneId.html#getId%2D%2D[getId]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getId%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-getRules-0]]<<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/ZoneId.html#getRules%2D%2D[getRules]()++ (link:{java9-javadoc}/java/time/ZoneId.html#getRules%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneId-normalized-0]]<<painless-api-reference-ZoneId,ZoneId>> link:{java8-javadoc}/java/time/ZoneId.html#normalized%2D%2D[normalized]()++ (link:{java9-javadoc}/java/time/ZoneId.html#normalized%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneOffset.asciidoc b/docs/reference/painless-api-reference/ZoneOffset.asciidoc
new file mode 100644
index 00000000000..84e8530f593
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneOffset.asciidoc
@@ -0,0 +1,17 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffset]]++ZoneOffset++::
+** [[painless-api-reference-ZoneOffset-MAX]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#MAX[MAX] (link:{java9-javadoc}/java/time/ZoneOffset.html#MAX[java 9])
+** [[painless-api-reference-ZoneOffset-MIN]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#MIN[MIN] (link:{java9-javadoc}/java/time/ZoneOffset.html#MIN[java 9])
+** [[painless-api-reference-ZoneOffset-UTC]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#UTC[UTC] (link:{java9-javadoc}/java/time/ZoneOffset.html#UTC[java 9])
+* ++[[painless-api-reference-ZoneOffset-from-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-of-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#of%2Djava.lang.String%2D[of](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#of%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHours-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHours%2Dint%2D[ofHours](int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHours%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHoursMinutes-2]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHoursMinutes%2Dint%2Dint%2D[ofHoursMinutes](int, int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHoursMinutes%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofHoursMinutesSeconds-3]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofHoursMinutesSeconds%2Dint%2Dint%2Dint%2D[ofHoursMinutesSeconds](int, int, int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofHoursMinutesSeconds%2Dint%2Dint%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-ofTotalSeconds-1]]static <<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/ZoneOffset.html#ofTotalSeconds%2Dint%2D[ofTotalSeconds](int)++ (link:{java9-javadoc}/java/time/ZoneOffset.html#ofTotalSeconds%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffset-getTotalSeconds-0]]int link:{java8-javadoc}/java/time/ZoneOffset.html#getTotalSeconds%2D%2D[getTotalSeconds]()++ (link:{java9-javadoc}/java/time/ZoneOffset.html#getTotalSeconds%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++, ++<<painless-api-reference-ZoneId,ZoneId>>++
diff --git a/docs/reference/painless-api-reference/ZoneOffsetTransition.asciidoc b/docs/reference/painless-api-reference/ZoneOffsetTransition.asciidoc
new file mode 100644
index 00000000000..43be4f56ee4
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneOffsetTransition.asciidoc
@@ -0,0 +1,19 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransition]]++ZoneOffsetTransition++::
+* ++[[painless-api-reference-ZoneOffsetTransition-of-3]]static <<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-compareTo-1]]int link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#compareTo%2Djava.time.zone.ZoneOffsetTransition%2D[compareTo](<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#compareTo%2Djava.time.zone.ZoneOffsetTransition%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDateTimeAfter-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeAfter%2D%2D[getDateTimeAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDateTimeBefore-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeBefore%2D%2D[getDateTimeBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDateTimeBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getDuration-0]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDuration%2D%2D[getDuration]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getDuration%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getInstant-0]]<<painless-api-reference-Instant,Instant>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getInstant%2D%2D[getInstant]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getInstant%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getOffsetAfter-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetAfter%2D%2D[getOffsetAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-getOffsetBefore-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetBefore%2D%2D[getOffsetBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#getOffsetBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isGap-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isGap%2D%2D[isGap]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isGap%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isOverlap-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isOverlap%2D%2D[isOverlap]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-isValidOffset-1]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#isValidOffset%2Djava.time.ZoneOffset%2D[isValidOffset](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#isValidOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransition-toEpochSecond-0]]long link:{java8-javadoc}/java/time/zone/ZoneOffsetTransition.html#toEpochSecond%2D%2D[toEpochSecond]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransition.html#toEpochSecond%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc b/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc
new file mode 100644
index 00000000000..45fb25e0f6e
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.TimeDefinition.asciidoc
@@ -0,0 +1,13 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition]]++ZoneOffsetTransitionRule.TimeDefinition++::
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-STANDARD]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#STANDARD[STANDARD] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#STANDARD[java 9])
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-UTC]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#UTC[UTC] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#UTC[java 9])
+** [[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-WALL]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#WALL[WALL] (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#WALL[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-valueOf-1]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#valueOf%2Djava.lang.String%2D[valueOf](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#valueOf%2Djava.lang.String%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-values-0]]static <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>>[] link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#values%2D%2D[values]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#values%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition-createDateTime-3]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#createDateTime%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[createDateTime](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule$TimeDefinition.html#createDateTime%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Enum,Enum>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.asciidoc b/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.asciidoc
new file mode 100644
index 00000000000..8e84bff0172
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneOffsetTransitionRule.asciidoc
@@ -0,0 +1,18 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneOffsetTransitionRule]]++ZoneOffsetTransitionRule++::
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-of-9]]static <<painless-api-reference-ZoneOffsetTransitionRule,ZoneOffsetTransitionRule>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#of%2Djava.time.Month%2Dint%2Djava.time.DayOfWeek%2Djava.time.LocalTime%2Dboolean%2Djava.time.zone.ZoneOffsetTransitionRule$TimeDefinition%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-Month,Month>>, int, <<painless-api-reference-DayOfWeek,DayOfWeek>>, <<painless-api-reference-LocalTime,LocalTime>>, boolean, <<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#of%2Djava.time.Month%2Dint%2Djava.time.DayOfWeek%2Djava.time.LocalTime%2Dboolean%2Djava.time.zone.ZoneOffsetTransitionRule$TimeDefinition%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-createTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#createTransition%2Dint%2D[createTransition](int)++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#createTransition%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getDayOfMonthIndicator-0]]int link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfMonthIndicator%2D%2D[getDayOfMonthIndicator]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfMonthIndicator%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getLocalTime-0]]<<painless-api-reference-LocalTime,LocalTime>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getLocalTime%2D%2D[getLocalTime]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getLocalTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getOffsetAfter-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetAfter%2D%2D[getOffsetAfter]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetAfter%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getOffsetBefore-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetBefore%2D%2D[getOffsetBefore]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getOffsetBefore%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getStandardOffset-0]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getStandardOffset%2D%2D[getStandardOffset]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getStandardOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-getTimeDefinition-0]]<<painless-api-reference-ZoneOffsetTransitionRule-TimeDefinition,ZoneOffsetTransitionRule.TimeDefinition>> link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getTimeDefinition%2D%2D[getTimeDefinition]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#getTimeDefinition%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneOffsetTransitionRule-isMidnightEndOfDay-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#isMidnightEndOfDay%2D%2D[isMidnightEndOfDay]()++ (link:{java9-javadoc}/java/time/zone/ZoneOffsetTransitionRule.html#isMidnightEndOfDay%2D%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneRules.asciidoc b/docs/reference/painless-api-reference/ZoneRules.asciidoc
new file mode 100644
index 00000000000..6c3104a7e02
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneRules.asciidoc
@@ -0,0 +1,21 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRules]]++ZoneRules++::
+* ++[[painless-api-reference-ZoneRules-of-1]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2D[of](<<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-of-5]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.util.List%2Djava.util.List%2Djava.util.List%2D[of](<<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>, <<painless-api-reference-List,List>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#of%2Djava.time.ZoneOffset%2Djava.time.ZoneOffset%2Djava.util.List%2Djava.util.List%2Djava.util.List%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getDaylightSavings-1]]<<painless-api-reference-Duration,Duration>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getDaylightSavings%2Djava.time.Instant%2D[getDaylightSavings](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getDaylightSavings%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getOffset-1]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getOffset%2Djava.time.Instant%2D[getOffset](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getOffset%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getStandardOffset-1]]<<painless-api-reference-ZoneOffset,ZoneOffset>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getStandardOffset%2Djava.time.Instant%2D[getStandardOffset](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getStandardOffset%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransition%2Djava.time.LocalDateTime%2D[getTransition](<<painless-api-reference-LocalDateTime,LocalDateTime>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransition%2Djava.time.LocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransitionRules-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransitionRules%2D%2D[getTransitionRules]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransitionRules%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getTransitions-0]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getTransitions%2D%2D[getTransitions]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getTransitions%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-getValidOffsets-1]]<<painless-api-reference-List,List>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#getValidOffsets%2Djava.time.LocalDateTime%2D[getValidOffsets](<<painless-api-reference-LocalDateTime,LocalDateTime>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#getValidOffsets%2Djava.time.LocalDateTime%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isDaylightSavings-1]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isDaylightSavings%2Djava.time.Instant%2D[isDaylightSavings](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isDaylightSavings%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isFixedOffset-0]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isFixedOffset%2D%2D[isFixedOffset]()++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isFixedOffset%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-isValidOffset-2]]boolean link:{java8-javadoc}/java/time/zone/ZoneRules.html#isValidOffset%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[isValidOffset](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#isValidOffset%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-nextTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#nextTransition%2Djava.time.Instant%2D[nextTransition](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#nextTransition%2Djava.time.Instant%2D[java 9])
+* ++[[painless-api-reference-ZoneRules-previousTransition-1]]<<painless-api-reference-ZoneOffsetTransition,ZoneOffsetTransition>> link:{java8-javadoc}/java/time/zone/ZoneRules.html#previousTransition%2Djava.time.Instant%2D[previousTransition](<<painless-api-reference-Instant,Instant>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRules.html#previousTransition%2Djava.time.Instant%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneRulesException.asciidoc b/docs/reference/painless-api-reference/ZoneRulesException.asciidoc
new file mode 100644
index 00000000000..8bb3ef0cc98
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneRulesException.asciidoc
@@ -0,0 +1,8 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRulesException]]++ZoneRulesException++::
+* ++[[painless-api-reference-ZoneRulesException-ZoneRulesException-1]]link:{java8-javadoc}/java/time/zone/ZoneRulesException.html#ZoneRulesException%2Djava.lang.String%2D[ZoneRulesException](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesException.html#ZoneRulesException%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Exception,Exception>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZoneRulesProvider.asciidoc b/docs/reference/painless-api-reference/ZoneRulesProvider.asciidoc
new file mode 100644
index 00000000000..be33f691f37
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZoneRulesProvider.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZoneRulesProvider]]++ZoneRulesProvider++::
+* ++[[painless-api-reference-ZoneRulesProvider-getAvailableZoneIds-0]]static <<painless-api-reference-Set,Set>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getAvailableZoneIds%2D%2D[getAvailableZoneIds]()++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getAvailableZoneIds%2D%2D[java 9])
+* ++[[painless-api-reference-ZoneRulesProvider-getRules-2]]static <<painless-api-reference-ZoneRules,ZoneRules>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getRules%2Djava.lang.String%2Dboolean%2D[getRules](<<painless-api-reference-String,String>>, boolean)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getRules%2Djava.lang.String%2Dboolean%2D[java 9])
+* ++[[painless-api-reference-ZoneRulesProvider-getVersions-1]]static <<painless-api-reference-NavigableMap,NavigableMap>> link:{java8-javadoc}/java/time/zone/ZoneRulesProvider.html#getVersions%2Djava.lang.String%2D[getVersions](<<painless-api-reference-String,String>>)++ (link:{java9-javadoc}/java/time/zone/ZoneRulesProvider.html#getVersions%2Djava.lang.String%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/ZonedDateTime.asciidoc b/docs/reference/painless-api-reference/ZonedDateTime.asciidoc
new file mode 100644
index 00000000000..e8ba7501c6c
--- /dev/null
+++ b/docs/reference/painless-api-reference/ZonedDateTime.asciidoc
@@ -0,0 +1,66 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-ZonedDateTime]]++ZonedDateTime++::
+* ++[[painless-api-reference-ZonedDateTime-from-1]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[from](<<painless-api-reference-TemporalAccessor,TemporalAccessor>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#from%2Djava.time.temporal.TemporalAccessor%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2D[of](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneId%2D[of](<<painless-api-reference-LocalDate,LocalDate>>, <<painless-api-reference-LocalTime,LocalTime>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Djava.time.LocalDate%2Djava.time.LocalTime%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-of-8]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneId%2D[of](int, int, int, int, int, int, int, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#of%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Dint%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofInstant-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-Instant,Instant>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.Instant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofInstant-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[ofInstant](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofInstant%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofLocal-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofLocal%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2Djava.time.ZoneOffset%2D[ofLocal](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneId,ZoneId>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofLocal%2Djava.time.LocalDateTime%2Djava.time.ZoneId%2Djava.time.ZoneOffset%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-ofStrict-3]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#ofStrict%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[ofStrict](<<painless-api-reference-LocalDateTime,LocalDateTime>>, <<painless-api-reference-ZoneOffset,ZoneOffset>>, <<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#ofStrict%2Djava.time.LocalDateTime%2Djava.time.ZoneOffset%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-parse-1]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-parse-2]]static <<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[parse](<<painless-api-reference-CharSequence,CharSequence>>, <<painless-api-reference-DateTimeFormatter,DateTimeFormatter>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#parse%2Djava.lang.CharSequence%2Djava.time.format.DateTimeFormatter%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfMonth-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfWeek-0]]<<painless-api-reference-DayOfWeek,DayOfWeek>> link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfWeek%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getDayOfYear-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getDayOfYear%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getHour-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getHour%2D%2D[getHour]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getHour%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMinute-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getMinute%2D%2D[getMinute]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMinute%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMonth-0]]<<painless-api-reference-Month,Month>> link:{java8-javadoc}/java/time/ZonedDateTime.html#getMonth%2D%2D[getMonth]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMonth%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getMonthValue-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getMonthValue%2D%2D[getMonthValue]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getMonthValue%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getNano-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getNano%2D%2D[getNano]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getNano%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getSecond-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getSecond%2D%2D[getSecond]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getSecond%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-getYear-0]]int link:{java8-javadoc}/java/time/ZonedDateTime.html#getYear%2D%2D[getYear]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#getYear%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minus-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[minus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minus-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[minus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusDays-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusDays%2Dlong%2D[minusDays](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusHours-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusHours%2Dlong%2D[minusHours](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusMinutes-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusMinutes%2Dlong%2D[minusMinutes](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusMonths-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusMonths%2Dlong%2D[minusMonths](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusNanos-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusNanos%2Dlong%2D[minusNanos](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusSeconds-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusSeconds%2Dlong%2D[minusSeconds](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusWeeks-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusWeeks%2Dlong%2D[minusWeeks](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-minusYears-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#minusYears%2Dlong%2D[minusYears](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#minusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plus-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[plus](<<painless-api-reference-TemporalAmount,TemporalAmount>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plus%2Djava.time.temporal.TemporalAmount%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plus-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[plus](long, <<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plus%2Dlong%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusDays-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusDays%2Dlong%2D[plusDays](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusDays%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusHours-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusHours%2Dlong%2D[plusHours](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusHours%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusMinutes-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusMinutes%2Dlong%2D[plusMinutes](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusMinutes%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusMonths-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusMonths%2Dlong%2D[plusMonths](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusMonths%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusNanos-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusNanos%2Dlong%2D[plusNanos](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusNanos%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusSeconds-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusSeconds%2Dlong%2D[plusSeconds](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusSeconds%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusWeeks-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusWeeks%2Dlong%2D[plusWeeks](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusWeeks%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-plusYears-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#plusYears%2Dlong%2D[plusYears](long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#plusYears%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toLocalDate-0]]<<painless-api-reference-LocalDate,LocalDate>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toLocalDate%2D%2D[toLocalDate]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toLocalDate%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toLocalDateTime-0]]<<painless-api-reference-LocalDateTime,LocalDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toLocalDateTime%2D%2D[toLocalDateTime]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toLocalDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-toOffsetDateTime-0]]<<painless-api-reference-OffsetDateTime,OffsetDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#toOffsetDateTime%2D%2D[toOffsetDateTime]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#toOffsetDateTime%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-truncatedTo-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[truncatedTo](<<painless-api-reference-TemporalUnit,TemporalUnit>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#truncatedTo%2Djava.time.temporal.TemporalUnit%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-with-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[with](<<painless-api-reference-TemporalAdjuster,TemporalAdjuster>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalAdjuster%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-with-2]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[with](<<painless-api-reference-TemporalField,TemporalField>>, long)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#with%2Djava.time.temporal.TemporalField%2Dlong%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withDayOfMonth-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withDayOfMonth%2Dint%2D[withDayOfMonth](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withDayOfMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withDayOfYear-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withDayOfYear%2Dint%2D[withDayOfYear](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withDayOfYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withEarlierOffsetAtOverlap-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[withEarlierOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withEarlierOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withFixedOffsetZone-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withFixedOffsetZone%2D%2D[withFixedOffsetZone]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withFixedOffsetZone%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withHour-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withHour%2Dint%2D[withHour](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withHour%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withLaterOffsetAtOverlap-0]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[withLaterOffsetAtOverlap]()++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withLaterOffsetAtOverlap%2D%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withMinute-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withMinute%2Dint%2D[withMinute](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withMinute%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withMonth-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withMonth%2Dint%2D[withMonth](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withMonth%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withNano-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withNano%2Dint%2D[withNano](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withNano%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withSecond-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withSecond%2Dint%2D[withSecond](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withSecond%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withYear-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withYear%2Dint%2D[withYear](int)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withYear%2Dint%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withZoneSameInstant-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[withZoneSameInstant](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withZoneSameInstant%2Djava.time.ZoneId%2D[java 9])
+* ++[[painless-api-reference-ZonedDateTime-withZoneSameLocal-1]]<<painless-api-reference-ZonedDateTime,ZonedDateTime>> link:{java8-javadoc}/java/time/ZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[withZoneSameLocal](<<painless-api-reference-ZoneId,ZoneId>>)++ (link:{java9-javadoc}/java/time/ZonedDateTime.html#withZoneSameLocal%2Djava.time.ZoneId%2D[java 9])
+* Inherits methods from ++<<painless-api-reference-ChronoZonedDateTime,ChronoZonedDateTime>>++, ++<<painless-api-reference-Temporal,Temporal>>++, ++<<painless-api-reference-TemporalAccessor,TemporalAccessor>>++
diff --git a/docs/reference/painless-api-reference/index.asciidoc b/docs/reference/painless-api-reference/index.asciidoc
new file mode 100644
index 00000000000..51e62715484
--- /dev/null
+++ b/docs/reference/painless-api-reference/index.asciidoc
@@ -0,0 +1,338 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+include::AbstractChronology.asciidoc[]
+include::AbstractCollection.asciidoc[]
+include::AbstractList.asciidoc[]
+include::AbstractMap.asciidoc[]
+include::AbstractMap.SimpleEntry.asciidoc[]
+include::AbstractMap.SimpleImmutableEntry.asciidoc[]
+include::AbstractQueue.asciidoc[]
+include::AbstractSequentialList.asciidoc[]
+include::AbstractSet.asciidoc[]
+include::Annotation.asciidoc[]
+include::Appendable.asciidoc[]
+include::ArithmeticException.asciidoc[]
+include::ArrayDeque.asciidoc[]
+include::ArrayIndexOutOfBoundsException.asciidoc[]
+include::ArrayList.asciidoc[]
+include::ArrayStoreException.asciidoc[]
+include::Arrays.asciidoc[]
+include::AttributedCharacterIterator.asciidoc[]
+include::AttributedCharacterIterator.Attribute.asciidoc[]
+include::AttributedString.asciidoc[]
+include::Base64.asciidoc[]
+include::Base64.Decoder.asciidoc[]
+include::Base64.Encoder.asciidoc[]
+include::BaseStream.asciidoc[]
+include::BiConsumer.asciidoc[]
+include::BiFunction.asciidoc[]
+include::BiPredicate.asciidoc[]
+include::Bidi.asciidoc[]
+include::BigDecimal.asciidoc[]
+include::BigInteger.asciidoc[]
+include::BinaryOperator.asciidoc[]
+include::BitSet.asciidoc[]
+include::Boolean.asciidoc[]
+include::BooleanSupplier.asciidoc[]
+include::BreakIterator.asciidoc[]
+include::Byte.asciidoc[]
+include::BytesRef.asciidoc[]
+include::Calendar.asciidoc[]
+include::Calendar.Builder.asciidoc[]
+include::CharSequence.asciidoc[]
+include::Character.asciidoc[]
+include::Character.Subset.asciidoc[]
+include::Character.UnicodeBlock.asciidoc[]
+include::Character.UnicodeScript.asciidoc[]
+include::CharacterIterator.asciidoc[]
+include::ChoiceFormat.asciidoc[]
+include::ChronoField.asciidoc[]
+include::ChronoLocalDate.asciidoc[]
+include::ChronoLocalDateTime.asciidoc[]
+include::ChronoPeriod.asciidoc[]
+include::ChronoUnit.asciidoc[]
+include::ChronoZonedDateTime.asciidoc[]
+include::Chronology.asciidoc[]
+include::ClassCastException.asciidoc[]
+include::ClassNotFoundException.asciidoc[]
+include::Clock.asciidoc[]
+include::CloneNotSupportedException.asciidoc[]
+include::CollationElementIterator.asciidoc[]
+include::CollationKey.asciidoc[]
+include::Collator.asciidoc[]
+include::Collection.asciidoc[]
+include::Collections.asciidoc[]
+include::Collector.asciidoc[]
+include::Collector.Characteristics.asciidoc[]
+include::Collectors.asciidoc[]
+include::Comparable.asciidoc[]
+include::Comparator.asciidoc[]
+include::ConcurrentModificationException.asciidoc[]
+include::Consumer.asciidoc[]
+include::Currency.asciidoc[]
+include::Date.asciidoc[]
+include::DateFormat.asciidoc[]
+include::DateFormat.Field.asciidoc[]
+include::DateFormatSymbols.asciidoc[]
+include::DateTimeException.asciidoc[]
+include::DateTimeFormatter.asciidoc[]
+include::DateTimeFormatterBuilder.asciidoc[]
+include::DateTimeParseException.asciidoc[]
+include::DayOfWeek.asciidoc[]
+include::Debug.asciidoc[]
+include::DecimalFormat.asciidoc[]
+include::DecimalFormatSymbols.asciidoc[]
+include::DecimalStyle.asciidoc[]
+include::Deque.asciidoc[]
+include::Dictionary.asciidoc[]
+include::Double.asciidoc[]
+include::DoubleBinaryOperator.asciidoc[]
+include::DoubleConsumer.asciidoc[]
+include::DoubleFunction.asciidoc[]
+include::DoublePredicate.asciidoc[]
+include::DoubleStream.asciidoc[]
+include::DoubleStream.Builder.asciidoc[]
+include::DoubleSummaryStatistics.asciidoc[]
+include::DoubleSupplier.asciidoc[]
+include::DoubleToIntFunction.asciidoc[]
+include::DoubleToLongFunction.asciidoc[]
+include::DoubleUnaryOperator.asciidoc[]
+include::DuplicateFormatFlagsException.asciidoc[]
+include::Duration.asciidoc[]
+include::EmptyStackException.asciidoc[]
+include::Enum.asciidoc[]
+include::EnumConstantNotPresentException.asciidoc[]
+include::Enumeration.asciidoc[]
+include::Era.asciidoc[]
+include::EventListener.asciidoc[]
+include::EventListenerProxy.asciidoc[]
+include::EventObject.asciidoc[]
+include::Exception.asciidoc[]
+include::FieldPosition.asciidoc[]
+include::Float.asciidoc[]
+include::Format.asciidoc[]
+include::Format.Field.asciidoc[]
+include::FormatFlagsConversionMismatchException.asciidoc[]
+include::FormatStyle.asciidoc[]
+include::Formattable.asciidoc[]
+include::FormattableFlags.asciidoc[]
+include::Formatter.asciidoc[]
+include::Formatter.BigDecimalLayoutForm.asciidoc[]
+include::FormatterClosedException.asciidoc[]
+include::Function.asciidoc[]
+include::GregorianCalendar.asciidoc[]
+include::HashMap.asciidoc[]
+include::HashSet.asciidoc[]
+include::Hashtable.asciidoc[]
+include::HijrahChronology.asciidoc[]
+include::HijrahDate.asciidoc[]
+include::HijrahEra.asciidoc[]
+include::IdentityHashMap.asciidoc[]
+include::IllegalAccessException.asciidoc[]
+include::IllegalArgumentException.asciidoc[]
+include::IllegalFormatCodePointException.asciidoc[]
+include::IllegalFormatConversionException.asciidoc[]
+include::IllegalFormatException.asciidoc[]
+include::IllegalFormatFlagsException.asciidoc[]
+include::IllegalFormatPrecisionException.asciidoc[]
+include::IllegalFormatWidthException.asciidoc[]
+include::IllegalMonitorStateException.asciidoc[]
+include::IllegalStateException.asciidoc[]
+include::IllegalThreadStateException.asciidoc[]
+include::IllformedLocaleException.asciidoc[]
+include::IndexOutOfBoundsException.asciidoc[]
+include::InputMismatchException.asciidoc[]
+include::Instant.asciidoc[]
+include::InstantiationException.asciidoc[]
+include::IntBinaryOperator.asciidoc[]
+include::IntConsumer.asciidoc[]
+include::IntFunction.asciidoc[]
+include::IntPredicate.asciidoc[]
+include::IntStream.asciidoc[]
+include::IntStream.Builder.asciidoc[]
+include::IntSummaryStatistics.asciidoc[]
+include::IntSupplier.asciidoc[]
+include::IntToDoubleFunction.asciidoc[]
+include::IntToLongFunction.asciidoc[]
+include::IntUnaryOperator.asciidoc[]
+include::Integer.asciidoc[]
+include::InterruptedException.asciidoc[]
+include::IsoChronology.asciidoc[]
+include::IsoEra.asciidoc[]
+include::IsoFields.asciidoc[]
+include::Iterable.asciidoc[]
+include::Iterator.asciidoc[]
+include::JapaneseChronology.asciidoc[]
+include::JapaneseDate.asciidoc[]
+include::JapaneseEra.asciidoc[]
+include::JulianFields.asciidoc[]
+include::LinkedHashMap.asciidoc[]
+include::LinkedHashSet.asciidoc[]
+include::LinkedList.asciidoc[]
+include::List.asciidoc[]
+include::ListIterator.asciidoc[]
+include::LocalDate.asciidoc[]
+include::LocalDateTime.asciidoc[]
+include::LocalTime.asciidoc[]
+include::Locale.asciidoc[]
+include::Locale.Builder.asciidoc[]
+include::Locale.Category.asciidoc[]
+include::Locale.FilteringMode.asciidoc[]
+include::Locale.LanguageRange.asciidoc[]
+include::Long.asciidoc[]
+include::LongBinaryOperator.asciidoc[]
+include::LongConsumer.asciidoc[]
+include::LongFunction.asciidoc[]
+include::LongPredicate.asciidoc[]
+include::LongStream.asciidoc[]
+include::LongStream.Builder.asciidoc[]
+include::LongSummaryStatistics.asciidoc[]
+include::LongSupplier.asciidoc[]
+include::LongToDoubleFunction.asciidoc[]
+include::LongToIntFunction.asciidoc[]
+include::LongUnaryOperator.asciidoc[]
+include::Map.asciidoc[]
+include::Map.Entry.asciidoc[]
+include::Matcher.asciidoc[]
+include::Math.asciidoc[]
+include::MathContext.asciidoc[]
+include::MessageFormat.asciidoc[]
+include::MessageFormat.Field.asciidoc[]
+include::MinguoChronology.asciidoc[]
+include::MinguoDate.asciidoc[]
+include::MinguoEra.asciidoc[]
+include::MissingFormatArgumentException.asciidoc[]
+include::MissingFormatWidthException.asciidoc[]
+include::MissingResourceException.asciidoc[]
+include::Month.asciidoc[]
+include::MonthDay.asciidoc[]
+include::NavigableMap.asciidoc[]
+include::NavigableSet.asciidoc[]
+include::NegativeArraySizeException.asciidoc[]
+include::NoSuchElementException.asciidoc[]
+include::NoSuchFieldException.asciidoc[]
+include::NoSuchMethodException.asciidoc[]
+include::Normalizer.asciidoc[]
+include::Normalizer.Form.asciidoc[]
+include::NullPointerException.asciidoc[]
+include::Number.asciidoc[]
+include::NumberFormat.asciidoc[]
+include::NumberFormat.Field.asciidoc[]
+include::NumberFormatException.asciidoc[]
+include::ObjDoubleConsumer.asciidoc[]
+include::ObjIntConsumer.asciidoc[]
+include::ObjLongConsumer.asciidoc[]
+include::Object.asciidoc[]
+include::Objects.asciidoc[]
+include::Observable.asciidoc[]
+include::Observer.asciidoc[]
+include::OffsetDateTime.asciidoc[]
+include::OffsetTime.asciidoc[]
+include::Optional.asciidoc[]
+include::OptionalDouble.asciidoc[]
+include::OptionalInt.asciidoc[]
+include::OptionalLong.asciidoc[]
+include::ParseException.asciidoc[]
+include::ParsePosition.asciidoc[]
+include::Pattern.asciidoc[]
+include::Period.asciidoc[]
+include::Predicate.asciidoc[]
+include::PrimitiveIterator.asciidoc[]
+include::PrimitiveIterator.OfDouble.asciidoc[]
+include::PrimitiveIterator.OfInt.asciidoc[]
+include::PrimitiveIterator.OfLong.asciidoc[]
+include::PriorityQueue.asciidoc[]
+include::Queue.asciidoc[]
+include::Random.asciidoc[]
+include::RandomAccess.asciidoc[]
+include::ReflectiveOperationException.asciidoc[]
+include::ResolverStyle.asciidoc[]
+include::RoundingMode.asciidoc[]
+include::RuleBasedCollator.asciidoc[]
+include::RuntimeException.asciidoc[]
+include::SecurityException.asciidoc[]
+include::Set.asciidoc[]
+include::Short.asciidoc[]
+include::SignStyle.asciidoc[]
+include::SimpleDateFormat.asciidoc[]
+include::SimpleTimeZone.asciidoc[]
+include::SortedMap.asciidoc[]
+include::SortedSet.asciidoc[]
+include::Spliterator.asciidoc[]
+include::Spliterator.OfDouble.asciidoc[]
+include::Spliterator.OfInt.asciidoc[]
+include::Spliterator.OfLong.asciidoc[]
+include::Spliterator.OfPrimitive.asciidoc[]
+include::Spliterators.asciidoc[]
+include::Stack.asciidoc[]
+include::StackTraceElement.asciidoc[]
+include::Stream.asciidoc[]
+include::Stream.Builder.asciidoc[]
+include::StrictMath.asciidoc[]
+include::String.asciidoc[]
+include::StringBuffer.asciidoc[]
+include::StringBuilder.asciidoc[]
+include::StringCharacterIterator.asciidoc[]
+include::StringIndexOutOfBoundsException.asciidoc[]
+include::StringJoiner.asciidoc[]
+include::StringTokenizer.asciidoc[]
+include::Supplier.asciidoc[]
+include::System.asciidoc[]
+include::Temporal.asciidoc[]
+include::TemporalAccessor.asciidoc[]
+include::TemporalAdjuster.asciidoc[]
+include::TemporalAdjusters.asciidoc[]
+include::TemporalAmount.asciidoc[]
+include::TemporalField.asciidoc[]
+include::TemporalQueries.asciidoc[]
+include::TemporalQuery.asciidoc[]
+include::TemporalUnit.asciidoc[]
+include::TextStyle.asciidoc[]
+include::ThaiBuddhistChronology.asciidoc[]
+include::ThaiBuddhistDate.asciidoc[]
+include::ThaiBuddhistEra.asciidoc[]
+include::TimeZone.asciidoc[]
+include::ToDoubleBiFunction.asciidoc[]
+include::ToDoubleFunction.asciidoc[]
+include::ToIntBiFunction.asciidoc[]
+include::ToIntFunction.asciidoc[]
+include::ToLongBiFunction.asciidoc[]
+include::ToLongFunction.asciidoc[]
+include::TooManyListenersException.asciidoc[]
+include::TreeMap.asciidoc[]
+include::TreeSet.asciidoc[]
+include::TypeNotPresentException.asciidoc[]
+include::UUID.asciidoc[]
+include::UnaryOperator.asciidoc[]
+include::UnknownFormatConversionException.asciidoc[]
+include::UnknownFormatFlagsException.asciidoc[]
+include::UnsupportedOperationException.asciidoc[]
+include::UnsupportedTemporalTypeException.asciidoc[]
+include::ValueRange.asciidoc[]
+include::Vector.asciidoc[]
+include::WeekFields.asciidoc[]
+include::Year.asciidoc[]
+include::YearMonth.asciidoc[]
+include::ZoneId.asciidoc[]
+include::ZoneOffset.asciidoc[]
+include::ZoneOffsetTransition.asciidoc[]
+include::ZoneOffsetTransitionRule.asciidoc[]
+include::ZoneOffsetTransitionRule.TimeDefinition.asciidoc[]
+include::ZoneRules.asciidoc[]
+include::ZoneRulesException.asciidoc[]
+include::ZoneRulesProvider.asciidoc[]
+include::ZonedDateTime.asciidoc[]
+include::org.elasticsearch.common.geo.GeoPoint.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc[]
+include::org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc[]
+include::org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc[]
+include::org.elasticsearch.painless.FeatureTest.asciidoc[]
+include::org.joda.time.ReadableDateTime.asciidoc[]
+include::org.joda.time.ReadableInstant.asciidoc[]
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc
new file mode 100644
index 00000000000..2a593d0ec63
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.common.geo.GeoPoint.asciidoc
@@ -0,0 +1,9 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint]]++org.elasticsearch.common.geo.GeoPoint++::
+* ++[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint-getLat-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/common/geo/GeoPoint.html#getLat%2D%2D[getLat]()++
+* ++[[painless-api-reference-org-elasticsearch-common-geo-GeoPoint-getLon-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/common/geo/GeoPoint.html#getLon%2D%2D[getLon]()++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc
new file mode 100644
index 00000000000..12aaed4003d
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Booleans.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans]]++org.elasticsearch.index.fielddata.ScriptDocValues.Booleans++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-get-1]]<<painless-api-reference-Boolean,Boolean>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Booleans.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-getValue-0]]boolean link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Booleans.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Booleans-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Booleans.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc
new file mode 100644
index 00000000000..fe11f968749
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs]]++org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-get-1]]<<painless-api-reference-BytesRef,BytesRef>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$BytesRefs.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-getValue-0]]<<painless-api-reference-BytesRef,BytesRef>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$BytesRefs.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-BytesRefs-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$BytesRefs.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc
new file mode 100644
index 00000000000..92d43072414
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Doubles.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles]]++org.elasticsearch.index.fielddata.ScriptDocValues.Doubles++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-get-1]]<<painless-api-reference-Double,Double>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Doubles.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-getValue-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Doubles.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Doubles-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Doubles.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc
new file mode 100644
index 00000000000..e55afb25594
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints.asciidoc
@@ -0,0 +1,20 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints]]++org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-arcDistance-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#arcDistance%2Ddouble%2Ddouble%2D[arcDistance](double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-arcDistanceWithDefault-3]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#arcDistanceWithDefault%2Ddouble%2Ddouble%2Ddouble%2D[arcDistanceWithDefault](double, double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-geohashDistance-1]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#geohashDistance%2Djava.lang.String%2D[geohashDistance](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-geohashDistanceWithDefault-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#geohashDistanceWithDefault%2Djava.lang.String%2Ddouble%2D[geohashDistanceWithDefault](<<painless-api-reference-String,String>>, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-get-1]]<<painless-api-reference-org-elasticsearch-common-geo-GeoPoint,org.elasticsearch.common.geo.GeoPoint>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLat-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getLat%2D%2D[getLat]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLats-0]]double[] link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getLats%2D%2D[getLats]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLon-0]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getLon%2D%2D[getLon]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getLons-0]]double[] link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getLons%2D%2D[getLons]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getValue-0]]<<painless-api-reference-org-elasticsearch-common-geo-GeoPoint,org.elasticsearch.common.geo.GeoPoint>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#getValues%2D%2D[getValues]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-planeDistance-2]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#planeDistance%2Ddouble%2Ddouble%2D[planeDistance](double, double)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-GeoPoints-planeDistanceWithDefault-3]]double link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$GeoPoints.html#planeDistanceWithDefault%2Ddouble%2Ddouble%2Ddouble%2D[planeDistanceWithDefault](double, double, double)++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc
new file mode 100644
index 00000000000..85e6e9289ef
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Longs.asciidoc
@@ -0,0 +1,12 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs]]++org.elasticsearch.index.fielddata.ScriptDocValues.Longs++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-get-1]]<<painless-api-reference-Long,Long>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Longs.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getDate-0]]<<painless-api-reference-org-joda-time-ReadableDateTime,org.joda.time.ReadableDateTime>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Longs.html#getDate%2D%2D[getDate]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getDates-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Longs.html#getDates%2D%2D[getDates]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getValue-0]]long link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Longs.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Longs-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Longs.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc
new file mode 100644
index 00000000000..c15c152cd65
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.fielddata.ScriptDocValues.Strings.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings]]++org.elasticsearch.index.fielddata.ScriptDocValues.Strings++::
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-get-1]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Strings.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-getValue-0]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Strings.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-fielddata-ScriptDocValues-Strings-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/fielddata/ScriptDocValues$Strings.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc
new file mode 100644
index 00000000000..e6ce5d5a57a
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues.asciidoc
@@ -0,0 +1,10 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues]]++org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues++::
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-get-1]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper$IpFieldType$IpScriptDocValues.html#get%2Dint%2D[get](int)++
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-getValue-0]]<<painless-api-reference-String,String>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper$IpFieldType$IpScriptDocValues.html#getValue%2D%2D[getValue]()++
+* ++[[painless-api-reference-org-elasticsearch-index-mapper-IpFieldMapper-IpFieldType-IpScriptDocValues-getValues-0]]<<painless-api-reference-List,List>> link:{elasticsearch-javadoc}/org/elasticsearch/index/mapper/IpFieldMapper$IpFieldType$IpScriptDocValues.html#getValues%2D%2D[getValues]()++
+* Inherits methods from ++<<painless-api-reference-Collection,Collection>>++, ++<<painless-api-reference-Iterable,Iterable>>++, ++<<painless-api-reference-List,List>>++, ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc b/docs/reference/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc
new file mode 100644
index 00000000000..40fdec50793
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.elasticsearch.painless.FeatureTest.asciidoc
@@ -0,0 +1,16 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-elasticsearch-painless-FeatureTest]]++org.elasticsearch.painless.FeatureTest++::
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-overloadedStatic-0]]static boolean link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#overloadedStatic%2D%2D[overloadedStatic]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-overloadedStatic-1]]static boolean link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#overloadedStatic%2Dboolean%2D[overloadedStatic](boolean)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-org.elasticsearch.painless.FeatureTest-0]]link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#org.elasticsearch.painless.FeatureTest%2D%2D[org.elasticsearch.painless.FeatureTest]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-org.elasticsearch.painless.FeatureTest-2]]link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#org.elasticsearch.painless.FeatureTest%2Dint%2Dint%2D[org.elasticsearch.painless.FeatureTest](int, int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-getX-0]]int link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#getX%2D%2D[getX]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-getY-0]]int link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#getY%2D%2D[getY]()++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-setX-1]]void link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#setX%2Dint%2D[setX](int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-setY-1]]void link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#setY%2Dint%2D[setY](int)++
+* ++[[painless-api-reference-org-elasticsearch-painless-FeatureTest-twoFunctionsOfX-2]]<<painless-api-reference-Object,Object>> link:{painless-javadoc}/org/elasticsearch/painless/FeatureTest.html#twoFunctionsOfX%2Djava.util.function.Function%2Djava.util.function.Function%2D[twoFunctionsOfX](<<painless-api-reference-Function,Function>>, <<painless-api-reference-Function,Function>>)++
+* Inherits methods from ++<<painless-api-reference-Object,Object>>++
diff --git a/docs/reference/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc b/docs/reference/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc
new file mode 100644
index 00000000000..a7a0a17c1ad
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.joda.time.ReadableDateTime.asciidoc
@@ -0,0 +1,27 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-joda-time-ReadableDateTime]]++org.joda.time.ReadableDateTime++::
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getCenturyOfEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getCenturyOfEra%2D%2D[getCenturyOfEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfMonth-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfMonth%2D%2D[getDayOfMonth]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfWeek-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfWeek%2D%2D[getDayOfWeek]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getDayOfYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getDayOfYear%2D%2D[getDayOfYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getEra%2D%2D[getEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getHourOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getHourOfDay%2D%2D[getHourOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMillisOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMillisOfDay%2D%2D[getMillisOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMillisOfSecond-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMillisOfSecond%2D%2D[getMillisOfSecond]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMinuteOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMinuteOfDay%2D%2D[getMinuteOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMinuteOfHour-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMinuteOfHour%2D%2D[getMinuteOfHour]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getMonthOfYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getMonthOfYear%2D%2D[getMonthOfYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getSecondOfDay-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getSecondOfDay%2D%2D[getSecondOfDay]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getSecondOfMinute-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getSecondOfMinute%2D%2D[getSecondOfMinute]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getWeekOfWeekyear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getWeekOfWeekyear%2D%2D[getWeekOfWeekyear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getWeekyear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getWeekyear%2D%2D[getWeekyear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYear-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYear%2D%2D[getYear]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYearOfCentury-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYearOfCentury%2D%2D[getYearOfCentury]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-getYearOfEra-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#getYearOfEra%2D%2D[getYearOfEra]()++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-toString-1]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#toString%2Djava.lang.String%2D[toString](<<painless-api-reference-String,String>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableDateTime-toString-2]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableDateTime.html#toString%2Djava.lang.String%2Djava.util.Locale%2D[toString](<<painless-api-reference-String,String>>, <<painless-api-reference-Locale,Locale>>)++
+* Inherits methods from ++<<painless-api-reference-Comparable,Comparable>>++, ++<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>++
diff --git a/docs/reference/painless-api-reference/org.joda.time.ReadableInstant.asciidoc b/docs/reference/painless-api-reference/org.joda.time.ReadableInstant.asciidoc
new file mode 100644
index 00000000000..96d1f61a81f
--- /dev/null
+++ b/docs/reference/painless-api-reference/org.joda.time.ReadableInstant.asciidoc
@@ -0,0 +1,14 @@
+////
+Automatically generated by PainlessDocGenerator. Do not edit.
+Rebuild by running `gradle generatePainlessApi`.
+////
+
+[[painless-api-reference-org-joda-time-ReadableInstant]]++org.joda.time.ReadableInstant++::
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-equals-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#equals%2Djava.lang.Object%2D[equals](<<painless-api-reference-Object,Object>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-getMillis-0]]long link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#getMillis%2D%2D[getMillis]()++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-hashCode-0]]int link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#hashCode%2D%2D[hashCode]()++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isAfter-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isAfter%2Dorg.joda.time.ReadableInstant%2D[isAfter](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isBefore-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isBefore%2Dorg.joda.time.ReadableInstant%2D[isBefore](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-isEqual-1]]boolean link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#isEqual%2Dorg.joda.time.ReadableInstant%2D[isEqual](<<painless-api-reference-org-joda-time-ReadableInstant,org.joda.time.ReadableInstant>>)++
+* ++[[painless-api-reference-org-joda-time-ReadableInstant-toString-0]]<<painless-api-reference-String,String>> link:{joda-time-javadoc}/org/joda/time/ReadableInstant.html#toString%2D%2D[toString]()++
+* Inherits methods from ++<<painless-api-reference-Comparable,Comparable>>++
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index 2a289910c43..977953affc5 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -13,7 +13,9 @@ occurrence types are:
 contribute to the score.
 
 |`filter` |The clause (query) must appear in matching documents. However unlike
-`must` the score of the query will be ignored.
+`must` the score of the query will be ignored. Filter clauses are executed
+in <<query-filter-context,filter context>>, meaning that scoring is ignored
+and clauses are considered for caching.
 
 |`should` |The clause (query) should appear in the matching document. In
 a boolean query with no `must` or `filter` clauses, one or more `should` clauses
@@ -23,7 +25,9 @@ be set using the
 parameter.
 
 |`must_not` |The clause (query) must not appear in the matching
-documents.
+documents.  Clauses are executed in <<query-filter-context,filter context>> meaning
+that scoring is ignored and clauses are considered for caching. Because scoring is
+ignored, a score of `0` for all documents is returned.
 |=======================================================================
 
 [IMPORTANT]
@@ -56,7 +60,7 @@ POST _search
       },
       "must_not" : {
         "range" : {
-          "age" : { "from" : 10, "to" : 20 }
+          "age" : { "gte" : 10, "lte" : 20 }
         }
       },
       "should" : [
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index bced9fc9fbe..aa7ee60aa5c 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -20,3 +20,6 @@ GET /_search
 }
 --------------------------------------------------
 // CONSOLE
+
+Filter clauses are executed in <<query-filter-context,filter context>>,
+meaning that scoring is ignored and clauses are considered for caching.
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index 2f82f2294b3..1f9fc53d66d 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -32,15 +32,15 @@ GET /_search
     "query": {
         "dis_max" : {
             "tie_breaker" : 0.7,
-                "boost" : 1.2,
-                "queries" : [
+            "boost" : 1.2,
+            "queries" : [
                 {
                     "term" : { "age" : 34 }
                 },
                 {
                     "term" : { "age" : 35 }
                 }
-                ]
+            ]
         }
     }
 }    
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index 4591ccf5c9e..eb419077a81 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -191,7 +191,7 @@ The following are options allowed on the filter:
 
 `distance_type`::
 
-    How to compute the distance. Can either be `sloppy_arc` (default), `arc` (slightly more precise but significantly slower) or `plane` (faster, but inaccurate on long distances and close to the poles).
+    How to compute the distance. Can either be `arc` (default), or `plane` (faster, but inaccurate on long distances and close to the poles).
 
 `optimize_bbox`::
 
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index f662dc825c0..c35f5111103 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -40,7 +40,7 @@ GET /my_index/_search
     "query": {
         "parent_id" : {
             "type" : "blog_tag",
-                "id" : "1"
+            "id" : "1"
         }
     }
 }
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 2dcfcde1ca0..cc8ac5068c4 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -197,7 +197,24 @@ GET /_search
 
 Another option is to provide the wildcard fields search in the query
 string itself (properly escaping the `*` sign), for example:
-`city.\*:something`.
+`city.\*:something`:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "query_string" : {
+            "query" : "city.\\*:(this AND that OR thus)",
+            "use_dis_max" : true
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+NOTE: Since `\` (backslash) is a special character in json strings, it needs to
+be escaped, hence the two backslashes in the above `query_string`.
 
 When running the `query_string` query against multiple fields, the
 following additional parameters are allowed:
diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc
index 9c900959196..c5d2e55442b 100644
--- a/docs/reference/query-dsl/query-string-syntax.asciidoc
+++ b/docs/reference/query-dsl/query-string-syntax.asciidoc
@@ -279,6 +279,10 @@ The reserved characters are:  `+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /`
 Failing to escape these special characters correctly could lead to a syntax
 error which prevents your query from running.
 
+NOTE: `<` and `>` can't be escaped at all. The only way to prevent them from
+attempting to create a range query is to remove them from the query string
+entirely.
+
 .Watch this space
 ****
 A space may also be a reserved character.  For instance, if you have a
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index c67ba5cd73e..7ef7ba60cc5 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -83,6 +83,30 @@ The `simple_query_string` supports the following special characters:
 In order to search for any of these special characters, they will need to
 be escaped with `\`.
 
+Be aware that this syntax may have a different behavior depending on the
+`default_operator` value. For example, consider the following query:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "simple_query_string" : {
+            "fields" : ["content"],
+            "query" : "foo bar -baz"
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+You may expect that documents containing only "foo" or "bar" will be returned,
+as long as they do not contain "baz", however, due to the `default_operator`
+being OR, this really means "match documents that contain "foo" or documents
+that contain "bar", or documents that don't contain "baz". If this is unintended
+then the query can be switched to `"foo bar +-baz"` which will not return
+documents that contain "baz".
+
 [float]
 ==== Default Field
 When not explicitly specifying the field to search on in the query
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index 3d1672753d2..b41906b5650 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -2,7 +2,7 @@
 === Span Multi Term Query
 
 The `span_multi` query allows you to wrap a `multi term query` (one of wildcard,
-fuzzy, prefix, term, range or regexp query) as a `span query`, so
+fuzzy, prefix, range or regexp query) as a `span query`, so
 it can be nested. Example:
 
 [source,js]
diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc
index 64c5963c87c..f036443e5af 100644
--- a/docs/reference/search.asciidoc
+++ b/docs/reference/search.asciidoc
@@ -129,8 +129,6 @@ include::search/explain.asciidoc[]
 
 include::search/profile.asciidoc[]
 
-include::search/percolate.asciidoc[]
+include::search/field-stats.asciidoc[]
 
 include::search/rank-eval.asciidoc[]
-
-include::search/field-stats.asciidoc[]
diff --git a/docs/reference/search/percolate.asciidoc b/docs/reference/search/percolate.asciidoc
deleted file mode 100644
index 755b5f7a24c..00000000000
--- a/docs/reference/search/percolate.asciidoc
+++ /dev/null
@@ -1,6 +0,0 @@
-[[search-percolate]]
-== Percolator
-
-deprecated[5.0.0,Percolate and multi percolate APIs are deprecated and have been replaced by the new <<query-dsl-percolate-query,`percolate` query>>]
-
-For indices created on or after version 5.0.0-alpha1 the percolator automatically indexes the query terms with the percolator queries. This allows the percolator to percolate documents more quickly. It is advisable to reindex any pre 5.0.0 indices to take advantage of this new optimization.
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 52b744d30e9..cb47957198c 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -58,7 +58,7 @@ This will yield the following result:
                     {
                        "type": "BooleanQuery",
                        "description": "message:message message:number",
-                       "time": "1.873811000ms",
+                       "time_in_nanos": "1873811",
                        "breakdown": {
                           "score": 51306,
                           "score_count": 4,
@@ -77,7 +77,7 @@ This will yield the following result:
                           {
                              "type": "TermQuery",
                              "description": "message:message",
-                             "time": "0.3919430000ms",
+                             "time_in_nanos": "391943",
                              "breakdown": {
                                 "score": 28776,
                                 "score_count": 4,
@@ -96,7 +96,7 @@ This will yield the following result:
                           {
                              "type": "TermQuery",
                              "description": "message:number",
-                             "time": "0.2106820000ms",
+                             "time_in_nanos": "210682",
                              "breakdown": {
                                 "score": 4552,
                                 "score_count": 4,
@@ -120,12 +120,12 @@ This will yield the following result:
                     {
                        "name": "CancellableCollector",
                        "reason": "search_cancelled",
-                       "time": "0.3043110000ms",
+                       "time_in_nanos": "304311",
                        "children": [
                          {
                            "name": "SimpleTopScoreDocCollector",
                            "reason": "search_top_hits",
-                           "time": "0.03227300000ms"
+                           "time_in_nanos": "32273"
                          }
                        ]
                     }
@@ -143,22 +143,22 @@ This will yield the following result:
 // TESTRESPONSE[s/"id": "\[2aE02wS1R8q_QFnYu6vDVQ\]\[twitter\]\[1\]"/"id": $body.profile.shards.0.id/]
 // TESTRESPONSE[s/"rewrite_time": 51443/"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time/]
 // TESTRESPONSE[s/"score": 51306/"score": $body.profile.shards.0.searches.0.query.0.breakdown.score/]
-// TESTRESPONSE[s/"time": "1.873811000ms"/"time": $body.profile.shards.0.searches.0.query.0.time/]
+// TESTRESPONSE[s/"time_in_nanos": "1873811"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos/]
 // TESTRESPONSE[s/"build_scorer": 2935582/"build_scorer": $body.profile.shards.0.searches.0.query.0.breakdown.build_scorer/]
 // TESTRESPONSE[s/"create_weight": 919297/"create_weight": $body.profile.shards.0.searches.0.query.0.breakdown.create_weight/]
 // TESTRESPONSE[s/"next_doc": 53876/"next_doc": $body.profile.shards.0.searches.0.query.0.breakdown.next_doc/]
-// TESTRESPONSE[s/"time": "0.3919430000ms"/"time": $body.profile.shards.0.searches.0.query.0.children.0.time/]
+// TESTRESPONSE[s/"time_in_nanos": "391943"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.0.time_in_nanos/]
 // TESTRESPONSE[s/"score": 28776/"score": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.score/]
 // TESTRESPONSE[s/"build_scorer": 784451/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.build_scorer/]
 // TESTRESPONSE[s/"create_weight": 1669564/"create_weight": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.create_weight/]
 // TESTRESPONSE[s/"next_doc": 10111/"next_doc": $body.profile.shards.0.searches.0.query.0.children.0.breakdown.next_doc/]
-// TESTRESPONSE[s/"time": "0.2106820000ms"/"time": $body.profile.shards.0.searches.0.query.0.children.1.time/]
+// TESTRESPONSE[s/"time_in_nanos": "210682"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.1.time_in_nanos/]
 // TESTRESPONSE[s/"score": 4552/"score": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.score/]
 // TESTRESPONSE[s/"build_scorer": 42602/"build_scorer": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.build_scorer/]
 // TESTRESPONSE[s/"create_weight": 89323/"create_weight": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.create_weight/]
 // TESTRESPONSE[s/"next_doc": 2852/"next_doc": $body.profile.shards.0.searches.0.query.0.children.1.breakdown.next_doc/]
-// TESTRESPONSE[s/"time": "0.3043110000ms"/"time": $body.profile.shards.0.searches.0.collector.0.time/]
-// TESTRESPONSE[s/"time": "0.03227300000ms"/"time": $body.profile.shards.0.searches.0.collector.0.children.0.time/]
+// TESTRESPONSE[s/"time_in_nanos": "304311"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.time_in_nanos/]
+// TESTRESPONSE[s/"time_in_nanos": "32273"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.children.0.time_in_nanos/]
 // Sorry for this mess....
 
 <1> Search results are returned, but were omitted here for brevity
@@ -216,6 +216,10 @@ a `query` array and a `collector` array.  Alongside the `search` object is an `a
 
 There will also be a `rewrite` metric showing the total time spent rewriting the query (in nanoseconds).
 
+NOTE: As with other statistics apis, the Profile API supports human readable outputs. This can be turned on by adding
+`?human=true` to the query string. In this case, the output contains the additional `time` field containing rounded,
+human readable timing information (e.g. `"time": "391,9ms"`, `"time": "123.3micros"`).
+
 === Profiling Queries
 
 [NOTE]
@@ -244,19 +248,19 @@ The overall structure of this query tree will resemble your original Elasticsear
     {
        "type": "BooleanQuery",
        "description": "message:message message:number",
-       "time": "1.873811000ms",
+       "time_in_nanos": "1873811",
        "breakdown": {...},               <1>
        "children": [
           {
              "type": "TermQuery",
              "description": "message:message",
-             "time": "0.3919430000ms",
+             "time_in_nanos": "391943",
              "breakdown": {...}
           },
           {
              "type": "TermQuery",
              "description": "message:number",
-             "time": "0.2106820000ms",
+             "time_in_nanos": "210682",
              "breakdown": {...}
           }
        ]
@@ -265,28 +269,28 @@ The overall structure of this query tree will resemble your original Elasticsear
 --------------------------------------------------
 // TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n/]
 // TESTRESPONSE[s/]$/],"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time, "collector": $body.profile.shards.0.searches.0.collector}], "aggregations": []}]}}/]
-// TESTRESPONSE[s/"time": "1.873811000ms",\n.+"breakdown": \{...\}/"time": $body.profile.shards.0.searches.0.query.0.time, "breakdown": $body.profile.shards.0.searches.0.query.0.breakdown/]
-// TESTRESPONSE[s/"time": "0.3919430000ms",\n.+"breakdown": \{...\}/"time": $body.profile.shards.0.searches.0.query.0.children.0.time, "breakdown": $body.profile.shards.0.searches.0.query.0.children.0.breakdown/]
-// TESTRESPONSE[s/"time": "0.2106820000ms",\n.+"breakdown": \{...\}/"time": $body.profile.shards.0.searches.0.query.0.children.1.time, "breakdown": $body.profile.shards.0.searches.0.query.0.children.1.breakdown/]
+// TESTRESPONSE[s/"time_in_nanos": "1873811",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.breakdown/]
+// TESTRESPONSE[s/"time_in_nanos": "391943",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.0.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.children.0.breakdown/]
+// TESTRESPONSE[s/"time_in_nanos": "210682",\n.+"breakdown": \{...\}/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.children.1.time_in_nanos, "breakdown": $body.profile.shards.0.searches.0.query.0.children.1.breakdown/]
 <1> The breakdown timings are omitted for simplicity
 
 Based on the profile structure, we can see that our `match` query was rewritten by Lucene into a BooleanQuery with two
-clauses (both holding a TermQuery).  The `"type"` field displays the Lucene class name, and often aligns with
-the equivalent name in Elasticsearch.  The `"description"` field displays the Lucene explanation text for the query, and
-is made available to help differentiating between parts of your query (e.g. both `"message:search"` and `"message:test"`
+clauses (both holding a TermQuery).  The `type` field displays the Lucene class name, and often aligns with
+the equivalent name in Elasticsearch.  The `description` field displays the Lucene explanation text for the query, and
+is made available to help differentiating between parts of your query (e.g. both `message:search` and `message:test`
 are TermQuery's and would appear identical otherwise.
 
-The `"time"` field shows that this query took ~15ms for the entire BooleanQuery to execute.  The recorded time is inclusive
+The `time_in_nanos` field shows that this query took ~1.8ms for the entire BooleanQuery to execute.  The recorded time is inclusive
 of all children.
 
-The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
-that in a moment.  Finally, the `"children"` array lists any sub-queries that may be present.  Because we searched for two
+The `breakdown` field will give detailed stats about how the time was spent, we'll look at
+that in a moment.  Finally, the `children` array lists any sub-queries that may be present.  Because we searched for two
 values ("search test"), our BooleanQuery holds two children TermQueries.  They have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
 ===== Timing Breakdown
 
-The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
+The `breakdown` component lists detailed timing statistics about low-level Lucene execution:
 
 [source,js]
 --------------------------------------------------
@@ -305,16 +309,16 @@ The `"breakdown"` component lists detailed timing statistics about low-level Luc
    "advance_count": 0
 }
 --------------------------------------------------
-// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:message message:number",\n"time": $body.profile.shards.0.searches.0.query.0.time,/]
+// TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n"query": [{\n"type": "BooleanQuery",\n"description": "message:message message:number",\n"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos,/]
 // TESTRESPONSE[s/}$/},\n"children": $body.profile.shards.0.searches.0.query.0.children}],\n"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time, "collector": $body.profile.shards.0.searches.0.collector}], "aggregations": []}]}}/]
 // TESTRESPONSE[s/"score": 51306/"score": $body.profile.shards.0.searches.0.query.0.breakdown.score/]
-// TESTRESPONSE[s/"time": "1.873811000ms"/"time": $body.profile.shards.0.searches.0.query.0.time/]
+// TESTRESPONSE[s/"time_in_nanos": "1873811"/"time_in_nanos": $body.profile.shards.0.searches.0.query.0.time_in_nanos/]
 // TESTRESPONSE[s/"build_scorer": 2935582/"build_scorer": $body.profile.shards.0.searches.0.query.0.breakdown.build_scorer/]
 // TESTRESPONSE[s/"create_weight": 919297/"create_weight": $body.profile.shards.0.searches.0.query.0.breakdown.create_weight/]
 // TESTRESPONSE[s/"next_doc": 53876/"next_doc": $body.profile.shards.0.searches.0.query.0.breakdown.next_doc/]
 
 Timings are listed in wall-clock nanoseconds and are not normalized at all.  All caveats about the overall
-`"time"` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
+`time_in_nanos` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
 actually eating time, and B) the magnitude of differences in times between the various components.  Like the overall time,
 the breakdown is inclusive of all children times.
 
@@ -401,12 +405,12 @@ Looking at the previous example:
    {
       "name": "CancellableCollector",
       "reason": "search_cancelled",
-      "time": "0.3043110000ms",
+      "time_in_nanos": "304311",
       "children": [
         {
           "name": "SimpleTopScoreDocCollector",
           "reason": "search_top_hits",
-          "time": "0.03227300000ms"
+          "time_in_nanos": "32273"
         }
       ]
    }
@@ -414,12 +418,12 @@ Looking at the previous example:
 --------------------------------------------------
 // TESTRESPONSE[s/^/{\n"took": $body.took,\n"timed_out": $body.timed_out,\n"_shards": $body._shards,\n"hits": $body.hits,\n"profile": {\n"shards": [ {\n"id": "$body.profile.shards.0.id",\n"searches": [{\n"query": $body.profile.shards.0.searches.0.query,\n"rewrite_time": $body.profile.shards.0.searches.0.rewrite_time,/]
 // TESTRESPONSE[s/]$/]}], "aggregations": []}]}}/]
-// TESTRESPONSE[s/"time": "0.3043110000ms"/"time": $body.profile.shards.0.searches.0.collector.0.time/]
-// TESTRESPONSE[s/"time": "0.03227300000ms"/"time": $body.profile.shards.0.searches.0.collector.0.children.0.time/]
+// TESTRESPONSE[s/"time_in_nanos": "304311"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.time_in_nanos/]
+// TESTRESPONSE[s/"time_in_nanos": "32273"/"time_in_nanos": $body.profile.shards.0.searches.0.collector.0.children.0.time_in_nanos/]
 
 We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting"
-`Collector` used by Elasticsearch.  The `"reason"` field attempts to give a plain english description of the class name.  The
-`"time"` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
+`Collector` used by Elasticsearch.  The `reason` field attempts to give a plain english description of the class name.  The
+`time_in_nanos` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
 all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by elasticsearch to detect if the current
 search was cancelled and stop collecting documents as soon as it occurs.
 
@@ -564,7 +568,7 @@ And the response:
                         {
                            "type": "TermQuery",
                            "description": "my_field:foo",
-                           "time": "0.4094560000ms",
+                           "time_in_nanos": "409456",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -583,7 +587,7 @@ And the response:
                         {
                            "type": "TermQuery",
                            "description": "message:search",
-                           "time": "0.3037020000ms",
+                           "time_in_nanos": "303702",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -605,24 +609,24 @@ And the response:
                         {
                            "name": "MultiCollector",
                            "reason": "search_multi",
-                           "time": "1.378943000ms",
+                           "time_in_nanos": "1378943",
                            "children": [
                               {
                                  "name": "FilteredCollector",
                                  "reason": "search_post_filter",
-                                 "time": "0.4036590000ms",
+                                 "time_in_nanos": "403659",
                                  "children": [
                                     {
                                        "name": "SimpleTopScoreDocCollector",
                                        "reason": "search_top_hits",
-                                       "time": "0.006391000000ms"
+                                       "time_in_nanos": "6391"
                                     }
                                  ]
                               },
                               {
                                  "name": "BucketCollector: [[non_global_term, another_agg]]",
                                  "reason": "aggregation",
-                                 "time": "0.9546020000ms"
+                                 "time_in_nanos": "954602"
                               }
                            ]
                         }
@@ -633,7 +637,7 @@ And the response:
                         {
                            "type": "MatchAllDocsQuery",
                            "description": "*:*",
-                           "time": "0.04829300000ms",
+                           "time_in_nanos": "48293",
                            "breakdown": {
                               "score": 0,
                               "score_count": 1,
@@ -655,7 +659,7 @@ And the response:
                         {
                            "name": "GlobalAggregator: [global_agg]",
                            "reason": "aggregation_global",
-                           "time": "0.1226310000ms"
+                           "time_in_nanos": "122631"
                         }
                      ]
                   }
@@ -738,7 +742,7 @@ Which yields the following aggregation profile output
   {
     "type": "org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator",
     "description": "property_type",
-    "time": "4280.456978ms",
+    "time_in_nanos": "4280456978",
     "breakdown": {
       "reduce": 0,
       "reduce_count": 0,
@@ -753,7 +757,7 @@ Which yields the following aggregation profile output
       {
         "type": "org.elasticsearch.search.aggregations.metrics.avg.AvgAggregator",
         "description": "avg_price",
-        "time": "1124.864392ms",
+        "time_in_nanos": "1124864392",
         "breakdown": {
           "reduce": 0,
           "reduce_count": 0,
@@ -773,16 +777,16 @@ Which yields the following aggregation profile output
 From the profile structure we can see our `property_type` terms aggregation which is internally represented by the
 `GlobalOrdinalsStringTermsAggregator` class and the sub aggregator `avg_price` which is internally represented by the `AvgAggregator` class. The `type` field displays the class used internally to represent the aggregation. The `description` field displays the name of the aggregation.
 
-The `"time"` field shows that it took ~4 seconds for the entire aggregation to execute.  The recorded time is inclusive
+The `time_in_nanos` field shows that it took ~4 seconds for the entire aggregation to execute.  The recorded time is inclusive
 of all children.
 
-The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
-that in a moment.  Finally, the `"children"` array lists any sub-aggregations that may be present.  Because we have an `avg_price` aggregation as a sub-aggregation to the `property_type` aggregation we see it listed as a child of the `property_type` aggregation.  the two aggregation outputs have identical information (type, time,
+The `breakdown` field will give detailed stats about how the time was spent, we'll look at
+that in a moment.  Finally, the `children` array lists any sub-aggregations that may be present.  Because we have an `avg_price` aggregation as a sub-aggregation to the `property_type` aggregation we see it listed as a child of the `property_type` aggregation.  the two aggregation outputs have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
 ===== Timing Breakdown
 
-The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
+The `breakdown` component lists detailed timing statistics about low-level Lucene execution:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index 852710df70b..13df8d471eb 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -183,4 +183,6 @@ include::request/named-queries-and-filters.asciidoc[]
 
 include::request/inner-hits.asciidoc[]
 
+include::request/collapse.asciidoc[]
+
 include::request/search-after.asciidoc[]
diff --git a/docs/reference/search/request/collapse.asciidoc b/docs/reference/search/request/collapse.asciidoc
new file mode 100644
index 00000000000..7845e20e152
--- /dev/null
+++ b/docs/reference/search/request/collapse.asciidoc
@@ -0,0 +1,72 @@
+[[search-request-collapse]]
+=== Field Collapsing
+
+Allows to collapse search results based on field values.
+The collapsing is done by selecting only the top sorted document per collapse key.
+For instance the query below retrieves the best tweet for each user and sorts them by number of likes.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "user" <1>
+    },
+    "sort": ["likes"], <2>
+    "from": 10 <3>
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+<1> collapse the result set using the "user" field
+<2> sort the top docs by number of likes
+<3> define the offset of the first collapsed result
+
+WARNING: The total number of hits in the response indicates the number of matching documents without collapsing.
+The total number of distinct group is unknown.
+
+The field used for collapsing must be a single valued <<keyword, `keyword`>> or <<number, `numeric`>> field with <<doc-values, `doc_values`>> activated
+
+NOTE: The collapsing is applied to the top hits only and does not affect aggregations.
+
+
+==== Expand collapse results
+
+It is also possible to expand each collapsed top hits with the `inner_hits` option.
+
+[source,js]
+--------------------------------------------------
+GET /twitter/tweet/_search
+{
+    "query": {
+        "match": {
+            "message": "elasticsearch"
+        }
+    },
+    "collapse" : {
+        "field" : "user", <1>
+        "inner_hits": {
+            "name": "last_tweets", <2>
+            "size": 5, <3>
+            "sort": [{ "date": "asc" }] <4>
+        }
+    },
+    "sort": ["likes"]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+<1> collapse the result set using the "user" field
+<2> the name used for the inner hit section in the response
+<3> the number of inner_hits to retrieve per collapse key
+<4> how to sort the document inside each group
+
+See <<search-request-inner-hits, inner hits>> for the complete list of supported options and the format of the response.
+
+WARNING: `collapse` cannot be used in conjunction with <<search-request-scroll, scroll>>,
+<<search-request-rescore, rescore>> or <<search-request-search-after, search after>>.
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index 73ade7a47d6..30c0e20d5bf 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -11,7 +11,7 @@ The following is an example of the search request body:
 GET /_search
 {
     "query" : {
-        "match": { "user": "kimchy" }
+        "match": { "content": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -126,6 +126,22 @@ the index to be bigger):
 }
 --------------------------------------------------
 
+==== Unified Highlighter
+
+experimental[]
+
+The `unified` highlighter can extract offsets from either postings, term vectors, or via re-analyzing text.
+Under the hood it uses Lucene UnifiedHighlighter which picks its strategy depending on the field and the query to highlight.
+Independently of the strategy this highlighter breaks the text into sentences and scores individual sentences as
+if they were documents in this corpus, using the BM25 algorithm.
+It supports accurate phrase and multi-term (fuzzy, prefix, regex) highlighting and can be used with the following options:
+
+* `force_source`
+* `encoder`
+* `highlight_query`
+* `pre_tags and `post_tags`
+* `require_field_match`
+
 ==== Force highlighter type
 
 The `type` field allows to force a specific highlighter type. This is useful
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 431f649aa08..d0ec2321a3c 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -165,7 +165,10 @@ scripts and sorting by geo distance.
 The `missing` parameter specifies how docs which are missing
 the field should be treated: The `missing` value can be
 set to `_last`, `_first`, or a custom value (that
-will be used for missing docs as the sort value). For example:
+will be used for missing docs as the sort value).
+The default is `_last`.
+
+For example:
 
 [source,js]
 --------------------------------------------------
@@ -226,7 +229,7 @@ GET /_search
                 "order" : "asc",
                 "unit" : "km",
 		"mode" : "min",
-		"distance_type" : "sloppy_arc"
+		"distance_type" : "arc"
             }
         }
     ],
@@ -241,7 +244,7 @@ GET /_search
 
 `distance_type`::
 
-    How to compute the distance. Can either be `sloppy_arc` (default), `arc` (slightly more precise but significantly slower) or `plane` (faster, but inaccurate on long distances and close to the poles).
+    How to compute the distance. Can either be `arc` (default), or `plane` (faster, but inaccurate on long distances and close to the poles).
 
 `mode`::
 
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
index f09c332553c..b2a64c7e141 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -52,7 +52,7 @@ things like `_source.obj1.field1` can be used, though not recommended, as
 
 ==== Disable stored fields entirely
 
-To disable the stored fields (and metadata fields) entirely use: `\_none_`:
+To disable the stored fields (and metadata fields) entirely use: `_none_`:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 7c6a0390ea6..2320c8bd501 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -333,6 +333,51 @@ via the REST API, should be written as a string:
 
 ==================================
 
+
+[float]
+===== Encoding URLs
+
+The `{{#url}}value{{/url}}` function can be used to encode a string value
+in a HTML encoding form as defined in by the http://www.w3.org/TR/html4/[HTML specification].
+
+As an example, it is useful to encode a URL:
+
+[source,js]
+------------------------------------------
+GET /_render/template
+{
+    "inline" : {
+        "query" : {
+            "term": {
+                "http_access_log": "{{#url}}{{host}}/{{page}}{{/url}}"
+            }
+        }
+    },
+    "params": {
+        "host": "https://www.elastic.co/",
+        "page": "learn"
+    }
+}
+------------------------------------------
+// CONSOLE
+
+The previous query will be rendered as:
+
+[source,js]
+------------------------------------------
+{
+    "template_output" : {
+        "query" : {
+            "term" : {
+                "http_access_log" : "https%3A%2F%2Fwww.elastic.co%2F%2Flearn"
+            }
+        }
+    }
+}
+------------------------------------------
+// TESTRESPONSE
+
+
 [float]
 [[pre-registered-templates]]
 ===== Pre-registered template
diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc
index 8b7b897934d..5c98f3f7f2d 100644
--- a/docs/reference/search/validate.asciidoc
+++ b/docs/reference/search/validate.asciidoc
@@ -87,7 +87,15 @@ due to dynamic mapping, and 'foo' does not correctly parse into a date:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?q=post_date:foo%5d
+GET twitter/tweet/_validate/query
+{
+  "query": {
+    "query_string": {
+      "query": "post_date:foo",
+      "lenient": false
+    }
+  }
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -102,7 +110,15 @@ about why a query failed:
 
 [source,js]
 --------------------------------------------------
-GET twitter/tweet/_validate/query?q=post_date:foo%5d&explain=true
+GET twitter/tweet/_validate/query?explain=true
+{
+  "query": {
+    "query_string": {
+      "query": "post_date:foo",
+      "lenient": false
+    }
+  }
+}
 --------------------------------------------------
 // CONSOLE
 
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 2608f461dba..e616b444dc2 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -46,6 +46,8 @@ include::setup/configuration.asciidoc[]
 
 include::setup/important-settings.asciidoc[]
 
+include::setup/secure-settings.asciidoc[]
+
 include::setup/bootstrap-checks.asciidoc[]
 
 include::setup/sysconfig.asciidoc[]
diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index ac8855bf3d8..706398f4c39 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -112,22 +112,30 @@ command line with `es.node.name` or in the config file with `node.name`.
 
 Elasticsearch uses http://logging.apache.org/log4j/2.x/[Log4j 2] for
 logging. Log4j 2 can be configured using the log4j2.properties
-file. Elasticsearch exposes a single property `${sys:es.logs}` that can be
-referenced in the configuration file to determine the location of the log files;
-this will resolve to a prefix for the Elasticsearch log file at runtime.
+file. Elasticsearch exposes three properties, `${sys:es.logs.base_path},
+`${sys:es.logs.cluster_name}`, and `${sys:es.logs.node_name} (if the node name
+is explicitly set via `node.name`) that can be referenced in the configuration
+file to determine the location of the log files. The property
+`${sys:es.logs.base_path}` will resolve to the log directory,
+`${sys:es.logs.cluster_name}` will resolve to the cluster name (used as the
+prefix of log filenames in the default configuration), and
+`${sys:es.logs.node_name}` will resolve to the node name (if the node name is
+explicitly set).
 
 For example, if your log directory (`path.logs`) is `/var/log/elasticsearch` and
-your cluster is named `production` then `${sys:es.logs}` will resolve to
-`/var/log/elasticsearch/production`.
+your cluster is named `production` then `${sys:es.logs.base_path}` will resolve
+to `/var/log/elasticsearch` and
+`${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log`
+will resolve to `/var/log/elasticsearch/production.log`.
 
 [source,properties]
 --------------------------------------------------
 appender.rolling.type = RollingFile <1>
 appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log <2>
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log <2>
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log <3>
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log <3>
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy <4>
 appender.rolling.policies.time.interval = 1 <5>
@@ -145,6 +153,31 @@ appender.rolling.policies.time.modulate = true <6>
 If you append `.gz` or `.zip` to `appender.rolling.filePattern`, then the logs
 will be compressed as they are rolled.
 
+If you want to retain log files for a specified period of time, you can use a
+rollover strategy with a delete action.
+
+[source,properties]
+--------------------------------------------------
+appender.rolling.strategy.type = DefaultRolloverStrategy <1>
+appender.rolling.strategy.action.type = Delete <2>
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} <3>
+appender.rolling.strategy.action.condition.type = IfLastModified <4>
+appender.rolling.strategy.action.condition.age = 7D <5>
+appender.rolling.strategy.action.PathConditions.type = IfFileName <6>
+appender.rolling.strategy.action.PathConditions.glob = ${sys:es.logs.cluster_name}-* <7>
+--------------------------------------------------
+
+<1> Configure the `DefaultRolloverStrategy`
+<2> Configure the `Delete` action for handling rollovers
+<3> The base path to the Elasticsearch logs
+<4> The condition to apply when handling rollovers
+<5> Retain logs for seven days
+<6> Only delete files older than seven days if they match the specified glob
+<7> Delete files from the base path matching the glob
+    `${sys:es.logs.cluster_name}-*`; this is the glob that log files are rolled
+    to; this is needed to only delete the rolled Elasticsearch logs but not also
+    delete the deprecation and slow logs
+
 Multiple configuration files can be loaded (in which case they will get merged)
 as long as they are named `log4j2.properties` and have the Elasticsearch config
 directory as an ancestor; this is useful for plugins that expose additional
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index bdf7c314476..fc43f5869a9 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -150,7 +150,7 @@ discovery.zen.ping.unicast.hosts:
    - 192.168.1.11 <1>
    - seeds.mydomain.com <2>
 --------------------------------------------------
-<1> The port will default to 9300 if not specified.
+<1> The port will default to `transport.profiles.default.port` and fallback to `transport.tcp.port` if not specified.
 <2> A hostname that resolves to multiple IP addresses will try all resolved addresses.
 
 [float]
diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index 6ca635ce06e..65965fa1bd0 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -62,18 +62,23 @@ echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main"
 endif::[]
 
 
-[WARNING]
+[NOTE]
 ==================================================
 
-Do not use `add-apt-repository` as it will add a `deb-src` entry as well, but
-we do not provide a source package. If you have added the `deb-src` entry, you
-will see an error like the following:
+These instructions do not use `add-apt-repository` for several reasons:
+
+. `add-apt-repository` adds entries to the system `/etc/apt/sources.list` file
+  rather than a clean per-repository file in `/etc/apt/sources.list.d`
+. `add-apt-repository` is not part of the default install on many distributions
+  and requires a number of non-default dependencies.
+. Older versions of `add-apt-repository` always add a `deb-src` entry which
+  will cause errors because we do not provide a source package. If you have added
+  the `deb-src` entry, you will see an error like the following until you delete
+  the `deb-src` line:
 
     Unable to find expected entry 'main/source/Sources' in Release file
     (Wrong sources.list entry or malformed file)
 
-Delete the `deb-src` entry from the `/etc/apt/sources.list` file and the
-installation should work as expected.
 ==================================================
 
 You can install the Elasticsearch Debian package with:
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
index 4f7f9647d2b..d5a82f7e1fa 100644
--- a/docs/reference/setup/install/docker.asciidoc
+++ b/docs/reference/setup/install/docker.asciidoc
@@ -248,7 +248,7 @@ In some environments, it may make more sense to prepare a custom image containin
 FROM docker.elastic.co/elasticsearch/elasticsearch:{version}
 ADD elasticsearch.yml /usr/share/elasticsearch/config/
 USER root
-chown elasticsearch:elasticsearch config/elasticsearch.yml
+RUN chown elasticsearch:elasticsearch config/elasticsearch.yml
 USER elasticsearch
 --------------------------------------------
 
diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc
index 981f2c240d3..f891aa2058e 100644
--- a/docs/reference/setup/rolling_upgrade.asciidoc
+++ b/docs/reference/setup/rolling_upgrade.asciidoc
@@ -69,7 +69,8 @@ default.
 It is a good idea to place these directories in a different location so that
 there is no chance of deleting them when upgrading Elasticsearch.  These
 custom paths can be <<path-settings,configured>> with the `path.conf`,
-`path.logs`, and `path.data` settings.
+`path.logs`, and `path.data` settings, and using `ES_JVM_OPTIONS` to specify
+the location of the `jvm.options` file.
 
 The <<deb,Debian>> and <<rpm,RPM>> packages place these directories in the
 appropriate place for each operating system.
@@ -88,8 +89,9 @@ To upgrade using a zip or compressed tarball:
     overwrite the `config` or `data` directories.
 
 *   Either copy the files in the `config` directory from your old installation
-    to your new installation, or use the `-E path.conf=` option on the command
-    line to point to an external config directory.
+    to your new installation, or set the environment variable `ES_JVM_OPTIONS`
+    to the location of the `jvm.options` file and use the `-E path.conf=`
+    option on the command line to point to an external config directory.
 
 *   Either copy the files in the `data` directory from your old installation
     to your new installation, or configure the location of the data directory
diff --git a/docs/reference/setup/secure-settings.asciidoc b/docs/reference/setup/secure-settings.asciidoc
new file mode 100644
index 00000000000..d66360abf6c
--- /dev/null
+++ b/docs/reference/setup/secure-settings.asciidoc
@@ -0,0 +1,65 @@
+[[secure-settings]]
+== Secure Settings
+
+Some settings are sensitive, and relying on filesystem permissions to protect
+their values is not sufficient. For this use case, elasticsearch provides a
+keystore, which may be password protected, and the `elasticsearch-keystore`
+tool to manage the settings in the keystore.
+
+NOTE: All commands here should be run as the user which will run elasticsearch.
+
+[float]
+[[creating-keystore]]
+=== Creating the keystore
+
+To create the `elasticsearch.keystore`, use the `create` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore create
+----------------------------------------------------------------
+
+The file `elasticsearch.keystore` will be created alongside `elasticsearch.yml`.
+
+[float]
+[[list-settings]]
+=== Listing settings in the keystore
+
+A list of the settings in the keystore is available with the `list` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore list 
+----------------------------------------------------------------
+
+[float]
+[[add-string-to-keystore]]
+=== Adding string settings
+
+Sensitive string settings, like authentication credentials for cloud
+plugins, can be adding using the `add` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore add the.setting.name.to.set
+----------------------------------------------------------------
+
+The tool will prompt for the value of the setting. To pass the value
+through stdin, use the `--stdin` flag:
+
+[source,sh]
+----------------------------------------------------------------
+cat /file/containing/setting/value | bin/elasticsearch-keystore add --stdin the.setting.name.to.set
+----------------------------------------------------------------
+
+[float]
+[[remove-settings]]
+=== Removing settings
+
+To remove a setting from the keystore, use the `remove` command:
+
+[source,sh]
+----------------------------------------------------------------
+bin/elasticsearch-keystore remove the.setting.name.to.remove
+----------------------------------------------------------------
+
diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc
index 08752e6fd93..19b6f751ee7 100644
--- a/docs/reference/setup/sysconfig/swap.asciidoc
+++ b/docs/reference/setup/sysconfig/swap.asciidoc
@@ -92,7 +92,7 @@ via `System Properties → Advanced → Performance → Advanced → Virtual mem
 [[swappiness]]
 ==== Configure `swappiness`
 
-The second option available on Linux systems is to ensure that the sysctl value
+Another option available on Linux systems is to ensure that the sysctl value
 `vm.swappiness` is set to `1`. This reduces the kernel's tendency to swap and
 should not lead to swapping under normal circumstances, while still allowing
 the whole system to swap in emergency conditions.
diff --git a/docs/src/test/cluster/config/scripts/my_script.js b/docs/src/test/cluster/config/scripts/my_script.js
deleted file mode 100644
index cc985c18a94..00000000000
--- a/docs/src/test/cluster/config/scripts/my_script.js
+++ /dev/null
@@ -1 +0,0 @@
-doc["num"].value * factor
diff --git a/docs/src/test/cluster/config/scripts/my_script.painless b/docs/src/test/cluster/config/scripts/my_script.painless
new file mode 100644
index 00000000000..55d0e99ed0f
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_script.painless
@@ -0,0 +1,2 @@
+// Simple script to load a field. Not really a good example, but a simple one.
+doc[params.field].value
diff --git a/docs/src/test/cluster/config/scripts/my_script.py b/docs/src/test/cluster/config/scripts/my_script.py
deleted file mode 100644
index cc985c18a94..00000000000
--- a/docs/src/test/cluster/config/scripts/my_script.py
+++ /dev/null
@@ -1 +0,0 @@
-doc["num"].value * factor
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
index 87ca5acf1ca..a79730339e3 100644
--- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
+++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 import java.util.List;
@@ -36,7 +35,7 @@ public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/migrate.sh b/migrate.sh
deleted file mode 100755
index 73b87f0805c..00000000000
--- a/migrate.sh
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/bin/bash
-set -e
-set -u
-set -o pipefail
-
-# This script creates a new branch then migrate all the maven stuff in it and commit that branch.
-# then it runs mvn install to check that everything is still running
-
-# Useful commands while testing
-# rm -rf migration/tmp plugins
-# git checkout pr/migration_script -f
-# git add .; git commit -m "Add migration script" --amend
-# ./migrate.sh 
-# mvn clean install -DskipTests
-
-GIT_BRANCH="refactoring/add_lang"
-
-# Insert a new text after a given line
-# insertLinesAfter text_to_find text_to_add newLine_separator filename
-# insertLinesAfter "<\/project>" "   <modules>=   <\/modules>" "§" "pom.xml"
-function insertLinesAfter() {
-	# echo "## modify $4 with $2"
-	sed "s/$1/$1$3$2/" $4 | tr "$3" "\n" > $4.new
-	mv $4.new $4
-}
-
-# Insert a new text before a given line
-# insertLinesBefore text_to_find text_to_add newLine_separator filename
-# insertLinesBefore "<\/project>" "   <modules>=   <\/modules>" "§" "pom.xml"
-function insertLinesBefore() {
-	# echo "## modify $4 with $2"
-	sed "s/$1/$2$3$1/" $4 | tr "$3" "\n" > $4.new
-	mv $4.new $4
-}
-
-# Replace text in a file
-# replaceLine old_text new_text filename
-# replaceLine "old" "new" "pom.xml"
-function replaceLine() {
-	# echo "## modify $3 with $2"
-	sed "s/^$1/$2/" $3 > $3.new
-	mv $3.new $3
-}
-
-# Remove lines in a file
-# removeLines from_text_included to_text_included filename
-# removeLines "old" "new" "pom.xml"
-function removeLines() {
-	# echo "## remove lines in $3 from $1 to $2"
-	sed "/$1/,/$2/d" $3 > $3.new
-	mv $3.new $3
-}
-
-# Migrate a plugin
-# migratePlugin short_name
-function migratePlugin() {
-	PLUGIN_GIT_REPO="https://github.com/elastic/elasticsearch-$1.git"
-	git remote add remote_$1 $PLUGIN_GIT_REPO
-	git fetch remote_$1 > /dev/null 2>/dev/null
-	echo "## Add $1 module from $PLUGIN_GIT_REPO"
-
-	# echo "### fetch $1 project from $PLUGIN_GIT_REPO in plugins"
-	# If you want to run that locally, uncomment this line and comment one below
-	#cp -R ../elasticsearch-$1/* plugins/$1
-	# git clone $PLUGIN_GIT_REPO plugins/$1 > /dev/null 2>/dev/null
-	git checkout -b migrate_$1 remote_$1/master > /dev/null 2>/dev/null
-	git remote rm remote_$1 > /dev/null 2>/dev/null
-	mkdir -p plugins/$1
-	git mv -k * plugins/$1 > /dev/null 2>/dev/null       
-	git rm .gitignore > /dev/null 2>/dev/null
-	# echo "### change $1 groupId to org.elasticsearch.plugin"
-	# Change the groupId to avoid conflicts with existing 2.0.0 versions.
-	replaceLine "    <groupId>org.elasticsearch<\/groupId>" "    <groupId>org.elasticsearch.plugin<\/groupId>" "plugins/$1/pom.xml"
-
-	# echo "### cleanup $1 pom.xml"
-	removeLines "<issueManagement>" "<\/issueManagement>" "plugins/$1/pom.xml"
-	removeLines "<repositories>" "<\/repositories>" "plugins/$1/pom.xml"
-	removeLines "<url>" "<\/scm>" "plugins/$1/pom.xml"
-
-        # echo "### remove version 5.0.0-SNAPSHOT from $1 pom.xml"
-        # All plugins for ES 5.0.0 uses 5.0.0-SNAPSHOT version number
-        replaceLine "    <version>5.0.0-SNAPSHOT<\/version>" "" "plugins/$1/pom.xml"
-
-	# echo "### remove unused dev-tools and .git dirs and LICENSE.txt and CONTRIBUTING.md files"
-	rm -r plugins/$1/dev-tools
-	rm -rf plugins/$1/.git
-	rm plugins/$1/LICENSE.txt
-	rm plugins/$1/CONTRIBUTING.md
-
-	# echo "### commit changes"
-	git add --all .
-	git commit -m "add $1 module" > /dev/null
-	git checkout $GIT_BRANCH
-	git merge -m "migrate branch for $1" migrate_$1 > /dev/null 2>/dev/null
-	# We can now add plugin module
-	insertLinesBefore "    <\/modules>" "        <module>$1<\/module>" "§" "plugins/pom.xml"
-	git add .
-	git commit -m "add $1 module" > /dev/null
-	git branch -D migrate_$1
-}
-
-echo "# STEP 0 : prepare the job"
-
-
-# echo "## create git $GIT_BRANCH work branch"
-
-# It first clean the existing branch if any
-echo "(You can safely ignore branch not found below)"
-git branch -D $GIT_BRANCH > /dev/null || :
-
-# Create the new branch
-git branch $GIT_BRANCH > /dev/null
-git checkout $GIT_BRANCH > /dev/null 2>/dev/null
-
-
-echo "# STEP 4 : Migrate plugins"
-
-# Analysis
-migratePlugin "lang-python" 
-migratePlugin "lang-javascript"
-
-
-echo "you can now run:"
-echo "mvn clean install -DskipTests"
-
-
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
index 95374069b14..40e227681c4 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -40,19 +39,18 @@ import java.util.Map;
 public class MatrixStatsAggregationBuilder
     extends MultiValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregationBuilder> {
     public static final String NAME = "matrix_stats";
-    public static final Type TYPE = new Type(NAME);
 
     private MultiValueMode multiValueMode = MultiValueMode.AVG;
 
     public MatrixStatsAggregationBuilder(String name) {
-        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MatrixStatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
@@ -72,7 +70,7 @@ public class MatrixStatsAggregationBuilder
     @Override
     protected MatrixStatsAggregatorFactory innerBuild(SearchContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new MatrixStatsAggregatorFactory(name, type, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
+        return new MatrixStatsAggregatorFactory(name, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
@@ -92,7 +90,7 @@ public class MatrixStatsAggregationBuilder
     }
 
     @Override
-    public String getWriteableName() {
+    public String getType() {
         return NAME;
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
index 94b0b37dc36..c991e2c5c86 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -22,7 +22,6 @@ import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -38,11 +37,11 @@ public class MatrixStatsAggregatorFactory
 
     private final MultiValueMode multiValueMode;
 
-    public MatrixStatsAggregatorFactory(String name, InternalAggregation.Type type,
+    public MatrixStatsAggregatorFactory(String name,
             Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, MultiValueMode multiValueMode,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+        super(name, configs, context, parent, subFactoriesBuilder, metaData);
         this.multiValueMode = multiValueMode;
     }
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
index 6e7eef863cb..1db49b50b88 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.matrix.stats;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceParser.NumericValuesSourceParser;
@@ -39,8 +38,8 @@ public class MatrixStatsParser extends NumericValuesSourceParser {
 
     @Override
     protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
-                            ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
-        if (parseFieldMatcher.match(currentFieldName, MULTIVALUE_MODE_FIELD)) {
+                            Map<ParseField, Object> otherOptions) throws IOException {
+        if (MULTIVALUE_MODE_FIELD.match(currentFieldName)) {
             if (token == XContentParser.Token.VALUE_STRING) {
                 otherOptions.put(MULTIVALUE_MODE_FIELD, parser.text());
                 return true;
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
index 1ae29e65761..f82c6df73be 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -39,13 +39,13 @@ class MatrixStatsResults implements Writeable {
     protected final Map<String, HashMap<String, Double>> correlation;
 
     /** Base ctor */
-    public MatrixStatsResults() {
+    MatrixStatsResults() {
         results = new RunningStats();
         this.correlation = new HashMap<>();
     }
 
     /** creates and computes result from provided stats */
-    public MatrixStatsResults(RunningStats stats) {
+    MatrixStatsResults(RunningStats stats) {
         this.results = stats.clone();
         this.correlation = new HashMap<>();
         this.compute();
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index fde3d24792c..8ac189167b6 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -29,12 +29,11 @@ import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -53,29 +52,29 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     public abstract static class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
             extends MultiValuesSourceAggregationBuilder<VS, AB> {
 
-        protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-            super(name, type, valuesSourceType, targetValueType);
+        protected LeafOnly(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            super(name, valuesSourceType, targetValueType);
         }
 
         /**
          * Read from a stream that does not serialize its targetValueType. This should be used by most subclasses.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
-            super(in, type, valuesSourceType, targetValueType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
+            super(in, valuesSourceType, targetValueType);
         }
 
         /**
          * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
          * {@link #serializeTargetValueType()} to return true.
          */
-        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-            super(in, type, valuesSourceType);
+        protected LeafOnly(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+            super(in, valuesSourceType);
         }
 
         @Override
         public AB subAggregations(Builder subFactories) {
             throw new AggregationInitializationException("Aggregator [" + name + "] of type [" +
-                type + "] cannot accept sub-aggregations");
+                getType() + "] cannot accept sub-aggregations");
         }
     }
 
@@ -87,8 +86,8 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
     private Object missing = null;
     private Map<String, Object> missingMap = Collections.emptyMap();
 
-    protected MultiValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
-        super(name, type);
+    protected MultiValuesSourceAggregationBuilder(String name, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        super(name);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
         }
@@ -96,17 +95,17 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
         this.targetValueType = targetValueType;
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType, ValueType targetValueType)
         throws IOException {
-        super(in, type);
+        super(in);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = targetValueType;
         read(in);
     }
 
-    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
-        super(in, type);
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, ValuesSourceType valuesSourceType) throws IOException {
+        super(in);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
         this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream);
@@ -246,7 +245,6 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
 
         if (field == null) {
             if (script == null) {
-                @SuppressWarnings("unchecked")
                 ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
                 return config.format(resolveFormat(null, valueType));
             }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
index 23e44a5da17..7d5c56a571b 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.support;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -36,10 +35,10 @@ public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource
 
     protected Map<String, ValuesSourceConfig<VS>> configs;
 
-    public MultiValuesSourceAggregatorFactory(String name, Type type, Map<String, ValuesSourceConfig<VS>> configs,
+    public MultiValuesSourceAggregatorFactory(String name, Map<String, ValuesSourceConfig<VS>> configs,
             SearchContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
             Map<String, Object> metaData) throws IOException {
-        super(name, type, context, parent, subFactoriesBuilder, metaData);
+        super(name, context, parent, subFactoriesBuilder, metaData);
         this.configs = configs;
     }
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
index 530c13ca2ce..c2e4b4f0c3f 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
@@ -20,13 +20,12 @@
 package org.elasticsearch.search.aggregations.support;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder.CommonFields;
+import org.elasticsearch.search.aggregations.Aggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -85,47 +84,45 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
         String format = null;
         Map<String, Object> missingMap = null;
         Map<ParseField, Object> otherOptions = new HashMap<>();
-        final ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
-
         XContentParser.Token token;
         String currentFieldName = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.VALUE_STRING) {
-                if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                if (CommonFields.FIELDS.match(currentFieldName)) {
                     fields = Collections.singletonList(parser.text());
-                } else if (formattable && parseFieldMatcher.match(currentFieldName, CommonFields.FORMAT)) {
+                } else if (formattable && CommonFields.FORMAT.match(currentFieldName)) {
                     format = parser.text();
-                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.VALUE_TYPE)) {
+                } else if (CommonFields.VALUE_TYPE.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseFieldMatcher.match(currentFieldName, CommonFields.MISSING)) {
+                if (CommonFields.MISSING.match(currentFieldName)) {
                     missingMap = new HashMap<>();
                     while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
                         parseMissingAndAdd(aggregationName, currentFieldName, parser, missingMap);
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
+                } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
 
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, Script.SCRIPT_PARSE_FIELD)) {
+                if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                     throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
                             "Multi-field aggregations do not support scripts.");
-                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                } else if (CommonFields.FIELDS.match(currentFieldName)) {
                     fields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -135,11 +132,11 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
                                     "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                         }
                     }
-                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                     throw new ParsingException(parser.getTokenLocation(),
                             "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
                 }
-            } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+            } else if (!token(aggregationName, currentFieldName, token, parser, otherOptions)) {
                 throw new ParsingException(parser.getTokenLocation(),
                         "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
             }
@@ -198,8 +195,7 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
      *            the target type of the final value output by the aggregation
      * @param otherOptions
      *            a {@link Map} containing the extra options parsed by the
-     *            {@link #token(String, String, org.elasticsearch.common.xcontent.XContentParser.Token,
-     *             XContentParser, ParseFieldMatcher, Map)}
+     *            {@link #token(String, String, XContentParser.Token, XContentParser, Map)}
      *            method
      * @return the created factory
      */
@@ -219,8 +215,6 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
      *            the current token for the parser
      * @param parser
      *            the parser
-     * @param parseFieldMatcher
-     *            the {@link ParseFieldMatcher} to use to match field names
      * @param otherOptions
      *            a {@link Map} of options to be populated by successive calls
      *            to this method which will then be passed to the
@@ -232,5 +226,5 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
      *             if an error occurs whilst parsing
      */
     protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
-        ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException;
+                                     Map<ParseField, Object> otherOptions) throws IOException;
 }
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
index 11ddd2dfd41..6e11a56d2c6 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -33,7 +32,7 @@ public class MatrixStatsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
index 4a97a7e7ab2..9ead2d05f7d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -22,6 +22,7 @@ package org.elasticsearch.ingest.common;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
 import java.util.Locale;
@@ -65,9 +66,8 @@ enum DateFormat {
     Joda {
         @Override
         Function<String, DateTime> getFunction(String format, DateTimeZone timezone, Locale locale) {
-            return DateTimeFormat.forPattern(format)
-                .withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear())
-                .withZone(timezone).withLocale(locale)::parseDateTime;
+            DateTimeFormatter parser = DateTimeFormat.forPattern(format).withZone(timezone).withLocale(locale);
+            return text -> parser.withDefaultYear((new DateTime(DateTimeZone.UTC)).getYear()).parseDateTime(text);
         }
     };
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
index 7120643f47c..1902d9900f7 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
@@ -61,12 +61,12 @@ final class Grok {
     private final String expression;
 
 
-    public Grok(Map<String, String> patternBank, String grokPattern) {
+    Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true);
     }
 
     @SuppressWarnings("unchecked")
-    public Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
+    Grok(Map<String, String> patternBank, String grokPattern, boolean namedCaptures) {
         this.patternBank = patternBank;
         this.namedCaptures = namedCaptures;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
index 35f8fd312ff..924e298e462 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
@@ -26,7 +26,7 @@ final class GrokMatchGroup {
     private final String type;
     private final String groupValue;
 
-    public GrokMatchGroup(String groupName, String groupValue) {
+    GrokMatchGroup(String groupName, String groupValue) {
         String[] parts = groupName.split(":");
         patternName = parts[0];
         if (parts.length >= 2) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
index 82d316dfa62..8b6c8e8bed8 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
@@ -63,6 +63,7 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin {
         processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
         processors.put(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory());
         processors.put(JsonProcessor.TYPE, new JsonProcessor.Factory());
+        processors.put(KeyValueProcessor.TYPE, new KeyValueProcessor.Factory());
         return Collections.unmodifiableMap(processors);
     }
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
index cb734e7bef4..d4c7efe0d50 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -19,7 +19,8 @@
 
 package org.elasticsearch.ingest.common;
 
-import com.fasterxml.jackson.core.JsonParseException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -65,7 +66,7 @@ public final class JsonProcessor extends AbstractProcessor {
     public void execute(IngestDocument document) throws Exception {
         String stringValue = document.getFieldValue(field, String.class);
         try {
-            Map<String, Object> mapValue = JsonXContent.jsonXContent.createParser(stringValue).map();
+            Map<String, Object> mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false);
             if (addToRoot) {
                 for (Map.Entry<String, Object> entry : mapValue.entrySet()) {
                     document.setFieldValue(entry.getKey(), entry.getValue());
@@ -73,7 +74,7 @@ public final class JsonProcessor extends AbstractProcessor {
             } else {
                 document.setFieldValue(targetField, mapValue);
             }
-        } catch (JsonParseException e) {
+        } catch (ElasticsearchParseException e) {
             throw new IllegalArgumentException(e);
         }
     }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
new file mode 100644
index 00000000000..3132439cd7d
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The KeyValueProcessor parses and extracts messages of the `key=value` variety into fields with values of the keys.
+ */
+public final class KeyValueProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "kv";
+
+    private final String field;
+    private final String fieldSplit;
+    private final String valueSplit;
+    private final List<String> includeKeys;
+    private final String targetField;
+    private final boolean ignoreMissing;
+
+    KeyValueProcessor(String tag, String field, String fieldSplit, String valueSplit, List<String> includeKeys,
+                      String targetField, boolean ignoreMissing) {
+        super(tag);
+        this.field = field;
+        this.targetField = targetField;
+        this.fieldSplit = fieldSplit;
+        this.valueSplit = valueSplit;
+        this.includeKeys = includeKeys;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    String getField() {
+        return field;
+    }
+
+    String getFieldSplit() {
+        return fieldSplit;
+    }
+
+    String getValueSplit() {
+        return valueSplit;
+    }
+
+    List<String> getIncludeKeys() {
+        return includeKeys;
+    }
+
+    String getTargetField() {
+        return targetField;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
+    }
+
+    public void append(IngestDocument document, String targetField, String value) {
+        if (document.hasField(targetField)) {
+            document.appendFieldValue(targetField, value);
+        } else {
+            document.setFieldValue(targetField, value);
+        }
+    }
+
+    @Override
+    public void execute(IngestDocument document) {
+        String oldVal = document.getFieldValue(field, String.class, ignoreMissing);
+
+        if (oldVal == null && ignoreMissing) {
+            return;
+        } else if (oldVal == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot extract key-value pairs.");
+        }
+
+        String fieldPathPrefix = (targetField == null) ? "" : targetField + ".";
+        Arrays.stream(oldVal.split(fieldSplit))
+            .map((f) -> {
+                String[] kv = f.split(valueSplit, 2);
+                if (kv.length != 2) {
+                    throw new IllegalArgumentException("field [" + field + "] does not contain value_split [" + valueSplit + "]");
+                }
+                return kv;
+            })
+            .filter((p) -> includeKeys == null || includeKeys.contains(p[0]))
+            .forEach((p) -> append(document, fieldPathPrefix + p[0], p[1]));
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static class Factory implements Processor.Factory {
+        @Override
+        public KeyValueProcessor create(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");
+            String fieldSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field_split");
+            String valueSplit = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "value_split");
+            List<String> includeKeys = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "include_keys");
+            if (includeKeys != null) {
+                includeKeys = Collections.unmodifiableList(includeKeys);
+            }
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new KeyValueProcessor(processorTag, field, fieldSplit, valueSplit, includeKeys, targetField, ignoreMissing);
+        }
+    }
+}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 4480ca1da58..33c06a3804a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.ingest.common;
 
-import java.util.Map;
-
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
@@ -30,7 +28,8 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptType;
+
+import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.Strings.hasLength;
@@ -139,7 +138,7 @@ public final class ScriptProcessor extends AbstractProcessor {
 
             // verify script is able to be compiled before successfully creating processor.
             try {
-                scriptService.compile(script, ScriptContext.Standard.INGEST, script.getOptions());
+                scriptService.compile(script, ScriptContext.Standard.INGEST);
             } catch (ScriptException e) {
                 throw newConfigurationException(TYPE, processorTag, scriptPropertyUsed, e);
             }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index 768f402ee4b..8ac5a56abb0 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -106,13 +106,13 @@ public class DateProcessorTests extends ESTestCase {
 
     public void testJodaPatternDefaultYear() {
         DateProcessor dateProcessor = new DateProcessor(randomAsciiOfLength(10), DateTimeZone.forID("Europe/Amsterdam"), Locale.ENGLISH,
-                "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
+            "date_as_string", Collections.singletonList("dd/MM"), "date_as_date");
         Map<String, Object> document = new HashMap<>();
         document.put("date_as_string", "12/06");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         dateProcessor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(DateTime.now().getYear() +
-                "-06-12T00:00:00.000+02:00"));
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class),
+            equalTo(DateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
     }
 
     public void testTAI64N() {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
index 1b678835c4b..4027a75a2d1 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestCommonClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index 2b2c521417c..5d087ebbc12 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -60,9 +60,9 @@ public class JsonProcessorTests extends ESTestCase {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(exception.getMessage(), equalTo("com.fasterxml.jackson.core.JsonParseException: Unrecognized token" +
-            " 'invalid': was expecting ('true', 'false' or 'null')\n" +
-            " at [Source: invalid json; line: 1, column: 8]"));
+        assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token"
+                + " 'invalid': was expecting ('true', 'false' or 'null')\n"
+                + " at [Source: invalid json; line: 1, column: 8]"));
     }
 
     public void testFieldMissing() {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
new file mode 100644
index 00000000000..4dc4e082655
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class KeyValueProcessorFactoryTests extends ESTestCase {
+
+    public void testCreateWithDefaults() throws Exception {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        config.put("value_split", "=");
+        String processorTag = randomAsciiOfLength(10);
+        KeyValueProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("field1"));
+        assertThat(processor.getFieldSplit(), equalTo("&"));
+        assertThat(processor.getValueSplit(), equalTo("="));
+        assertThat(processor.getIncludeKeys(), is(nullValue()));
+        assertThat(processor.getTargetField(), is(nullValue()));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testCreateWithAllFieldsSet() throws Exception {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        config.put("value_split", "=");
+        config.put("target_field", "target");
+        config.put("include_keys", Arrays.asList("a", "b"));
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+        KeyValueProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("field1"));
+        assertThat(processor.getFieldSplit(), equalTo("&"));
+        assertThat(processor.getValueSplit(), equalTo("="));
+        assertThat(processor.getIncludeKeys(), equalTo(Arrays.asList("a", "b")));
+        assertThat(processor.getTargetField(), equalTo("target"));
+        assertTrue(processor.isIgnoreMissing());
+    }
+
+    public void testCreateWithMissingField() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        String processorTag = randomAsciiOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
+    }
+
+    public void testCreateWithMissingFieldSplit() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        String processorTag = randomAsciiOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[field_split] required property is missing"));
+    }
+
+    public void testCreateWithMissingValueSplit() {
+        KeyValueProcessor.Factory factory = new KeyValueProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "field1");
+        config.put("field_split", "&");
+        String processorTag = randomAsciiOfLength(10);
+        ElasticsearchException exception = expectThrows(ElasticsearchParseException.class,
+            () -> factory.create(null, processorTag, config));
+        assertThat(exception.getMessage(), equalTo("[value_split] required property is missing"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
new file mode 100644
index 00000000000..f5db4be1435
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
+
+public class KeyValueProcessorTests extends ESTestCase {
+
+    public void test() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), fieldName, "&", "=", null, "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("target.second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    public void testRootTarget() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        ingestDocument.setFieldValue("myField", "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), "myField", "&", "=", null, null, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("first", String.class), equalTo("hello"));
+        assertThat(ingestDocument.getFieldValue("second", List.class), equalTo(Arrays.asList("world", "universe")));
+    }
+
+    public void testKeySameAsSourceField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        ingestDocument.setFieldValue("first", "first=hello");
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), "first", "&", "=", null, null, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("first", List.class), equalTo(Arrays.asList("first=hello", "hello")));
+    }
+
+    public void testIncludeKeys() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello&second=world&second=universe");
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), fieldName, "&", "=",
+            Collections.singletonList("first"), "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello"));
+        assertFalse(ingestDocument.hasField("target.second"));
+    }
+
+    public void testMissingField() {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), "unknown", "&", "=", null, "target", false);
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [unknown] not present as part of path [unknown]"));
+    }
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap(fieldName, null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), fieldName, "", "", null, "target", true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), "unknown", "", "", null, "target", true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testFailFieldSplitMatch() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "first=hello|second=world|second=universe");
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), fieldName, "&", "=", null, "target", false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target.first", String.class), equalTo("hello|second=world|second=universe"));
+        assertFalse(ingestDocument.hasField("target.second"));
+    }
+
+    public void testFailValueSplitMatch() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("foo", "bar"));
+        Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), "foo", "&", "=", null, "target", false);
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [foo] does not contain value_split [=]"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index 73b971ee297..60b42979466 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -107,7 +107,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         ScriptService mockedScriptService = mock(ScriptService.class);
         ScriptException thrownException = new ScriptException("compile-time exception", new RuntimeException(),
             Collections.emptyList(), "script", "mockscript");
-        when(mockedScriptService.compile(any(), any(), any())).thenThrow(thrownException);
+        when(mockedScriptService.compile(any(), any())).thenThrow(thrownException);
         factory = new ScriptProcessor.Factory(mockedScriptService);
 
         Map<String, Object> configMap = new HashMap<>();
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
index 87c1f5a8abf..a58c329a7c5 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
@@ -20,12 +20,13 @@
     - match:  { nodes.$master.ingest.processors.8.type: gsub }
     - match:  { nodes.$master.ingest.processors.9.type: join }
     - match:  { nodes.$master.ingest.processors.10.type: json }
-    - match:  { nodes.$master.ingest.processors.11.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.12.type: remove }
-    - match:  { nodes.$master.ingest.processors.13.type: rename }
-    - match:  { nodes.$master.ingest.processors.14.type: script }
-    - match:  { nodes.$master.ingest.processors.15.type: set }
-    - match:  { nodes.$master.ingest.processors.16.type: sort }
-    - match:  { nodes.$master.ingest.processors.17.type: split }
-    - match:  { nodes.$master.ingest.processors.18.type: trim }
-    - match:  { nodes.$master.ingest.processors.19.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.11.type: kv }
+    - match:  { nodes.$master.ingest.processors.12.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.13.type: remove }
+    - match:  { nodes.$master.ingest.processors.14.type: rename }
+    - match:  { nodes.$master.ingest.processors.15.type: script }
+    - match:  { nodes.$master.ingest.processors.16.type: set }
+    - match:  { nodes.$master.ingest.processors.17.type: sort }
+    - match:  { nodes.$master.ingest.processors.18.type: split }
+    - match:  { nodes.$master.ingest.processors.19.type: trim }
+    - match:  { nodes.$master.ingest.processors.20.type: uppercase }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml
new file mode 100644
index 00000000000..a1ecf10278c
--- /dev/null
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yaml
@@ -0,0 +1,43 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "1"
+        ignore: 404
+
+---
+"Test KV Processor":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "kv" : {
+                  "field" : "foo",
+                  "field_split": " ",
+                  "value_split": "="
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "1"
+        body: {
+          foo: "goodbye=everybody hello=world"
+        }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.goodbye: "everybody" }
+  - match: { _source.hello: "world" }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
index b12a65f8fb6..b041e0664bb 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
@@ -50,6 +50,46 @@ teardown:
       catch: missing
       ingest.get_pipeline:
         id: "my_pipeline"
+---
+"Test wildcard pipeline delete":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field" : "field2",
+                  "value": "_value"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
+  - do:
+      ingest.delete_pipeline:
+        id: "my_*"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ingest.get_pipeline:
+        id: "my_pipeline"
+
+  - do:
+      catch: missing
+      ingest.delete_pipeline:
+        id: "my_*"
+  - match: { "error.type": "resource_not_found_exception" }
+  - match: { "error.reason": "pipeline [my_*] is missing" }
 
 ---
 "Test Get All Pipelines":
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-ec38570.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index b68a4d5cbd7..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-031d34e0a604a7cbb5c8ba816d49d9f622adaa3f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.4.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.4.0.jar.sha1
new file mode 100644
index 00000000000..b44295b0ad9
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.4.0.jar.sha1
@@ -0,0 +1 @@
+09a755871c89fec98d3ca8372f283c1639dcadaf
\ No newline at end of file
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
index 85a82ac79da..f5d95f6c88a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
@@ -19,25 +19,22 @@
 
 package org.elasticsearch.script.expression;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import org.apache.lucene.expressions.Expression;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.GeneralScriptException;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * A bridge to evaluate an {@link Expression} against a map of variables in the context
  * of an {@link ExecutableScript}.
  */
 public class ExpressionExecutableScript implements ExecutableScript {
-
-    private final int NO_DOCUMENT = -1;
-
     public final CompiledScript compiledScript;
-    public final Map<String, ReplaceableConstFunctionValues> functionValuesMap;
-    public final ReplaceableConstFunctionValues[] functionValuesArray;
+    public final Map<String, ReplaceableConstDoubleValues> functionValuesMap;
+    public final ReplaceableConstDoubleValues[] functionValuesArray;
 
     public ExpressionExecutableScript(CompiledScript compiledScript, Map<String, Object> vars) {
         this.compiledScript = compiledScript;
@@ -51,12 +48,12 @@ public class ExpressionExecutableScript implements ExecutableScript {
                     " [" + vars.size() + "].");
         }
 
-        functionValuesArray = new ReplaceableConstFunctionValues[functionValuesLength];
+        functionValuesArray = new ReplaceableConstDoubleValues[functionValuesLength];
         functionValuesMap = new HashMap<>();
 
         for (int functionValuesIndex = 0; functionValuesIndex < functionValuesLength; ++functionValuesIndex) {
             String variableName = expression.variables[functionValuesIndex];
-            functionValuesArray[functionValuesIndex] = new ReplaceableConstFunctionValues();
+            functionValuesArray[functionValuesIndex] = new ReplaceableConstDoubleValues();
             functionValuesMap.put(variableName, functionValuesArray[functionValuesIndex]);
         }
 
@@ -85,7 +82,7 @@ public class ExpressionExecutableScript implements ExecutableScript {
     @Override
     public Object run() {
         try {
-            return ((Expression) compiledScript.compiled()).evaluate(NO_DOCUMENT, functionValuesArray);
+            return ((Expression) compiledScript.compiled()).evaluate(functionValuesArray);
         } catch (Exception exception) {
             throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
         }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index c78825758e2..46461ee5acb 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -78,9 +78,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
         // classloader created here
         final SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
         return AccessController.doPrivileged(new PrivilegedAction<Expression>() {
             @Override
             public Expression run() {
@@ -117,14 +115,15 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
         // instead of complicating SimpleBindings (which should stay simple)
         SimpleBindings bindings = new SimpleBindings();
-        ReplaceableConstValueSource specialValue = null;
-
+        ReplaceableConstDoubleValueSource specialValue = null;
+        boolean needsScores = false;
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
                     bindings.add(new SortField("_score", SortField.Type.SCORE));
+                    needsScores = true;
                 } else if (variable.equals("_value")) {
-                    specialValue = new ReplaceableConstValueSource();
+                    specialValue = new ReplaceableConstDoubleValueSource();
                     bindings.add("_value", specialValue);
                     // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
                     // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
@@ -136,7 +135,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field?
                     Object value = vars.get(variable);
                     if (value instanceof Number) {
-                        bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()));
+                        bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()).asDoubleValuesSource());
                     } else {
                         throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
                     }
@@ -225,16 +224,14 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     } else {
                         throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                     }
-
-                    bindings.add(variable, valueSource);
+                    needsScores |= valueSource.getSortField(false).needsScores();
+                    bindings.add(variable, valueSource.asDoubleValuesSource());
                 }
             } catch (Exception e) {
                 // we defer "binding" of variables until here: give context for that variable
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
             }
         }
-
-        final boolean needsScores = expr.getSortField(bindings, false).needsScores();
         return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index af89450613c..11a17d38c59 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -19,23 +19,21 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-
 import org.apache.lucene.expressions.Bindings;
 import org.apache.lucene.expressions.Expression;
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
 import org.apache.lucene.search.Scorer;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.SearchScript;
 
+import java.io.IOException;
+import java.util.Map;
+
 /**
  * A bridge to evaluate an {@link Expression} against {@link Bindings} in the context
  * of a {@link SearchScript}.
@@ -44,16 +42,16 @@ class ExpressionSearchScript implements SearchScript {
 
     final CompiledScript compiledScript;
     final SimpleBindings bindings;
-    final ValueSource source;
-    final ReplaceableConstValueSource specialValue; // _value
+    final DoubleValuesSource source;
+    final ReplaceableConstDoubleValueSource specialValue; // _value
     final boolean needsScores;
     Scorer scorer;
     int docid;
 
-    ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstValueSource v, boolean needsScores) {
+    ExpressionSearchScript(CompiledScript c, SimpleBindings b, ReplaceableConstDoubleValueSource v, boolean needsScores) {
         compiledScript = c;
         bindings = b;
-        source = ((Expression)compiledScript.compiled()).getValueSource(bindings);
+        source = ((Expression)compiledScript.compiled()).getDoubleValuesSource(bindings);
         specialValue = v;
         this.needsScores = needsScores;
     }
@@ -63,15 +61,25 @@ class ExpressionSearchScript implements SearchScript {
         return needsScores;
     }
 
+
     @Override
     public LeafSearchScript getLeafSearchScript(final LeafReaderContext leaf) throws IOException {
         return new LeafSearchScript() {
+            // Fake the scorer until setScorer is called.
+            DoubleValues values = source.getValues(leaf, new DoubleValues() {
+                @Override
+                public double doubleValue() throws IOException {
+                    return Double.NaN;
+                }
 
-            FunctionValues values = source.getValues(Collections.singletonMap("scorer", Lucene.illegalScorer("Scores are not available in the current context")), leaf);
-
+                @Override
+                public boolean advanceExact(int doc) throws IOException {
+                    return true;
+                }
+            });
             double evaluate() {
                 try {
-                    return values.doubleVal(docid);
+                    return values.doubleValue();
                 } catch (Exception exception) {
                     throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
                 }
@@ -89,6 +97,11 @@ class ExpressionSearchScript implements SearchScript {
             @Override
             public void setDocument(int d) {
                 docid = d;
+                try {
+                    values.advanceExact(d);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Can't advance to doc using " + compiledScript, e);
+                }
             }
 
             @Override
@@ -96,7 +109,17 @@ class ExpressionSearchScript implements SearchScript {
                 scorer = s;
                 try {
                     // We have a new binding for the scorer so we need to reset the values
-                    values = source.getValues(Collections.singletonMap("scorer", scorer), leaf);
+                    values = source.getValues(leaf, new DoubleValues() {
+                        @Override
+                        public double doubleValue() throws IOException {
+                            return scorer.score();
+                        }
+
+                        @Override
+                        public boolean advanceExact(int doc) throws IOException {
+                            return true;
+                        }
+                    });
                 } catch (IOException e) {
                     throw new IllegalStateException("Can't get values using " + compiledScript, e);
                 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
similarity index 64%
rename from modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
index ae0f862ca70..f8daa6ddf88 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java
@@ -19,29 +19,32 @@
 
 package org.elasticsearch.script.expression;
 
-import java.io.IOException;
-import java.util.Map;
-
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.DoubleValues;
+import org.apache.lucene.search.DoubleValuesSource;
+
+import java.io.IOException;
 
 /**
- * A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double.
+ * A {@link DoubleValuesSource} which has a stub {@link DoubleValues} that holds a dynamically replaceable constant double.
  */
-final class ReplaceableConstValueSource extends ValueSource {
-    final ReplaceableConstFunctionValues fv;
+final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
+    final ReplaceableConstDoubleValues fv;
 
-    ReplaceableConstValueSource() {
-        fv = new ReplaceableConstFunctionValues();
+    ReplaceableConstDoubleValueSource() {
+        fv = new ReplaceableConstDoubleValues();
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
-    public FunctionValues getValues(Map map, LeafReaderContext atomicReaderContext) throws IOException {
+    public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
         return fv;
     }
 
+    @Override
+    public boolean needsScores() {
+        return false;
+    }
+
     @Override
     public boolean equals(Object o) {
         return o == this;
@@ -52,11 +55,6 @@ final class ReplaceableConstValueSource extends ValueSource {
         return System.identityHashCode(this);
     }
 
-    @Override
-    public String description() {
-        return "replaceableConstDouble";
-    }
-
     public void setValue(double v) {
         fv.setValue(v);
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
similarity index 77%
rename from modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
rename to modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
index eab9131bf10..557f9ad812e 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValues.java
@@ -19,13 +19,15 @@
 
 package org.elasticsearch.script.expression;
 
-import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.search.DoubleValues;
+
+import java.io.IOException;
 
 /**
  * A support class for an executable expression script that allows the double returned
- * by a {@link FunctionValues} to be modified.
+ * by a {@link DoubleValues} to be modified.
  */
-final class ReplaceableConstFunctionValues extends FunctionValues {
+final class ReplaceableConstDoubleValues extends DoubleValues {
     private double value = 0;
 
     void setValue(double value) {
@@ -33,12 +35,12 @@ final class ReplaceableConstFunctionValues extends FunctionValues {
     }
 
     @Override
-    public double doubleVal(int doc) {
+    public double doubleValue() throws IOException {
         return value;
     }
 
     @Override
-    public String toString(int i) {
-        return "ReplaceableConstFunctionValues: " + value;
+    public boolean advanceExact(int doc) throws IOException {
+        return true;
     }
 }
diff --git a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
index c11af51e464..4835ab0530e 100644
--- a/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/lang-expression/src/main/plugin-metadata/plugin-security.policy
@@ -24,7 +24,7 @@ grant {
   // expression runtime
   permission org.elasticsearch.script.ClassPermission "java.lang.String";
   permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression";
-  permission org.elasticsearch.script.ClassPermission "org.apache.lucene.queries.function.FunctionValues";
+  permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues";
   // available functions
   permission org.elasticsearch.script.ClassPermission "java.lang.Math";
   permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil";
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
index 9a30def83e1..a20050b80ad 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class LangExpressionClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
similarity index 91%
rename from modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java
rename to modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
index 71c87937618..697dbf62575 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/IndexedExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.script.expression;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -35,7 +36,7 @@ import java.util.Collections;
 import static org.hamcrest.Matchers.containsString;
 
 //TODO: please convert to unit tests!
-public class IndexedExpressionTests extends ESIntegTestCase {
+public class StoredExpressionTests extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
@@ -51,18 +52,18 @@ public class IndexedExpressionTests extends ESIntegTestCase {
 
     public void testAllOpsDisabledIndexedScripts() throws IOException {
         client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(ExpressionScriptEngineService.NAME)
+                .setLang(ExpressionScriptEngineService.NAME)
                 .setId("script1")
-                .setSource(new BytesArray("{\"script\":\"2\"}"))
+                .setContent(new BytesArray("{\"script\":\"2\"}"), XContentType.JSON)
                 .get();
-        client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}").get();
+        client().prepareIndex("test", "scriptTest", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON).get();
         try {
             client().prepareUpdate("test", "scriptTest", "1")
                     .setScript(new Script(ScriptType.STORED, ExpressionScriptEngineService.NAME, "script1", Collections.emptyMap())).get();
             fail("update script should have been rejected");
         } catch(Exception e) {
             assertThat(e.getMessage(), containsString("failed to execute script"));
-            assertThat(e.getCause().getMessage(), containsString("scripts of type [stored], operation [update] and lang [expression] are disabled"));
+            assertThat(e.getCause().getMessage(), containsString("scripts of type [stored], operation [update] and lang [expression] are not supported"));
         }
         try {
             client().prepareSearch()
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
index 662cd86f799..43232cc8f65 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
@@ -106,7 +106,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
     class CustomMustacheVisitor extends DefaultMustacheVisitor {
 
-        public CustomMustacheVisitor(DefaultMustacheFactory df) {
+        CustomMustacheVisitor(DefaultMustacheFactory df) {
             super(df);
         }
 
@@ -133,7 +133,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
         private final String code;
 
-        public CustomCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String code) {
+        CustomCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String code) {
             super(tc, df, mustache, extractVariableName(code, mustache, tc));
             this.code = Objects.requireNonNull(code);
         }
@@ -188,7 +188,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
         private static final String CODE = "toJson";
 
-        public ToJsonCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+        ToJsonCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache, CODE);
             if (CODE.equalsIgnoreCase(variable) == false) {
                 throw new MustacheException("Mismatch function code [" + CODE + "] cannot be applied to [" + variable + "]");
@@ -239,12 +239,12 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
         private final String delimiter;
 
-        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String delimiter) {
+        JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String delimiter) {
             super(tc, df, mustache, CODE);
             this.delimiter = delimiter;
         }
 
-        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache) {
+        JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache) {
             this(tc, df, mustache, DEFAULT_DELIMITER);
         }
 
@@ -273,7 +273,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
         private static final Pattern PATTERN = Pattern.compile("^(?:" + CODE + " delimiter='(.*)')$");
 
-        public CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+        CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache, extractDelimiter(variable));
         }
 
@@ -299,7 +299,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
         private static final String CODE = "url";
         private final Encoder encoder;
 
-        public UrlEncoderCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+        UrlEncoderCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
             super(tc, df, mustache.getCodes(), variable);
             this.encoder = new UrlEncoder();
         }
@@ -336,7 +336,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
          * @param s      The string to encode
          * @param writer The {@link Writer} to which the encoded string will be written to
          */
-        void encode(final String s, final Writer writer) throws IOException;
+        void encode(String s, Writer writer) throws IOException;
     }
 
     /**
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
index dd3055ba8e8..eef9d7af8dd 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
@@ -54,7 +54,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler {
         private final Object array;
         private final int length;
 
-        public ArrayMap(Object array) {
+        ArrayMap(Object array) {
             this.array = array;
             this.length = Array.getLength(array);
         }
@@ -113,7 +113,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler {
 
         private final Collection<Object> col;
 
-        public CollectionMap(Collection<Object> col) {
+        CollectionMap(Collection<Object> col) {
             this.col = col;
         }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
index 6201e9305ef..37d72ac4c0d 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateResponse.java
@@ -22,18 +22,19 @@ package org.elasticsearch.script.mustache;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-public class MultiSearchTemplateResponse extends ActionResponse implements Iterable<MultiSearchTemplateResponse.Item>, ToXContent {
+public class MultiSearchTemplateResponse extends ActionResponse implements Iterable<MultiSearchTemplateResponse.Item>, ToXContentObject {
 
     /**
      * A search template response item, holding the actual search template response, or an error message if it failed.
@@ -146,36 +147,28 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject();
         builder.startArray(Fields.RESPONSES);
         for (Item item : items) {
-            builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
+                builder.startObject();
+                ElasticsearchException.generateFailureXContent(builder, params, item.getFailure(), true);
+                builder.endObject();
             } else {
                 item.getResponse().toXContent(builder, params);
             }
-            builder.endObject();
         }
         builder.endArray();
+        builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final String RESPONSES = "responses";
-        static final String ERROR = "error";
-        static final String ROOT_CAUSE = "root_cause";
     }
 
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index a5a0ded3bee..36f598a8214 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -21,16 +21,23 @@ package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptEngineService;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -53,8 +60,15 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestSearchTemplateAction.class, RestMultiSearchTemplateAction.class, RestGetSearchTemplateAction.class,
-                RestPutSearchTemplateAction.class, RestDeleteSearchTemplateAction.class, RestRenderSearchTemplateAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestSearchTemplateAction(settings, restController),
+                new RestMultiSearchTemplateAction(settings, restController),
+                new RestGetSearchTemplateAction(settings, restController),
+                new RestPutSearchTemplateAction(settings, restController),
+                new RestDeleteSearchTemplateAction(settings, restController),
+                new RestRenderSearchTemplateAction(settings, restController));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 08c0e1643bc..c7964a716f8 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -127,9 +127,6 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
         // Nothing to do here
     }
 
-    // permission checked before doing crazy reflection
-    static final SpecialPermission SPECIAL_PERMISSION = new SpecialPermission();
-
     /**
      * Used at query execution time by script service in order to execute a query template.
      * */
@@ -143,7 +140,7 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
          * @param template the compiled template object wrapper
          * @param vars the parameters to fill above object with
          **/
-        public MustacheExecutableScript(CompiledScript template, Map<String, Object> vars) {
+        MustacheExecutableScript(CompiledScript template, Map<String, Object> vars) {
             this.template = template;
             this.vars = vars == null ? Collections.<String, Object>emptyMap() : vars;
         }
@@ -158,10 +155,7 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
             final BytesStreamOutput result = new BytesStreamOutput();
             try (UTF8StreamWriter writer = utf8StreamWriter().setOutput(result)) {
                 // crazy reflection here
-                SecurityManager sm = System.getSecurityManager();
-                if (sm != null) {
-                    sm.checkPermission(SPECIAL_PERMISSION);
-                }
+                SpecialPermission.check();
                 AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
                     ((Mustache) template.compiled()).execute(writer, vars);
                     return null;
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
index a6ea92e2818..61a394462f9 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestDeleteSearchTemplateAction.java
@@ -18,24 +18,32 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
+import org.elasticsearch.rest.action.AcknowledgedRestListener;
+import org.elasticsearch.script.Script;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 
-public class RestDeleteSearchTemplateAction extends RestDeleteStoredScriptAction {
+public class RestDeleteSearchTemplateAction extends BaseRestHandler {
 
-    @Inject
     public RestDeleteSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
+        super(settings);
+
         controller.registerHandler(DELETE, "/_search/template/{id}", this);
     }
 
     @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+
+        DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG);
+        return channel -> client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
index 16ab14bc6c8..7d1ed4b57a4 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestGetSearchTemplateAction.java
@@ -18,31 +18,64 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestGetStoredScriptAction;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.StoredScriptSource;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-public class RestGetSearchTemplateAction extends RestGetStoredScriptAction {
+public class RestGetSearchTemplateAction extends BaseRestHandler {
 
-    private static final String TEMPLATE = "template";
+    public static final ParseField _ID_PARSE_FIELD = new ParseField("_id");
+
+    public static final ParseField FOUND_PARSE_FIELD = new ParseField("found");
 
-    @Inject
     public RestGetSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
+        super(settings);
+
         controller.registerHandler(GET, "/_search/template/{id}", this);
     }
 
     @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
-    }
+    public RestChannelConsumer prepareRequest(final RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
 
-    @Override
-    protected String getScriptFieldName() {
-        return TEMPLATE;
+        GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG);
+
+        return channel -> client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(GetStoredScriptResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                builder.field(_ID_PARSE_FIELD.getPreferredName(), id);
+
+                builder.field(StoredScriptSource.LANG_PARSE_FIELD.getPreferredName(), Script.DEFAULT_TEMPLATE_LANG);
+
+                StoredScriptSource source = response.getSource();
+                boolean found = source != null;
+                builder.field(FOUND_PARSE_FIELD.getPreferredName(), found);
+
+                if (found) {
+                    builder.field(StoredScriptSource.TEMPLATE_PARSE_FIELD.getPreferredName(), source.getCode());
+                }
+
+                builder.endObject();
+
+                return new BytesRestResponse(found ? RestStatus.OK : RestStatus.NOT_FOUND, builder);
+            }
+        });
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
index 7eb134f74ff..6b0497e53fc 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -22,12 +22,10 @@ package org.elasticsearch.script.mustache;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
 import org.elasticsearch.rest.action.RestToXContentListener;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
 
@@ -40,7 +38,6 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
 
-    @Inject
     public RestMultiSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
         this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
@@ -67,7 +64,6 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest}
      */
     public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
-
         MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
         RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex,
                 (searchRequest, bytes) -> {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
index 718afb608e6..83925f0ec03 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestPutSearchTemplateAction.java
@@ -18,26 +18,36 @@
  */
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
+import org.elasticsearch.rest.action.AcknowledgedRestListener;
+import org.elasticsearch.script.Script;
+
+import java.io.IOException;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
-public class RestPutSearchTemplateAction extends RestPutStoredScriptAction {
+public class RestPutSearchTemplateAction extends BaseRestHandler {
 
-    @Inject
     public RestPutSearchTemplateAction(Settings settings, RestController controller) {
-        super(settings, controller, false);
+        super(settings);
+
         controller.registerHandler(POST, "/_search/template/{id}", this);
         controller.registerHandler(PUT, "/_search/template/{id}", this);
     }
 
     @Override
-    protected String getScriptLang(RestRequest request) {
-        return "mustache";
+    public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String id = request.param("id");
+        BytesReference content = request.content();
+
+        PutStoredScriptRequest put = new PutStoredScriptRequest(id, Script.DEFAULT_TEMPLATE_LANG, content, request.getXContentType());
+        return channel -> client.admin().cluster().putStoredScript(put, new AcknowledgedRestListener<>(channel));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
index 482b2a13c89..6647bd4bd45 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestRenderSearchTemplateAction.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,8 +34,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestRenderSearchTemplateAction extends BaseRestHandler {
-
-    @Inject
     public RestRenderSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(GET, "/_render/template", this);
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
index a6d705386fc..bb263dd5a26 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -23,10 +23,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -38,7 +35,6 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.search.SearchRequestParsers;
 
 import java.io.IOException;
 
@@ -47,7 +43,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSearchTemplateAction extends BaseRestHandler {
 
-    private static final ObjectParser<SearchTemplateRequest, ParseFieldMatcherSupplier> PARSER;
+    private static final ObjectParser<SearchTemplateRequest, Void> PARSER;
     static {
         PARSER = new ObjectParser<>("search_template");
         PARSER.declareField((parser, request, s) ->
@@ -77,12 +73,8 @@ public class RestSearchTemplateAction extends BaseRestHandler {
         }, new ParseField("inline", "template"), ObjectParser.ValueType.OBJECT_OR_STRING);
     }
 
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public RestSearchTemplateAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) {
+    public RestSearchTemplateAction(Settings settings, RestController controller) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
 
         controller.registerHandler(GET, "/_search/template", this);
         controller.registerHandler(POST, "/_search/template", this);
@@ -100,12 +92,12 @@ public class RestSearchTemplateAction extends BaseRestHandler {
 
         // Creates the search request with all required params
         SearchRequest searchRequest = new SearchRequest();
-        RestSearchAction.parseSearchRequest(searchRequest, request, searchRequestParsers, parseFieldMatcher, null);
+        RestSearchAction.parseSearchRequest(searchRequest, request, null);
 
         // Creates the search template request
         SearchTemplateRequest searchTemplateRequest;
         try (XContentParser parser = request.contentOrSourceParamParser()) {
-            searchTemplateRequest = PARSER.parse(parser, new SearchTemplateRequest(), () -> ParseFieldMatcher.EMPTY);
+            searchTemplateRequest = PARSER.parse(parser, new SearchTemplateRequest(), null);
         }
         searchTemplateRequest.setRequest(searchRequest);
 
@@ -113,6 +105,6 @@ public class RestSearchTemplateAction extends BaseRestHandler {
     }
 
     public static SearchTemplateRequest parse(XContentParser parser) throws IOException {
-        return PARSER.parse(parser, new SearchTemplateRequest(), () -> ParseFieldMatcher.EMPTY);
+        return PARSER.parse(parser, new SearchTemplateRequest(), null);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
index f4647b713de..6158e80c241 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java
@@ -24,13 +24,13 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.StatusToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
-public class SearchTemplateResponse  extends ActionResponse implements StatusToXContent {
+public class SearchTemplateResponse  extends ActionResponse implements StatusToXContentObject {
 
     /** Contains the source of the rendered template **/
     private BytesReference source;
@@ -80,7 +80,9 @@ public class SearchTemplateResponse  extends ActionResponse implements StatusToX
         if (hasResponse()) {
             response.toXContent(builder, params);
         } else {
+            builder.startObject();
             builder.rawField("template_output", source);
+            builder.endObject();
         }
         return builder;
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
index 59e2e8eac52..66a5c09977b 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TemplateQueryBuilder.java
@@ -60,8 +60,9 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     }
 
     public TemplateQueryBuilder(String template, ScriptType scriptType, Map<String, Object> params, XContentType ct) {
-        this(new Script(scriptType, "mustache", template,
-            ct == null ? Collections.emptyMap() : Collections.singletonMap(Script.CONTENT_TYPE_OPTION, ct.mediaType()), params));
+        this(new Script(scriptType, "mustache", template, scriptType == ScriptType.INLINE ?
+            (ct == null ? Collections.emptyMap() : Collections.singletonMap(Script.CONTENT_TYPE_OPTION, ct.mediaType()))
+            : null, params));
     }
 
     TemplateQueryBuilder(Script template) {
@@ -118,7 +119,8 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         BytesReference querySource = queryRewriteContext.getTemplateBytes(template);
-        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
+        try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(queryRewriteContext.getXContentRegistry(),
+                querySource)) {
             final QueryParseContext queryParseContext = queryRewriteContext.newParseContext(qSourceParser);
             final QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder();
             if (boost() != DEFAULT_BOOST || queryName() != null) {
@@ -136,7 +138,14 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
      */
     public static TemplateQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
-        Script template = Script.parse(parser, parseContext.getParseFieldMatcher(), "mustache");
+        Script template = Script.parse(parser, Script.DEFAULT_TEMPLATE_LANG);
+
+        // for deprecation of stored script namespaces the default lang is ignored,
+        // so the template lang must be set for a stored script
+        if (template.getType() == ScriptType.STORED) {
+            template = new Script(ScriptType.STORED, Script.DEFAULT_TEMPLATE_LANG, template.getIdOrCode(), template.getParams());
+        }
+
         return new TemplateQueryBuilder(template);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
index e366b34e3d0..7d83fcaf581 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java
@@ -29,13 +29,13 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -50,17 +50,18 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
 
     private final ScriptService scriptService;
     private final TransportSearchAction searchAction;
-    private final SearchRequestParsers searchRequestParsers;
+    private final NamedXContentRegistry xContentRegistry;
 
     @Inject
     public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                          ActionFilters actionFilters, IndexNameExpressionResolver resolver,
                                          ScriptService scriptService,
-                                         TransportSearchAction searchAction, SearchRequestParsers searchRequestParsers) {
+                                         TransportSearchAction searchAction,
+                                         NamedXContentRegistry xContentRegistry) {
         super(settings, SearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver, SearchTemplateRequest::new);
         this.scriptService = scriptService;
         this.searchAction = searchAction;
-        this.searchRequestParsers = searchRequestParsers;
+        this.xContentRegistry = xContentRegistry;
     }
 
     @Override
@@ -82,10 +83,9 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
             // Executes the search
             SearchRequest searchRequest = request.getRequest();
 
-            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
+            try (XContentParser parser = XContentFactory.xContent(source).createParser(xContentRegistry, source)) {
                 SearchSourceBuilder builder = SearchSourceBuilder.searchSource();
-                builder.parseXContent(new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher),
-                    searchRequestParsers.aggParsers, searchRequestParsers.suggesters, searchRequestParsers.searchExtParsers);
+                builder.parseXContent(new QueryParseContext(parser));
                 builder.explain(request.isExplain());
                 builder.profile(request.isProfile());
                 searchRequest.source(builder);
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
index 377fa870c41..72eb9f2ad79 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class LangMustacheClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
index bec7e24d3a4..7dad299a40f 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.test.ESTestCase;
@@ -34,7 +35,8 @@ public class MultiSearchTemplateRequestTests extends ESTestCase {
 
     public void testParseRequest() throws Exception {
         byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/script/mustache/simple-msearch-template.json");
-        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
+        RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
+            .withContent(new BytesArray(data), XContentType.JSON).build();
 
         MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index 6a936260ca5..e6cca8fc667 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -19,8 +19,6 @@
 package org.elasticsearch.script.mustache;
 
 import com.github.mustachejava.MustacheFactory;
-
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -87,7 +85,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
                 + "\"params\":{\"template\":\"all\"}"
                 + "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
-        Script script = Script.parse(parser, new ParseFieldMatcher(false));
+        Script script = Script.parse(parser);
         CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache",
                 qe.compile(null, script.getIdOrCode(), Collections.emptyMap()));
         ExecutableScript executableScript = qe.executable(compiledScript, script.getParams());
@@ -103,7 +101,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
                 + "  }"
                 + "}";
         XContentParser parser = createParser(JsonXContent.jsonXContent, templateString);
-        Script script = Script.parse(parser, new ParseFieldMatcher(false));
+        Script script = Script.parse(parser);
         CompiledScript compiledScript = new CompiledScript(ScriptType.INLINE, null, "mustache",
                 qe.compile(null, script.getIdOrCode(), Collections.emptyMap()));
         ExecutableScript executableScript = qe.executable(compiledScript, script.getParams());
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index ba19febfd21..2b11df19392 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -279,7 +279,8 @@ public class MustacheTests extends ESTestCase {
                         .endArray()
                     .endObject();
 
-        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+        Map<String, Object> ctx =
+            Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2());
 
         assertScript("{{#ctx.bulks}}{{#toJson}}.{{/toJson}}{{/ctx.bulks}}", ctx,
                 equalTo("{\"index\":\"index-1\",\"id\":1,\"type\":\"type-1\"}{\"index\":\"index-2\",\"id\":2,\"type\":\"type-2\"}"));
@@ -320,7 +321,8 @@ public class MustacheTests extends ESTestCase {
                                                     .endArray()
                                                 .endObject();
 
-        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+        Map<String, Object> ctx =
+            Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2());
 
         assertScript("{{#join}}ctx.people.0.emails{{/join}}", ctx,
                 equalTo("john@smith.com,john.smith@email.com,jsmith@email.com"));
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
index 5bed13c78dc..19a70238336 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/SearchTemplateIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptType;
@@ -150,39 +151,39 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
 
     public void testIndexedTemplateClient() throws Exception {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("testTemplate")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}")));
+                        "}"), XContentType.JSON));
 
 
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
-                .setId("testTemplate").setSource(new BytesArray("{" +
+                .setLang(MustacheScriptEngineService.NAME)
+                .setId("testTemplate").setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}")));
+                        "}"), XContentType.JSON));
 
         GetStoredScriptResponse getResponse = client().admin().cluster()
                 .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
-        assertNotNull(getResponse.getStoredScript());
+        assertNotNull(getResponse.getSource());
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -200,20 +201,20 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
 
         getResponse = client().admin().cluster()
                 .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
-        assertNull(getResponse.getStoredScript());
+        assertNull(getResponse.getSource());
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest("test").types("type"))
                 .setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get());
-        assertThat(e.getMessage(), containsString("Illegal index script format [/template_index/mustache/1000] should be /lang/id"));
+        assertThat(e.getMessage(), containsString("illegal stored script format [/template_index/mustache/1000] use only <id>"));
     }
 
     public void testIndexedTemplate() throws Exception {
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("1a")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
@@ -221,37 +222,37 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
                         "       }" +
                         "}" +
                         "}"
-                ))
+                ), XContentType.JSON)
         );
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("2")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "                \"query\":{" +
                         "                   \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
                         "}" +
-                        "}"))
+                        "}"), XContentType.JSON)
         );
         assertAcked(client().admin().cluster().preparePutStoredScript()
-                .setScriptLang(MustacheScriptEngineService.NAME)
+                .setLang(MustacheScriptEngineService.NAME)
                 .setId("3")
-                .setSource(new BytesArray("{" +
+                .setContent(new BytesArray("{" +
                         "\"template\":{" +
                         "             \"match\":{" +
                         "                    \"theField\" : \"{{fieldParam}}\"}" +
                         "       }" +
-                        "}"))
+                        "}"), XContentType.JSON)
         );
 
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -260,7 +261,7 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest().indices("test").types("type"))
-                .setScript("/mustache/1a")
+                .setScript("1a")
                 .setScriptType(ScriptType.STORED)
                 .setScriptParams(templateParams)
                 .get();
@@ -286,6 +287,8 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
                 .setScript("/mustache/2").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 1);
+        assertWarnings("use of </lang/id> [/mustache/2] for looking up" +
+            " stored scripts/templates has been deprecated, use only <id> [2] instead");
 
         Map<String, Object> vars = new HashMap<>();
         vars.put("fieldParam", "bar");
@@ -294,6 +297,7 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
         SearchResponse sr = client().prepareSearch().setQuery(builder)
                 .execute().actionGet();
         assertHitCount(sr, 1);
+        assertWarnings("[template] query is deprecated, use search template api instead");
     }
 
     // Relates to #10397
@@ -309,14 +313,14 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
         int iterations = randomIntBetween(2, 11);
         for (int i = 1; i < iterations; i++) {
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                    .setLang(MustacheScriptEngineService.NAME)
                     .setId("git01")
-                    .setSource(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
-                            "\"type\": \"ooophrase_prefix\"}}}}}")));
+                    .setContent(new BytesArray("{\"template\":{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                            "\"type\": \"ooophrase_prefix\"}}}}}"), XContentType.JSON));
 
             GetStoredScriptResponse getResponse = client().admin().cluster()
                     .prepareGetStoredScript(MustacheScriptEngineService.NAME, "git01").get();
-            assertNotNull(getResponse.getStoredScript());
+            assertNotNull(getResponse.getSource());
 
             Map<String, Object> templateParams = new HashMap<>();
             templateParams.put("P_Keyword1", "dev");
@@ -326,18 +330,20 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
                     .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get());
             assertThat(e.getMessage(), containsString("[match] query does not support type ooophrase_prefix"));
+            assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]");
 
             assertAcked(client().admin().cluster().preparePutStoredScript()
-                    .setScriptLang(MustacheScriptEngineService.NAME)
+                    .setLang(MustacheScriptEngineService.NAME)
                     .setId("git01")
-                    .setSource(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
-                            "\"type\": \"phrase_prefix\"}}}}")));
+                    .setContent(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
+                            "\"type\": \"phrase_prefix\"}}}}"), XContentType.JSON));
 
             SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                     .setRequest(new SearchRequest("testindex").types("test"))
                     .setScript("git01").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                     .get();
             assertHitCount(searchResponse.getResponse(), 1);
+            assertWarnings("Deprecated field [type] used, replaced by [match_phrase and match_phrase_prefix query]");
         }
     }
 
@@ -345,16 +351,16 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
         String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}";
         assertAcked(
                 client().admin().cluster().preparePutStoredScript()
-                        .setScriptLang(MustacheScriptEngineService.NAME)
+                        .setLang(MustacheScriptEngineService.NAME)
                         .setId("4")
-                        .setSource(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes())
+                        .setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON)
         );
         BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}"));
-        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}"));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
+        bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
         bulkRequestBuilder.get();
         client().admin().indices().prepareRefresh().get();
 
@@ -364,7 +370,7 @@ public class SearchTemplateIT extends ESSingleNodeTestCase {
 
         SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
                 .setRequest(new SearchRequest("test").types("type"))
-                .setScript("/mustache/4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams)
+                .setScript("4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams)
                 .get();
         assertHitCount(searchResponse.getResponse(), 5);
     }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
index 34e06410a78..3b70c5df626 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/TemplateQueryBuilderTests.java
@@ -59,11 +59,12 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
     private QueryBuilder templateBase;
 
     /**
-     * All tests create deprecation warnings when an new {@link TemplateQueryBuilder} is created.
-     * Instead of having to check them once in every single test, this is done here after each test is run
+     * All tests in this class cause deprecation warnings when a new {@link TemplateQueryBuilder} is created.
+     * Instead of having to check them in every single test, we do it after each test is run
      */
-    @After void checkWarningHeaders() throws IOException {
-        checkWarningHeaders("[template] query is deprecated, use search template api instead");
+    @After
+    public void checkWarning() {
+        assertWarnings("[template] query is deprecated, use search template api instead");
     }
 
     @Override
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
index 9533cde81ac..644319d50ec 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yaml
@@ -54,7 +54,7 @@
         body: { "template": { "query": { "match{{}}_all": {}}, "size": "{{my_size}}" } }
 
   - do:
-      catch: /Unable\sto\sparse.*/
+      catch: /failed\sto\sparse.*/
       put_template:
         id: "1"
         body: { "template": { "query": { "match{{}}_all": {}}, "size": "{{my_size}}" } }
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
index 9a9a1b0c017..f6fc458a08e 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yaml
@@ -150,7 +150,7 @@
   - match: { hits.total: 1 }
 
   - do:
-      catch: /Unable.to.find.on.disk.file.script.\[simple1\].using.lang.\[mustache\]/
+      catch: /unable.to.find.file.script.\[simple1\].using.lang.\[mustache\]/
       search_template:
         body: { "file" : "simple1"}
 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
index cfa97b8bc9f..a21184650f5 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml
@@ -52,7 +52,7 @@
       warnings:
           - '[template] query is deprecated, use search template api instead'
       search:
-        body: { "query": { "template": { "stored": "/mustache/1", "params": { "my_value": "value1" } } } }
+        body: { "query": { "template": { "stored": "1", "params": { "my_value": "value1" } } } }
 
   - match: { hits.total: 1 }
 
diff --git a/modules/lang-painless/ant.xml b/modules/lang-painless/ant.xml
index 9d3cf9ece91..90e66b7b1a9 100644
--- a/modules/lang-painless/ant.xml
+++ b/modules/lang-painless/ant.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <project name="ant-stuff">
 
-<!-- 
+<!--
  grammar regeneration logic
  we do this with ant for several reasons:
  * remove generated tabs for forbidden-apis
@@ -9,7 +9,7 @@
  * fix CRLF line endings for windows consistency
  * ability to make classes package-private
  * keeping in source code control is easier on IDEs
- * regeneration should be rare, no reason to be religious about generated files 
+ * regeneration should be rare, no reason to be religious about generated files
  * all logic already written and battle tested in lucene build
 -->
   <target name="regenerate" description="Regenerate antlr lexer and parser" depends="run-antlr"/>
@@ -136,6 +136,10 @@
       <replaceregexp match="public ((interface|class) \Q@{grammar}\E\w+)" replace="\1" encoding="UTF-8">
         <fileset refid="grammar.fileset"/>
       </replaceregexp>
+      <!-- make the lexer abstract -->
+      <replaceregexp match="(class \Q@{grammar}\ELexer)" replace="abstract \1" encoding="UTF-8">
+        <fileset refid="grammar.fileset"/>
+      </replaceregexp>
       <!-- nuke timestamps/filenames in generated files -->
       <replaceregexp match="\Q// Generated from \E.*" replace="\/\/ ANTLR GENERATED CODE: DO NOT EDIT" encoding="UTF-8">
         <fileset refid="grammar.fileset"/>
diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 531e06d4eba..dc56039afee 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -52,3 +52,23 @@ integTest {
     setting 'script.max_compilations_per_minute', '1000'
   }
 }
+
+/* Build Javadoc for the Java classes in Painless's public API that are in the
+ * Painless plugin */
+task apiJavadoc(type: Javadoc) {
+  source = sourceSets.main.allJava
+  include '**/org/elasticsearch/painless/api/'
+  destinationDir = new File(docsDir, 'apiJavadoc')
+}
+task apiJavadocJar(type: Jar) {
+  classifier = 'apiJavadoc'
+  from apiJavadoc
+}
+assemble.dependsOn apiJavadocJar
+
+// Reference documentation for Painless's public API.
+task generatePainlessApi(type: JavaExec) {
+  main = 'org.elasticsearch.painless.PainlessDocGenerator'
+  classpath = sourceSets.test.runtimeClasspath
+  args file('../../docs/reference/painless-api-reference')
+}
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 9504a3d911a..f60d48efcc7 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -19,8 +19,20 @@
 
 lexer grammar PainlessLexer;
 
-@header {
-import org.elasticsearch.painless.Definition;
+@members{
+/**
+ * Check against the current whitelist to determine whether a token is a type
+ * or not. Called by the {@code TYPE} token defined in {@code PainlessLexer.g4}.
+ * See also
+ * <a href="https://en.wikipedia.org/wiki/The_lexer_hack">The lexer hack</a>.
+ */
+protected abstract boolean isSimpleType(String name);
+
+/**
+ * Is the preceding {@code /} a the beginning of a regex (true) or a division
+ * (false).
+ */
+protected abstract boolean slashIsRegex();
 }
 
 WS: [ \t\n\r]+ -> skip;
@@ -59,7 +71,7 @@ INSTANCEOF: 'instanceof';
 BOOLNOT: '!';
 BWNOT:   '~';
 MUL:     '*';
-DIV:     '/' { false == SlashStrategy.slashIsRegex(this) }?;
+DIV:     '/' { false == slashIsRegex() }?;
 REM:     '%';
 ADD:     '+';
 SUB:     '-';
@@ -107,8 +119,8 @@ HEX: '0' [xX] [0-9a-fA-F]+ [lL]?;
 INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fFdD]?;
 
-STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
-REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { SlashStrategy.slashIsRegex(this) }?;
+STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\'] )*? '\'' );
+REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { slashIsRegex() }?;
 
 TRUE:  'true';
 FALSE: 'false';
@@ -121,7 +133,7 @@ NULL: 'null';
 // or not.  Note this works by processing one character at a time
 // and the rule is added or removed as this happens.  This is also known
 // as "the lexer hack."  See (https://en.wikipedia.org/wiki/The_lexer_hack).
-TYPE: ID ( DOT ID )* { Definition.isSimpleType(getText()) }?;
+TYPE: ID ( DOT ID )* { isSimpleType(getText()) }?;
 ID: [_a-zA-Z] [_a-zA-Z0-9]*;
 
 mode AFTER_DOT;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 9ef1b2ccf12..378cca7f58f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -41,7 +41,7 @@ public final class CompilerSettings {
      * Constant to be used for enabling additional internal compilation checks (slower).
      */
     public static final String PICKY = "picky";
-    
+
     /**
      * For testing: do not use.
      */
@@ -49,15 +49,17 @@ public final class CompilerSettings {
 
     /**
      * The maximum number of statements allowed to be run in a loop.
+     * For now the number is set fairly high to accommodate users
+     * doing large update queries.
      */
-    private int maxLoopCounter = 10000;
+    private int maxLoopCounter = 1000000;
 
     /**
      * Whether to throw exception on ambiguity or other internal parsing issues. This option
      * makes things slower too, it is only for debugging.
      */
     private boolean picky = false;
-    
+
     /**
      * For testing. Do not use.
      */
@@ -102,7 +104,7 @@ public final class CompilerSettings {
     public void setPicky(boolean picky) {
       this.picky = picky;
     }
-    
+
     /**
      * Returns initial call site depth. This means we pretend we've already seen N different types,
      * to better exercise fallback code in tests.
@@ -110,7 +112,7 @@ public final class CompilerSettings {
     public int getInitialCallSiteDepth() {
         return initialCallSiteDepth;
     }
-    
+
     /**
      * For testing megamorphic fallbacks. Do not use.
      * @see #getInitialCallSiteDepth()
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Debug.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Debug.java
deleted file mode 100644
index 65effdb8a41..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Debug.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.script.ScriptException;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-
-import static java.util.Collections.singletonList;
-
-/**
- * Utility methods for debugging painless scripts that are accessible to painless scripts.
- */
-public class Debug {
-    private Debug() {}
-
-    /**
-     * Throw an {@link Error} that "explains" an object.
-     */
-    public static void explain(Object objectToExplain) throws PainlessExplainError {
-        throw new PainlessExplainError(objectToExplain);
-    }
-
-    /**
-     * Thrown by {@link Debug#explain(Object)} to explain an object. Subclass of {@linkplain Error} so it cannot be caught by painless
-     * scripts.
-     */
-    public static class PainlessExplainError extends Error {
-        private final Object objectToExplain;
-
-        public PainlessExplainError(Object objectToExplain) {
-            this.objectToExplain = objectToExplain;
-        }
-
-        Object getObjectToExplain() {
-            return objectToExplain;
-        }
-
-        /**
-         * Headers to be added to the {@link ScriptException} for structured rendering.
-         */
-        Map<String, List<String>> getHeaders() {
-            Map<String, List<String>> headers = new TreeMap<>();
-            headers.put("es.class", singletonList(objectToExplain == null ? "null" : objectToExplain.getClass().getName()));
-            headers.put("es.to_string", singletonList(Objects.toString(objectToExplain)));
-            return headers;
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 6de116da0e9..1438dab084f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -268,7 +268,7 @@ public final class Def {
              if (lambdaArgs.get(i - 1)) {
                  // decode signature of form 'type.call,2' 
                  String signature = (String) args[upTo++];
-                 int separator = signature.indexOf('.');
+                 int separator = signature.lastIndexOf('.');
                  int separator2 = signature.indexOf(',');
                  String type = signature.substring(1, separator);
                  String call = signature.substring(separator+1, separator2);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e0bdfbf17c2..9f03540c2d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless;
 
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
+import org.elasticsearch.painless.api.Augmentation;
 
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -33,6 +34,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -461,16 +463,22 @@ public final class Definition {
     }
 
     public static final class RuntimeClass {
+        private final Struct struct;
         public final Map<MethodKey, Method> methods;
         public final Map<String, MethodHandle> getters;
         public final Map<String, MethodHandle> setters;
 
-        private RuntimeClass(final Map<MethodKey, Method> methods,
+        private RuntimeClass(final Struct struct, final Map<MethodKey, Method> methods,
                              final Map<String, MethodHandle> getters, final Map<String, MethodHandle> setters) {
+            this.struct = struct;
             this.methods = Collections.unmodifiableMap(methods);
             this.getters = Collections.unmodifiableMap(getters);
             this.setters = Collections.unmodifiableMap(setters);
         }
+
+        public Struct getStruct() {
+            return struct;
+        }
     }
 
     /** Returns whether or not a non-array type exists. */
@@ -503,6 +511,11 @@ public final class Definition {
         return INSTANCE.runtimeMap.get(clazz);
     }
 
+    /** Collection of all simple types. Used by {@code PainlessDocGenerator} to generate an API reference. */
+    static Collection<Type> allSimpleTypes() {
+        return INSTANCE.simpleTypesMap.values();
+    }
+
     // INTERNAL IMPLEMENTATION:
 
     private final Map<Class<?>, RuntimeClass> runtimeMap;
@@ -566,11 +579,11 @@ public final class Definition {
                         }
                         if (line.startsWith("class ")) {
                             String elements[] = line.split("\u0020");
-                            assert elements[2].equals("->");
+                            assert elements[2].equals("->") : "Invalid struct definition [" + String.join(" ", elements) +"]";
                             if (elements.length == 7) {
                                 hierarchy.put(elements[1], Arrays.asList(elements[5].split(",")));
                             } else {
-                                assert elements.length == 5;
+                                assert elements.length == 5 : "Invalid struct definition [" + String.join(" ", elements) + "]";
                             }
                             String className = elements[1];
                             String javaPeer = elements[3];
@@ -612,7 +625,7 @@ public final class Definition {
                     }
                 }
             } catch (Exception e) {
-                throw new RuntimeException("syntax error in " + file + ", line: " + currentLine, e);
+                throw new RuntimeException("error in " + file + ", line: " + currentLine, e);
             }
         }
         return hierarchy;
@@ -1047,7 +1060,7 @@ public final class Definition {
             }
         }
 
-        runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
+        runtimeMap.put(struct.clazz, new RuntimeClass(struct, methods, getters, setters));
     }
 
     /** computes the functional interface method for a class, or returns null */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index c6e38374aa7..c02c66ffc10 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -48,7 +48,7 @@ public abstract class Executable {
         return source;
     }
 
-    /** 
+    /**
      * Finds the start of the first statement boundary that is
      * on or before {@code offset}. If one is not found, {@code -1}
      * is returned.
@@ -56,8 +56,8 @@ public abstract class Executable {
     public int getPreviousStatement(int offset) {
         return statements.previousSetBit(offset);
     }
-    
-    /** 
+
+    /**
      * Finds the start of the first statement boundary that is
      * after {@code offset}. If one is not found, {@code -1}
      * is returned.
@@ -66,6 +66,5 @@ public abstract class Executable {
         return statements.nextSetBit(offset+1);
     }
 
-    public abstract Object execute(
-        final Map<String, Object> params, final Scorer scorer, final LeafDocLookup doc, final Object value);
+    public abstract Object execute(Map<String, Object> params, Scorer scorer, LeafDocLookup doc, Object value);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
new file mode 100644
index 00000000000..75a1d2392cd
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.api.Debug;
+import org.elasticsearch.script.ScriptException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * Thrown by {@link Debug#explain(Object)} to explain an object. Subclass of {@linkplain Error} so it cannot be caught by painless
+ * scripts.
+ */
+public class PainlessExplainError extends Error {
+    private final Object objectToExplain;
+
+    public PainlessExplainError(Object objectToExplain) {
+        this.objectToExplain = objectToExplain;
+    }
+
+    Object getObjectToExplain() {
+        return objectToExplain;
+    }
+
+    /**
+     * Headers to be added to the {@link ScriptException} for structured rendering.
+     */
+    Map<String, List<String>> getHeaders() {
+        Map<String, List<String>> headers = new TreeMap<>();
+        String toString = "null";
+        String javaClassName = null;
+        String painlessClassName = null;
+        if (objectToExplain != null) {
+            toString = objectToExplain.toString();
+            javaClassName = objectToExplain.getClass().getName();
+            Definition.RuntimeClass runtimeClass = Definition.getRuntimeClass(objectToExplain.getClass());
+            if (runtimeClass != null) {
+                painlessClassName = runtimeClass.getStruct().name;
+            }
+        }
+
+        headers.put("es.to_string", singletonList(toString));
+        if (painlessClassName != null) {
+            headers.put("es.painless_class", singletonList(painlessClassName));
+        }
+        if (javaClassName != null) {
+            headers.put("es.java_class", singletonList(javaClassName));
+        }
+        return headers;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 968f5ee2966..4d20a20bd66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -149,11 +149,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         }
 
         // Check we ourselves are not being called by unprivileged code.
-        final SecurityManager sm = System.getSecurityManager();
-
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
 
         // Create our loader (which loads compiled code with no permissions).
         final Loader loader = AccessController.doPrivileged(new PrivilegedAction<Loader>() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 89c8984aaa9..7af6a65d251 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -121,9 +121,9 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     public Object run() {
         try {
             return executable.execute(variables, scorer, doc, aggregationValue);
-        // Note that it is safe to catch any of the following errors since Painless is stateless.
-        } catch (Debug.PainlessExplainError e) {
+        } catch (PainlessExplainError e) {
             throw convertToScriptException(e, e.getHeaders());
+            // Note that it is safe to catch any of the following errors since Painless is stateless.
         } catch (PainlessError | BootstrapMethodError | OutOfMemoryError | StackOverflowError | Exception e) {
             throw convertToScriptException(e, emptyMap());
         }
@@ -135,7 +135,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      * @param t The throwable to build an exception around.
      * @return The generated ScriptException.
      */
-    private ScriptException convertToScriptException(Throwable t, Map<String, List<String>> headers) {
+    private ScriptException convertToScriptException(Throwable t, Map<String, List<String>> metadata) {
         // create a script stack: this is just the script portion
         List<String> scriptStack = new ArrayList<>();
         for (StackTraceElement element : t.getStackTrace()) {
@@ -179,8 +179,8 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
             name = executable.getName();
         }
         ScriptException scriptException = new ScriptException("runtime error", t, scriptStack, name, PainlessScriptEngineService.NAME);
-        for (Map.Entry<String, List<String>> header : headers.entrySet()) {
-            scriptException.addHeader(header.getKey(), header.getValue());
+        for (Map.Entry<String, List<String>> entry : metadata.entrySet()) {
+            scriptException.addMetadata(entry.getKey(), entry.getValue());
         }
         return scriptException;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 684f9a59ee2..32019909a60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.search.Scorer;
+import org.elasticsearch.painless.api.Augmentation;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
index 94a2c258974..640d9c29b20 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
@@ -26,15 +26,18 @@ import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.misc.Pair;
+import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 
 /**
- * A lexer that is customized for painless. It will:
+ * A lexer that is customized for painless. It:
  * <ul>
- * <li>will override the default error behavior to fail on the first error
- * <li>store the last token in case we need to do lookbehind for semicolon insertion and regex vs division detection
- * <li>insert semicolons where they'd improve the language's readability. Rather than hack this into the parser and create a ton of
+ * <li>Overrides the default error behavior to fail on the first error.
+ * <li>Stores the last token in case we need to do lookbehind for semicolon insertion and regex vs division detection.
+ * <li>Implements the regex vs division detection.
+ * <li>Insert semicolons where they'd improve the language's readability. Rather than hack this into the parser and create a ton of
  * ambiguity we hack them here where we can use heuristics to do it quickly.
+ * <li>Enhances the error message when a string contains invalid escape sequences to include a list of valid escape sequences.
  * </ul>
  */
 final class EnhancedPainlessLexer extends PainlessLexer {
@@ -77,7 +80,42 @@ final class EnhancedPainlessLexer extends PainlessLexer {
         final String text = charStream.getText(Interval.of(startIndex, charStream.index()));
 
         Location location = new Location(sourceName, _tokenStartCharIndex);
-        throw location.createError(new IllegalArgumentException("unexpected character [" + getErrorDisplay(text) + "].", lnvae));
+        String message = "unexpected character [" + getErrorDisplay(text) + "].";
+        char firstChar = text.charAt(0);
+        if ((firstChar == '\'' || firstChar == '"') && text.length() - 2 > 0 && text.charAt(text.length() - 2) == '\\') {
+            /* Use a simple heuristic to guess if the unrecognized characters were trying to be a string but has a broken escape sequence.
+             * If it was add an extra message about valid string escape sequences. */
+            message += " The only valid escape sequences in strings starting with [" + firstChar + "] are [\\\\] and [\\"
+                    + firstChar + "].";
+        }
+        throw location.createError(new IllegalArgumentException(message, lnvae));
+    }
+
+    @Override
+    protected boolean isSimpleType(String name) {
+        return Definition.isSimpleType(name);
+    }
+
+    @Override
+    protected boolean slashIsRegex() {
+        Token lastToken = getPreviousToken();
+        if (lastToken == null) {
+            return true;
+        }
+        switch (lastToken.getType()) {
+        case PainlessLexer.RBRACE:
+        case PainlessLexer.RP:
+        case PainlessLexer.OCTAL:
+        case PainlessLexer.HEX:
+        case PainlessLexer.INTEGER:
+        case PainlessLexer.DECIMAL:
+        case PainlessLexer.ID:
+        case PainlessLexer.DOTINTEGER:
+        case PainlessLexer.DOTID:
+            return false;
+        default:
+            return true;
+        }
     }
 
     private static boolean insertSemicolon(Token previous, Token next) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 529ae6f89ea..44972061b59 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -1,7 +1,6 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
 
-import org.elasticsearch.painless.Definition;
 
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
@@ -13,7 +12,7 @@ import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.misc.*;
 
 @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"})
-class PainlessLexer extends Lexer {
+abstract class PainlessLexer extends Lexer {
   static { RuntimeMetaData.checkVersion("4.5.1", RuntimeMetaData.VERSION); }
 
   protected static final DFA[] _decisionToDFA;
@@ -106,6 +105,21 @@ class PainlessLexer extends Lexer {
   }
 
 
+  /**
+   * Check against the current whitelist to determine whether a token is a type
+   * or not. Called by the {@code TYPE} token defined in {@code PainlessLexer.g4}.
+   * See also
+   * <a href="https://en.wikipedia.org/wiki/The_lexer_hack">The lexer hack</a>.
+   */
+  protected abstract boolean isSimpleType(String name);
+
+  /**
+   * Is the preceding {@code /} a the beginning of a regex (true) or a division
+   * (false).
+   */
+  protected abstract boolean slashIsRegex();
+
+
   public PainlessLexer(CharStream input) {
     super(input);
     _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache);
@@ -141,21 +155,21 @@ class PainlessLexer extends Lexer {
   private boolean DIV_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 0:
-      return  false == SlashStrategy.slashIsRegex(this) ;
+      return  false == slashIsRegex() ;
     }
     return true;
   }
   private boolean REGEX_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return  SlashStrategy.slashIsRegex(this) ;
+      return  slashIsRegex() ;
     }
     return true;
   }
   private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 2:
-      return  Definition.isSimpleType(getText()) ;
+      return  isSimpleType(getText()) ;
     }
     return true;
   }
@@ -203,51 +217,51 @@ class PainlessLexer extends Lexer {
     "\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64"+
     "h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008a"+
     "F\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009e"+
-    "P\u00a0Q\u00a2R\u00a4S\u00a6T\u00a8U\u00aaV\4\2\3\24\5\2\13\f\17\17\""+
+    "P\u00a0Q\u00a2R\u00a4S\u00a6T\u00a8U\u00aaV\4\2\3\25\5\2\13\f\17\17\""+
     "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+
-    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2\f\f\61\61\3\2\f"+
-    "\f\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0277\2\4\3\2\2\2\2\6\3"+
-    "\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2"+
-    "\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3"+
-    "\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3"+
-    "\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64"+
-    "\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3"+
-    "\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2"+
-    "\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2"+
-    "Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3"+
-    "\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2"+
-    "\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2"+
-    "\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088"+
-    "\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2"+
-    "\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2\2\2\u009a"+
-    "\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2\2\2\u00a2\3\2\2"+
-    "\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\3\u00a8\3\2\2\2\3\u00aa\3\2\2\2\4\u00ad"+
-    "\3\2\2\2\6\u00c8\3\2\2\2\b\u00cc\3\2\2\2\n\u00ce\3\2\2\2\f\u00d0\3\2\2"+
-    "\2\16\u00d2\3\2\2\2\20\u00d4\3\2\2\2\22\u00d6\3\2\2\2\24\u00d8\3\2\2\2"+
-    "\26\u00dc\3\2\2\2\30\u00e1\3\2\2\2\32\u00e3\3\2\2\2\34\u00e5\3\2\2\2\36"+
-    "\u00e8\3\2\2\2 \u00eb\3\2\2\2\"\u00f0\3\2\2\2$\u00f6\3\2\2\2&\u00f9\3"+
-    "\2\2\2(\u00fd\3\2\2\2*\u0106\3\2\2\2,\u010c\3\2\2\2.\u0113\3\2\2\2\60"+
-    "\u0117\3\2\2\2\62\u011b\3\2\2\2\64\u0121\3\2\2\2\66\u0127\3\2\2\28\u012c"+
-    "\3\2\2\2:\u0137\3\2\2\2<\u0139\3\2\2\2>\u013b\3\2\2\2@\u013d\3\2\2\2B"+
-    "\u0140\3\2\2\2D\u0142\3\2\2\2F\u0144\3\2\2\2H\u0146\3\2\2\2J\u0149\3\2"+
-    "\2\2L\u014c\3\2\2\2N\u0150\3\2\2\2P\u0152\3\2\2\2R\u0155\3\2\2\2T\u0157"+
-    "\3\2\2\2V\u015a\3\2\2\2X\u015d\3\2\2\2Z\u0161\3\2\2\2\\\u0164\3\2\2\2"+
-    "^\u0168\3\2\2\2`\u016a\3\2\2\2b\u016c\3\2\2\2d\u016e\3\2\2\2f\u0171\3"+
-    "\2\2\2h\u0174\3\2\2\2j\u0176\3\2\2\2l\u0178\3\2\2\2n\u017b\3\2\2\2p\u017e"+
-    "\3\2\2\2r\u0181\3\2\2\2t\u0184\3\2\2\2v\u0188\3\2\2\2x\u018b\3\2\2\2z"+
-    "\u018e\3\2\2\2|\u0190\3\2\2\2~\u0193\3\2\2\2\u0080\u0196\3\2\2\2\u0082"+
-    "\u0199\3\2\2\2\u0084\u019c\3\2\2\2\u0086\u019f\3\2\2\2\u0088\u01a2\3\2"+
-    "\2\2\u008a\u01a5\3\2\2\2\u008c\u01a8\3\2\2\2\u008e\u01ac\3\2\2\2\u0090"+
-    "\u01b0\3\2\2\2\u0092\u01b5\3\2\2\2\u0094\u01be\3\2\2\2\u0096\u01d0\3\2"+
-    "\2\2\u0098\u01dd\3\2\2\2\u009a\u020d\3\2\2\2\u009c\u020f\3\2\2\2\u009e"+
-    "\u0220\3\2\2\2\u00a0\u0225\3\2\2\2\u00a2\u022b\3\2\2\2\u00a4\u0230\3\2"+
-    "\2\2\u00a6\u023b\3\2\2\2\u00a8\u024a\3\2\2\2\u00aa\u024e\3\2\2\2\u00ac"+
-    "\u00ae\t\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00ad\3\2"+
-    "\2\2\u00af\u00b0\3\2\2\2\u00b0\u00b1\3\2\2\2\u00b1\u00b2\b\2\2\2\u00b2"+
-    "\5\3\2\2\2\u00b3\u00b4\7\61\2\2\u00b4\u00b5\7\61\2\2\u00b5\u00b9\3\2\2"+
-    "\2\u00b6\u00b8\13\2\2\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2\u00b9"+
-    "\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9\3\2"+
-    "\2\2\u00bc\u00c9\t\3\2\2\u00bd\u00be\7\61\2\2\u00be\u00bf\7,\2\2\u00bf"+
+    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2))^^\4\2\f\f\61"+
+    "\61\3\2\f\f\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0277\2\4\3\2"+
+    "\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20"+
+    "\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2"+
+    "\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3"+
+    "\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3"+
+    "\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3"+
+    "\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2"+
+    "\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2"+
+    "X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3"+
+    "\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2"+
+    "\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2"+
+    "~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2"+
+    "\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2"+
+    "\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098"+
+    "\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2"+
+    "\2\2\u00a2\3\2\2\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\3\u00a8\3\2\2\2\3\u00aa"+
+    "\3\2\2\2\4\u00ad\3\2\2\2\6\u00c8\3\2\2\2\b\u00cc\3\2\2\2\n\u00ce\3\2\2"+
+    "\2\f\u00d0\3\2\2\2\16\u00d2\3\2\2\2\20\u00d4\3\2\2\2\22\u00d6\3\2\2\2"+
+    "\24\u00d8\3\2\2\2\26\u00dc\3\2\2\2\30\u00e1\3\2\2\2\32\u00e3\3\2\2\2\34"+
+    "\u00e5\3\2\2\2\36\u00e8\3\2\2\2 \u00eb\3\2\2\2\"\u00f0\3\2\2\2$\u00f6"+
+    "\3\2\2\2&\u00f9\3\2\2\2(\u00fd\3\2\2\2*\u0106\3\2\2\2,\u010c\3\2\2\2."+
+    "\u0113\3\2\2\2\60\u0117\3\2\2\2\62\u011b\3\2\2\2\64\u0121\3\2\2\2\66\u0127"+
+    "\3\2\2\28\u012c\3\2\2\2:\u0137\3\2\2\2<\u0139\3\2\2\2>\u013b\3\2\2\2@"+
+    "\u013d\3\2\2\2B\u0140\3\2\2\2D\u0142\3\2\2\2F\u0144\3\2\2\2H\u0146\3\2"+
+    "\2\2J\u0149\3\2\2\2L\u014c\3\2\2\2N\u0150\3\2\2\2P\u0152\3\2\2\2R\u0155"+
+    "\3\2\2\2T\u0157\3\2\2\2V\u015a\3\2\2\2X\u015d\3\2\2\2Z\u0161\3\2\2\2\\"+
+    "\u0164\3\2\2\2^\u0168\3\2\2\2`\u016a\3\2\2\2b\u016c\3\2\2\2d\u016e\3\2"+
+    "\2\2f\u0171\3\2\2\2h\u0174\3\2\2\2j\u0176\3\2\2\2l\u0178\3\2\2\2n\u017b"+
+    "\3\2\2\2p\u017e\3\2\2\2r\u0181\3\2\2\2t\u0184\3\2\2\2v\u0188\3\2\2\2x"+
+    "\u018b\3\2\2\2z\u018e\3\2\2\2|\u0190\3\2\2\2~\u0193\3\2\2\2\u0080\u0196"+
+    "\3\2\2\2\u0082\u0199\3\2\2\2\u0084\u019c\3\2\2\2\u0086\u019f\3\2\2\2\u0088"+
+    "\u01a2\3\2\2\2\u008a\u01a5\3\2\2\2\u008c\u01a8\3\2\2\2\u008e\u01ac\3\2"+
+    "\2\2\u0090\u01b0\3\2\2\2\u0092\u01b5\3\2\2\2\u0094\u01be\3\2\2\2\u0096"+
+    "\u01d0\3\2\2\2\u0098\u01dd\3\2\2\2\u009a\u020d\3\2\2\2\u009c\u020f\3\2"+
+    "\2\2\u009e\u0220\3\2\2\2\u00a0\u0225\3\2\2\2\u00a2\u022b\3\2\2\2\u00a4"+
+    "\u0230\3\2\2\2\u00a6\u023b\3\2\2\2\u00a8\u024a\3\2\2\2\u00aa\u024e\3\2"+
+    "\2\2\u00ac\u00ae\t\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af"+
+    "\u00ad\3\2\2\2\u00af\u00b0\3\2\2\2\u00b0\u00b1\3\2\2\2\u00b1\u00b2\b\2"+
+    "\2\2\u00b2\5\3\2\2\2\u00b3\u00b4\7\61\2\2\u00b4\u00b5\7\61\2\2\u00b5\u00b9"+
+    "\3\2\2\2\u00b6\u00b8\13\2\2\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2"+
+    "\u00b9\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9"+
+    "\3\2\2\2\u00bc\u00c9\t\3\2\2\u00bd\u00be\7\61\2\2\u00be\u00bf\7,\2\2\u00bf"+
     "\u00c3\3\2\2\2\u00c0\u00c2\13\2\2\2\u00c1\u00c0\3\2\2\2\u00c2\u00c5\3"+
     "\2\2\2\u00c3\u00c4\3\2\2\2\u00c3\u00c1\3\2\2\2\u00c4\u00c6\3\2\2\2\u00c5"+
     "\u00c3\3\2\2\2\u00c6\u00c7\7,\2\2\u00c7\u00c9\7\61\2\2\u00c8\u00b3\3\2"+
@@ -340,14 +354,14 @@ class PainlessLexer extends Lexer {
     "\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fe\3\2\2\2\u01fd\u01fb\3\2"+
     "\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200\u020e\7$\2\2\u0201"+
     "\u0209\7)\2\2\u0202\u0203\7^\2\2\u0203\u0208\7)\2\2\u0204\u0205\7^\2\2"+
-    "\u0205\u0208\7^\2\2\u0206\u0208\n\16\2\2\u0207\u0202\3\2\2\2\u0207\u0204"+
+    "\u0205\u0208\7^\2\2\u0206\u0208\n\17\2\2\u0207\u0202\3\2\2\2\u0207\u0204"+
     "\3\2\2\2\u0207\u0206\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u020a\3\2\2\2\u0209"+
     "\u0207\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c\u020e\7)"+
     "\2\2\u020d\u01f5\3\2\2\2\u020d\u0201\3\2\2\2\u020e\u009b\3\2\2\2\u020f"+
-    "\u0213\7\61\2\2\u0210\u0214\n\17\2\2\u0211\u0212\7^\2\2\u0212\u0214\n"+
-    "\20\2\2\u0213\u0210\3\2\2\2\u0213\u0211\3\2\2\2\u0214\u0215\3\2\2\2\u0215"+
+    "\u0213\7\61\2\2\u0210\u0214\n\20\2\2\u0211\u0212\7^\2\2\u0212\u0214\n"+
+    "\21\2\2\u0213\u0210\3\2\2\2\u0213\u0211\3\2\2\2\u0214\u0215\3\2\2\2\u0215"+
     "\u0213\3\2\2\2\u0215\u0216\3\2\2\2\u0216\u0217\3\2\2\2\u0217\u021b\7\61"+
-    "\2\2\u0218\u021a\t\21\2\2\u0219\u0218\3\2\2\2\u021a\u021d\3\2\2\2\u021b"+
+    "\2\2\u0218\u021a\t\22\2\2\u0219\u0218\3\2\2\2\u021a\u021d\3\2\2\2\u021b"+
     "\u0219\3\2\2\2\u021b\u021c\3\2\2\2\u021c\u021e\3\2\2\2\u021d\u021b\3\2"+
     "\2\2\u021e\u021f\6N\3\2\u021f\u009d\3\2\2\2\u0220\u0221\7v\2\2\u0221\u0222"+
     "\7t\2\2\u0222\u0223\7w\2\2\u0223\u0224\7g\2\2\u0224\u009f\3\2\2\2\u0225"+
@@ -357,14 +371,14 @@ class PainlessLexer extends Lexer {
     "\u0236\5\u00a6S\2\u0231\u0232\5\24\n\2\u0232\u0233\5\u00a6S\2\u0233\u0235"+
     "\3\2\2\2\u0234\u0231\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236"+
     "\u0237\3\2\2\2\u0237\u0239\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u023a\6R"+
-    "\4\2\u023a\u00a5\3\2\2\2\u023b\u023f\t\22\2\2\u023c\u023e\t\23\2\2\u023d"+
+    "\4\2\u023a\u00a5\3\2\2\2\u023b\u023f\t\23\2\2\u023c\u023e\t\24\2\2\u023d"+
     "\u023c\3\2\2\2\u023e\u0241\3\2\2\2\u023f\u023d\3\2\2\2\u023f\u0240\3\2"+
     "\2\2\u0240\u00a7\3\2\2\2\u0241\u023f\3\2\2\2\u0242\u024b\7\62\2\2\u0243"+
     "\u0247\t\b\2\2\u0244\u0246\t\t\2\2\u0245\u0244\3\2\2\2\u0246\u0249\3\2"+
     "\2\2\u0247\u0245\3\2\2\2\u0247\u0248\3\2\2\2\u0248\u024b\3\2\2\2\u0249"+
     "\u0247\3\2\2\2\u024a\u0242\3\2\2\2\u024a\u0243\3\2\2\2\u024b\u024c\3\2"+
-    "\2\2\u024c\u024d\bT\4\2\u024d\u00a9\3\2\2\2\u024e\u0252\t\22\2\2\u024f"+
-    "\u0251\t\23\2\2\u0250\u024f\3\2\2\2\u0251\u0254\3\2\2\2\u0252\u0250\3"+
+    "\2\2\u024c\u024d\bT\4\2\u024d\u00a9\3\2\2\2\u024e\u0252\t\23\2\2\u024f"+
+    "\u0251\t\24\2\2\u0250\u024f\3\2\2\2\u0251\u0254\3\2\2\2\u0252\u0250\3"+
     "\2\2\2\u0252\u0253\3\2\2\2\u0253\u0255\3\2\2\2\u0254\u0252\3\2\2\2\u0255"+
     "\u0256\bU\4\2\u0256\u00ab\3\2\2\2$\2\3\u00af\u00b9\u00c3\u00c8\u01b9\u01bc"+
     "\u01c3\u01c6\u01cd\u01d0\u01d3\u01da\u01dd\u01e3\u01e5\u01e9\u01ee\u01f0"+
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 84d58afa62d..9de833e5419 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -799,9 +799,27 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
     @Override
     public ANode visitString(StringContext ctx) {
-        String string = ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1);
+        StringBuilder string = new StringBuilder(ctx.STRING().getText());
 
-        return new EString(location(ctx), string);
+        // Strip the leading and trailing quotes and replace the escape sequences with their literal equivalents
+        int src = 1;
+        int dest = 0;
+        int end = string.length() - 1;
+        assert string.charAt(0) == '"' || string.charAt(0) == '\'' : "expected string to start with a quote but was [" + string + "]";
+        assert string.charAt(end) == '"' || string.charAt(end) == '\'' : "expected string to end with a quote was [" + string + "]";
+        while (src < end) {
+            char current = string.charAt(src);
+            if (current == '\\') {
+                src++;
+                current = string.charAt(src);
+            }
+            string.setCharAt(dest, current);
+            src++;
+            dest++;
+        }
+        string.setLength(dest);
+
+        return new EString(location(ctx), string.toString());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java
new file mode 100644
index 00000000000..041118ba029
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Lexer, parser, and tree {@link Walker} responsible for turning the code
+ * generating nodes in {@link org.elasticsearch.painless.node}.
+ */
+package org.elasticsearch.painless.antlr;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
similarity index 95%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
index 9302f3c899c..0b751b7d2f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Augmentation.java
@@ -17,9 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.api;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -486,4 +488,19 @@ public class Augmentation {
     private static int initialBufferForReplaceWith(CharSequence seq) {
         return seq.length() + 16;
     }
+
+    /**
+     * Encode a String in Base64. Use {@link Base64.Encoder#encodeToString(byte[])} if you have to encode bytes rather than a string.
+     */
+    public static String encodeBase64(String receiver) {
+        return Base64.getEncoder().encodeToString(receiver.getBytes(StandardCharsets.UTF_8));
+    }
+
+    /**
+     * Decode some Base64 bytes and build a UTF-8 encoded string. Use {@link Base64.Decoder#decode(String)} if you'd prefer bytes to work
+     * with bytes.
+     */
+    public static String decodeBase64(String receiver) {
+        return new String(Base64.getDecoder().decode(receiver.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java
similarity index 65%
rename from core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java
index 0b0370b490e..de0c9b3adbd 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/FromXContentBuilder.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/Debug.java
@@ -17,18 +17,20 @@
  * under the License.
  */
 
-package org.elasticsearch.common.xcontent;
+package org.elasticsearch.painless.api;
 
-import org.elasticsearch.common.ParseFieldMatcher;
-
-import java.io.IOException;
+import org.elasticsearch.painless.PainlessExplainError;
 
 /**
- * Indicates that the class supports XContent deserialization.
+ * Utility methods for debugging painless scripts that are accessible to painless scripts.
  */
-public interface FromXContentBuilder<T> {
+public class Debug {
+    private Debug() {}
+
     /**
-     * Parses an object with the type T from parser
+     * Throw an {@link Error} that "explains" an object.
      */
-    T fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException;
+    public static void explain(Object objectToExplain) throws PainlessExplainError {
+        throw new PainlessExplainError(objectToExplain);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java
new file mode 100644
index 00000000000..b1c78e28ec1
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Java code publicly exposed to Painless code.
+ */
+package org.elasticsearch.painless.api;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
index 86553096e35..5665f7163af 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java
@@ -39,7 +39,7 @@ final class PSubCallInvoke extends AExpression {
     private final Type box;
     private final List<AExpression> arguments;
 
-    public PSubCallInvoke(Location location, Method method, Type box, List<AExpression> arguments) {
+    PSubCallInvoke(Location location, Method method, Type box, List<AExpression> arguments) {
         super(location);
 
         this.method = Objects.requireNonNull(method);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
index a7a01caac46..efed81c4c81 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java
@@ -37,7 +37,7 @@ final class PSubField extends AStoreable {
 
     private final Field field;
 
-    public PSubField(Location location, Field field) {
+    PSubField(Location location, Field field) {
         super(location);
 
         this.field = Objects.requireNonNull(field);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
index edbad88e2c2..c153eacbf92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java
@@ -47,7 +47,7 @@ final class SSubEachArray extends AStatement {
     private Variable index = null;
     private Type indexed = null;
 
-    public SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachArray(Location location, Variable variable, AExpression expression, SBlock block) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
index 5a1410d0343..845cc264530 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java
@@ -55,7 +55,7 @@ final class SSubEachIterable extends AStatement {
     private Variable iterator = null;
     private Method method = null;
 
-    public SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
+    SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) {
         super(location);
 
         this.variable = Objects.requireNonNull(variable);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java
new file mode 100644
index 00000000000..d45d4fb6d49
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Implementation of the Painless language.
+ */
+package org.elasticsearch.painless;
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 13f28d3ebeb..a1cde1711bc 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -756,6 +756,8 @@ class String -> java.lang.String extends CharSequence,Comparable,Object {
   boolean contentEquals(CharSequence)
   String copyValueOf(char[])
   String copyValueOf(char[],int,int)
+  String decodeBase64*()
+  String encodeBase64*()
   boolean endsWith(String)
   boolean equalsIgnoreCase(String)
   String format(Locale,String,def[])
@@ -1101,4 +1103,3 @@ class UnsupportedOperationException -> java.lang.UnsupportedOperationException e
   UnsupportedOperationException <init>()
   UnsupportedOperationException <init>(String)
 }
-
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index bd2f1c1b5db..61baabbc951 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -60,7 +60,7 @@ class def -> java.lang.Object {
 
 #### Painless debugging API
 
-class Debug -> org.elasticsearch.painless.Debug extends Object {
+class Debug -> org.elasticsearch.painless.api.Debug extends Object {
   void explain(Object)
 }
 
@@ -82,6 +82,7 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsear
   long getValue()
   List getValues()
   org.joda.time.ReadableDateTime getDate()
+  List getDates()
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
@@ -114,6 +115,26 @@ class org.elasticsearch.index.fielddata.ScriptDocValues.Booleans -> org.elastics
   List getValues()
 }
 
+class org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs -> org.elasticsearch.index.fielddata.ScriptDocValues$BytesRefs extends List,Collection,Iterable,Object {
+  BytesRef get(int)
+  BytesRef getValue()
+  List getValues()
+}
+
+class BytesRef -> org.apache.lucene.util.BytesRef extends Object {
+  byte[] bytes
+  int offset
+  int length
+  boolean bytesEquals(BytesRef)
+  String utf8ToString()
+}
+
+class org.elasticsearch.index.mapper.IpFieldMapper.IpFieldType.IpScriptDocValues -> org.elasticsearch.index.mapper.IpFieldMapper$IpFieldType$IpScriptDocValues extends List,Collection,Iterable,Object {
+  String get(int)
+  String getValue()
+  List getValues()
+}
+
 # for testing.
 # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
 class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index 7c0694d67ba..ef2ddad5452 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -42,6 +42,7 @@ public class BasicExpressionTests extends ScriptTestCase {
         assertEquals((byte)255, exec("return (byte)255"));
         assertEquals((short)5, exec("return (short)5"));
         assertEquals("string", exec("return \"string\""));
+        assertEquals("string", exec("return 'string'"));
         assertEquals(true, exec("return true"));
         assertEquals(false, exec("return false"));
         assertNull(exec("return null"));
@@ -55,6 +56,37 @@ public class BasicExpressionTests extends ScriptTestCase {
         assertEquals('蚠', exec("return (char)100000;"));
     }
 
+    public void testStringEscapes() {
+        // The readability of this test suffers from having to escape `\` and `"` in java strings. Please be careful. Sorry!
+        // `\\` is a `\`
+        assertEquals("\\string", exec("\"\\\\string\""));
+        assertEquals("\\string", exec("'\\\\string'"));
+        // `\"` is a `"` if surrounded by `"`s
+        assertEquals("\"string", exec("\"\\\"string\""));
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'\\\"string'", false));
+        assertEquals("unexpected character ['\\\"]. The only valid escape sequences in strings starting with ['] are [\\\\] and [\\'].",
+                e.getMessage());
+        // `\'` is a `'` if surrounded by `'`s
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("\"\\'string\"", false));
+        assertEquals("unexpected character [\"\\']. The only valid escape sequences in strings starting with [\"] are [\\\\] and [\\\"].",
+                e.getMessage());
+        assertEquals("'string", exec("'\\'string'"));
+        // We don't break native escapes like new line
+        assertEquals("\nstring", exec("\"\nstring\""));
+        assertEquals("\nstring", exec("'\nstring'"));
+
+        // And we're ok with strings with multiple escape sequences
+        assertEquals("\\str\"in\\g", exec("\"\\\\str\\\"in\\\\g\""));
+        assertEquals("st\\r'i\\ng", exec("'st\\\\r\\'i\\\\ng'"));
+    }
+
+    public void testStringTermination() {
+        // `'` inside of a string delimited with `"` should be ok
+        assertEquals("test'", exec("\"test'\""));
+        // `"` inside of a string delimited with `'` should be ok
+        assertEquals("test\"", exec("'test\"'"));
+    }
+
     /** declaring variables for primitive types */
     public void testDeclareVariable() {
         assertEquals(5, exec("int i = 5; return i;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
index 3b948bd2590..0c96251a51a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java
@@ -30,49 +30,54 @@ import java.util.Map;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
 
 public class DebugTests extends ScriptTestCase {
     public void testExplain() {
+        // Debug.explain can explain an object
         Object dummy = new Object();
-        Map<String, Object> params = singletonMap("a", dummy);
-
-        Debug.PainlessExplainError e = expectScriptThrows(Debug.PainlessExplainError.class, () -> exec(
-                "Debug.explain(params.a)", params, true));
+        PainlessExplainError e = expectScriptThrows(PainlessExplainError.class, () -> exec(
+                "Debug.explain(params.a)", singletonMap("a", dummy), true));
         assertSame(dummy, e.getObjectToExplain());
-        assertThat(e.getHeaders(), hasEntry("es.class", singletonList("java.lang.Object")));
         assertThat(e.getHeaders(), hasEntry("es.to_string", singletonList(dummy.toString())));
+        assertThat(e.getHeaders(), hasEntry("es.java_class", singletonList("java.lang.Object")));
+        assertThat(e.getHeaders(), hasEntry("es.painless_class", singletonList("Object")));
 
         // Null should be ok
-        e = expectScriptThrows(Debug.PainlessExplainError.class, () -> exec("Debug.explain(null)"));
+        e = expectScriptThrows(PainlessExplainError.class, () -> exec("Debug.explain(null)"));
         assertNull(e.getObjectToExplain());
-        assertThat(e.getHeaders(), hasEntry("es.class", singletonList("null")));
         assertThat(e.getHeaders(), hasEntry("es.to_string", singletonList("null")));
+        assertThat(e.getHeaders(), not(hasKey("es.java_class")));
+        assertThat(e.getHeaders(), not(hasKey("es.painless_class")));
 
         // You can't catch the explain exception
-        e = expectScriptThrows(Debug.PainlessExplainError.class, () -> exec(
+        e = expectScriptThrows(PainlessExplainError.class, () -> exec(
                 "try {\n"
               + "  Debug.explain(params.a)\n"
               + "} catch (Exception e) {\n"
               + "  return 1\n"
-              + "}", params, true));
+              + "}", singletonMap("a", dummy), true));
         assertSame(dummy, e.getObjectToExplain());
     }
 
     /**
-     * {@link Debug.PainlessExplainError} doesn't serialize but the headers still make it.
+     * {@link PainlessExplainError} doesn't serialize but the headers still make it.
      */
     public void testPainlessExplainErrorSerialization() throws IOException {
         Map<String, Object> params = singletonMap("a", "jumped over the moon");
         ScriptException e = expectThrows(ScriptException.class, () -> exec("Debug.explain(params.a)", params, true));
-        assertEquals(singletonList("java.lang.String"), e.getHeader("es.class"));
-        assertEquals(singletonList("jumped over the moon"), e.getHeader("es.to_string"));
+        assertEquals(singletonList("jumped over the moon"), e.getMetadata("es.to_string"));
+        assertEquals(singletonList("java.lang.String"), e.getMetadata("es.java_class"));
+        assertEquals(singletonList("String"), e.getMetadata("es.painless_class"));
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeException(e);
             try (StreamInput in = out.bytes().streamInput()) {
                 ElasticsearchException read = (ScriptException) in.readException();
-                assertEquals(singletonList("java.lang.String"), read.getHeader("es.class"));
-                assertEquals(singletonList("jumped over the moon"), read.getHeader("es.to_string"));
+                assertEquals(singletonList("jumped over the moon"), read.getMetadata("es.to_string"));
+                assertEquals(singletonList("java.lang.String"), read.getMetadata("es.java_class"));
+                assertEquals(singletonList("String"), read.getMetadata("es.painless_class"));
             }
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 7136807bd7e..0f9ef54d614 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -19,6 +19,13 @@
 
 package org.elasticsearch.painless;
 
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.startsWith;
+
 public class FunctionRefTests extends ScriptTestCase {
 
     public void testStaticMethodReference() {
@@ -37,6 +44,30 @@ public class FunctionRefTests extends ScriptTestCase {
         assertEquals(2, exec("def l = new ArrayList(); l.add(1); l.add(1); return l.stream().mapToInt(Integer::intValue).sum();"));
     }
 
+    public void testQualifiedStaticMethodReference() {
+        assertEquals(true,
+                exec("List l = [true]; l.stream().map(org.elasticsearch.painless.FeatureTest::overloadedStatic).findFirst().get()"));
+    }
+
+    public void testQualifiedStaticMethodReferenceDef() {
+        assertEquals(true,
+                exec("def l = [true]; l.stream().map(org.elasticsearch.painless.FeatureTest::overloadedStatic).findFirst().get()"));
+    }
+
+    public void testQualifiedVirtualMethodReference() {
+        long instant = randomLong();
+        assertEquals(instant, exec(
+                "List l = [params.d]; return l.stream().mapToLong(org.joda.time.ReadableDateTime::getMillis).sum()",
+                singletonMap("d", new DateTime(instant, DateTimeZone.UTC)), true));
+    }
+
+    public void testQualifiedVirtualMethodReferenceDef() {
+        long instant = randomLong();
+        assertEquals(instant, exec(
+                "def l = [params.d]; return l.stream().mapToLong(org.joda.time.ReadableDateTime::getMillis).sum()",
+                singletonMap("d", new DateTime(instant, DateTimeZone.UTC)), true));
+    }
+
     public void testCtorMethodReference() {
         assertEquals(3.0D, 
                 exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
@@ -144,10 +175,33 @@ public class FunctionRefTests extends ScriptTestCase {
     }
 
     public void testMethodMissing() {
-        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
-            exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::bogus); return l.get(0);");
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);");
         });
-        assertTrue(expected.getMessage().contains("Unknown reference"));
+        assertThat(e.getMessage(), startsWith("Unknown reference"));
+    }
+
+    public void testQualifiedMethodMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(org.joda.time.ReadableDateTime::bogus); return l.get(0);", false);
+        });
+        assertThat(e.getMessage(), startsWith("Unknown reference"));
+    }
+
+    public void testClassMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(Bogus::bogus); return l.get(0);", false);
+        });
+        assertThat(e.getMessage(), endsWith("Variable [Bogus] is not defined."));
+    }
+
+    public void testQualifiedClassMissing() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = [2, 1]; l.sort(org.joda.time.BogusDateTime::bogus); return l.get(0);", false);
+        });
+        /* Because the type isn't known and we use the lexer hack this fails to parse. I find this error message confusing but it is the one
+         * we have... */
+        assertEquals("invalid sequence of tokens near ['::'].", e.getMessage());
     }
 
     public void testNotFunctionalInterface() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
index 55d3f1c8101..9d055b74cc8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -36,7 +35,7 @@ public class LangPainlessClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
new file mode 100644
index 00000000000..910c4940ab5
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.api.Augmentation;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Consumer;
+
+import static java.util.Comparator.comparing;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Generates an API reference from the method and type whitelists in {@link Definition}.
+ */
+public class PainlessDocGenerator {
+    private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class);
+    private static final Comparator<Field> FIELD_NAME = comparing(f -> f.name);
+    private static final Comparator<Method> METHOD_NAME = comparing(m -> m.name);
+    private static final Comparator<Method> NUMBER_OF_ARGS = comparing(m -> m.arguments.size());
+
+    public static void main(String[] args) throws IOException {
+        Path apiRootPath = PathUtils.get(args[0]);
+
+        // Blow away the last execution and recreate it from scratch
+        IOUtils.rm(apiRootPath);
+        Files.createDirectories(apiRootPath);
+
+        Path indexPath = apiRootPath.resolve("index.asciidoc");
+        logger.info("Starting to write [index.asciidoc]");
+        try (PrintStream indexStream = new PrintStream(
+                Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
+                false, StandardCharsets.UTF_8.name())) {
+            emitGeneratedWarning(indexStream);
+            List<Type> types = Definition.allSimpleTypes().stream().sorted(comparing(t -> t.name)).collect(toList());
+            for (Type type : types) {
+                if (type.sort.primitive) {
+                    // Primitives don't have methods to reference
+                    continue;
+                }
+                if ("def".equals(type.name)) {
+                    // def is special but doesn't have any methods all of its own.
+                    continue;
+                }
+                indexStream.print("include::");
+                indexStream.print(type.struct.name);
+                indexStream.println(".asciidoc[]");
+
+                Path typePath = apiRootPath.resolve(type.struct.name + ".asciidoc");
+                logger.info("Writing [{}.asciidoc]", type.name);
+                try (PrintStream typeStream = new PrintStream(
+                        Files.newOutputStream(typePath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE),
+                        false, StandardCharsets.UTF_8.name())) {
+                    emitGeneratedWarning(typeStream);
+                    typeStream.print("[[");
+                    emitAnchor(typeStream, type.struct);
+                    typeStream.print("]]++");
+                    typeStream.print(type.name);
+                    typeStream.println("++::");
+
+                    Consumer<Field> documentField = field -> PainlessDocGenerator.documentField(typeStream, field);
+                    Consumer<Method> documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method);
+                    type.struct.staticMembers.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    type.struct.members.values().stream().sorted(FIELD_NAME).forEach(documentField);
+                    type.struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod);
+                    type.struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod);
+                    Map<String, Struct> inherited = new TreeMap<>();
+                    type.struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> {
+                        if (method.owner == type.struct) {
+                            documentMethod(typeStream, method);
+                        } else {
+                            inherited.put(method.owner.name, method.owner);
+                        }
+                    });
+
+                    if (false == inherited.isEmpty()) {
+                        typeStream.print("* Inherits methods from ");
+                        boolean first = true;
+                        for (Struct inheritsFrom : inherited.values()) {
+                            if (first) {
+                                first = false;
+                            } else {
+                                typeStream.print(", ");
+                            }
+                            typeStream.print("++");
+                            emitStruct(typeStream, inheritsFrom);
+                            typeStream.print("++");
+                        }
+                        typeStream.println();
+                    }
+                }
+            }
+        }
+        logger.info("Done writing [index.asciidoc]");
+    }
+
+    private static void documentField(PrintStream stream, Field field) {
+        stream.print("** [[");
+        emitAnchor(stream, field);
+        stream.print("]]");
+
+        if (Modifier.isStatic(field.modifiers)) {
+            stream.print("static ");
+        }
+
+        emitType(stream, field.type);
+        stream.print(' ');
+
+        String javadocRoot = javadocRoot(field);
+        emitJavadocLink(stream, javadocRoot, field);
+        stream.print('[');
+        stream.print(field.name);
+        stream.print(']');
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", field);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
+    /**
+     * Document a method.
+     */
+    private static void documentMethod(PrintStream stream, Method method) {
+        stream.print("* ++[[");
+        emitAnchor(stream, method);
+        stream.print("]]");
+
+        if (false == method.augmentation && Modifier.isStatic(method.modifiers)) {
+            stream.print("static ");
+        }
+
+        if (false == method.name.equals("<init>")) {
+            emitType(stream, method.rtn);
+            stream.print(' ');
+        }
+
+        String javadocRoot = javadocRoot(method);
+        emitJavadocLink(stream, javadocRoot, method);
+        stream.print('[');
+
+        stream.print(methodName(method));
+
+        stream.print("](");
+        boolean first = true;
+        for (Type arg : method.arguments) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print(", ");
+            }
+            emitType(stream, arg);
+        }
+        stream.print(")++");
+
+        if (javadocRoot.equals("java8")) {
+            stream.print(" (");
+            emitJavadocLink(stream, "java9", method);
+            stream.print("[java 9])");
+        }
+
+        stream.println();
+    }
+
+    /**
+     * Anchor text for a {@link Struct}.
+     */
+    private static void emitAnchor(PrintStream stream, Struct struct) {
+        stream.print("painless-api-reference-");
+        stream.print(struct.name.replace('.', '-'));
+    }
+
+    /**
+     * Anchor text for a {@link Method}.
+     */
+    private static void emitAnchor(PrintStream stream, Method method) {
+        emitAnchor(stream, method.owner);
+        stream.print('-');
+        stream.print(methodName(method));
+        stream.print('-');
+        stream.print(method.arguments.size());
+    }
+
+    /**
+     * Anchor text for a {@link Field}.
+     */
+    private static void emitAnchor(PrintStream stream, Field field) {
+        emitAnchor(stream, field.owner);
+        stream.print('-');
+        stream.print(field.name);
+    }
+
+    private static String methodName(Method method) {
+        return method.name.equals("<init>") ? method.owner.name : method.name;
+    }
+
+    /**
+     * Emit a {@link Type}. If the type is primitive or an array of primitives this just emits the name of the type. Otherwise this emits an
+     * internal link with the text.
+     */
+    private static void emitType(PrintStream stream, Type type) {
+        emitStruct(stream, type.struct);
+        for (int i = 0; i < type.dimensions; i++) {
+            stream.print("[]");
+        }
+    }
+
+    /**
+     * Emit a {@link Struct}. If the {@linkplain Struct} is primitive or def this just emits the name of the struct. Otherwise this emits an
+     * internal link with the name.
+     */
+    private static void emitStruct(PrintStream stream, Struct struct) {
+        if (false == struct.clazz.isPrimitive() && false == struct.name.equals("def")) {
+            stream.print("<<");
+            emitAnchor(stream, struct);
+            stream.print(',');
+            stream.print(struct.name);
+            stream.print(">>");
+        } else {
+            stream.print(struct.name);
+        }
+    }
+
+    /**
+     * Emit an external link to Javadoc for a {@link Method}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, Method method) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print((method.augmentation ? Augmentation.class : method.owner.clazz).getName().replace('.', '/'));
+        stream.print(".html#");
+        stream.print(methodName(method));
+        stream.print("%2D");
+        boolean first = true;
+        if (method.augmentation) {
+            first = false;
+            stream.print(method.owner.clazz.getName());
+        }
+        for (Type arg: method.arguments) {
+            if (first) {
+                first = false;
+            } else {
+                stream.print("%2D");
+            }
+            stream.print(arg.struct.clazz.getName());
+            if (arg.dimensions > 0) {
+                stream.print(":A");
+            }
+        }
+        stream.print("%2D");
+    }
+
+    /**
+     * Emit an external link to Javadoc for a {@link Field}.
+     *
+     * @param root name of the root uri variable
+     */
+    private static void emitJavadocLink(PrintStream stream, String root, Field field) {
+        stream.print("link:{");
+        stream.print(root);
+        stream.print("-javadoc}/");
+        stream.print(field.owner.clazz.getName().replace('.', '/'));
+        stream.print(".html#");
+        stream.print(field.javaName);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Method}.
+     */
+    private static String javadocRoot(Method method) {
+        if (method.augmentation) {
+            return "painless";
+        }
+        return javadocRoot(method.owner);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Field}.
+     */
+    private static String javadocRoot(Field field) {
+        return javadocRoot(field.owner);
+    }
+
+    /**
+     * Pick the javadoc root for a {@link Struct}.
+     */
+    private static String javadocRoot(Struct struct) {
+        String classPackage = struct.clazz.getPackage().getName();
+        if (classPackage.startsWith("java")) {
+            return "java8";
+        }
+        if (classPackage.startsWith("org.elasticsearch.painless")) {
+            return "painless";
+        }
+        if (classPackage.startsWith("org.elasticsearch")) {
+            return "elasticsearch";
+        }
+        if (classPackage.startsWith("org.joda.time")) {
+            return "joda-time";
+        }
+        if (classPackage.startsWith("org.apache.lucene")) {
+            return "lucene-core";
+        }
+        throw new IllegalArgumentException("Unrecognized packge: " + classPackage);
+    }
+
+    private static void emitGeneratedWarning(PrintStream stream) {
+        stream.println("////");
+        stream.println("Automatically generated by PainlessDocGenerator. Do not edit.");
+        stream.println("Rebuild by running `gradle generatePainlessApi`.");
+        stream.println("////");
+        stream.println();
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index b5b3e2cfbf6..da4558a693a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -19,12 +19,13 @@
 
 package org.elasticsearch.painless;
 
-import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
-
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
+
 public class StringTests extends ScriptTestCase {
 
     public void testAppend() {
@@ -238,4 +239,14 @@ public class StringTests extends ScriptTestCase {
     public void testAppendStringIntoMap() {
         assertEquals("nullcat", exec("def a = new HashMap(); a.cat += 'cat'"));
     }
+
+    public void testBase64Augmentations() {
+        assertEquals("Y2F0", exec("'cat'.encodeBase64()"));
+        assertEquals("cat", exec("'Y2F0'.decodeBase64()"));
+        assertEquals("6KiA6Kqe", exec("'\u8A00\u8A9E'.encodeBase64()"));
+        assertEquals("\u8A00\u8A9E", exec("'6KiA6Kqe'.decodeBase64()"));
+
+        String rando = randomRealisticUnicodeOfLength(between(5, 1000));
+        assertEquals(rando, exec("params.rando.encodeBase64().decodeBase64()", singletonMap("rando", rando), true));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index f23b13341cc..aaa337ae821 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -148,10 +148,10 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
 
     public void testLoopLimits() {
         // right below limit: ok
-        exec("for (int x = 0; x < 9999; ++x) {}");
+        exec("for (int x = 0; x < 999999; ++x) {}");
 
         PainlessError expected = expectScriptThrows(PainlessError.class, () -> {
-            exec("for (int x = 0; x < 10000; ++x) {}");
+            exec("for (int x = 0; x < 1000000; ++x) {}");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
@@ -267,4 +267,20 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         assertEquals("invalid sequence of tokens near ['.'].", e.getMessage());
     }
 
+    public void testBadStringEscape() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'\\a'", false));
+        assertEquals("unexpected character ['\\a]. The only valid escape sequences in strings starting with ['] are [\\\\] and [\\'].",
+                e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("\"\\a\"", false));
+        assertEquals("unexpected character [\"\\a]. The only valid escape sequences in strings starting with [\"] are [\\\\] and [\\\"].",
+                e.getMessage());
+    }
+
+    public void testRegularUnexpectedCharacter() {
+        Exception e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'", false));
+        assertEquals("unexpected character ['].", e.getMessage());
+        e = expectScriptThrows(IllegalArgumentException.class, () -> exec("'cat", false));
+        assertEquals("unexpected character ['cat].", e.getMessage());
+    }
+
 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml
index 524f203b2fc..eb7cb01da33 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yaml
@@ -1,54 +1,47 @@
 ---
-"Indexed script":
+"Stored script":
+  - skip:
+      features: warnings
 
   - do:
       put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * doc['myParent.weight'].value" }
+        lang: "1"
+        body: { "script": {"lang": "painless", "code": "_score * doc['myParent.weight'].value" } }
   - match: { acknowledged: true }
 
   - do:
      get_script:
-       id: "1"
-       lang: "painless"
+       lang: "1"
   - match: { found: true }
-  - match: { lang: painless }
   - match: { _id: "1" }
-  - match: { "script":  "_score * doc['myParent.weight'].value" }
+  - match: { "script": {"lang": "painless", "code": "_score * doc['myParent.weight'].value"} }
 
   - do:
      catch: missing
      get_script:
-       id: "2"
-       lang: "painless"
+       lang: "2"
   - match: { found: false }
-  - match: { lang: painless }
   - match: { _id: "2" }
   - is_false: script
 
   - do:
      delete_script:
-       id: "1"
-       lang: "painless"
+       lang: "1"
   - match: { acknowledged: true }
 
   - do:
      catch: missing
      delete_script:
-       id: "non_existing"
-       lang: "painless"
+       lang: "non_existing"
 
   - do:
       catch: request
       put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
+        lang: "1"
+        body: { "script": {"lang": "painless", "code": "_score * foo bar + doc['myParent.weight'].value"} }
 
   - do:
       catch: /compile error/
       put_script:
-        id: "1"
-        lang: "painless"
-        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
+        lang: "1"
+        body: { "script": {"lang": "painless", "code": "_score * foo bar + doc['myParent.weight'].value"} }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml
index f626a77bf93..2702d8f3095 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yaml
@@ -12,12 +12,22 @@ setup:
                   type: keyword
                 missing:
                   type: keyword
+                date:
+                  type: date
+                  format: yyyy/MM/dd
+                dates:
+                  type: date
+                  format: yyyy/MM/dd
     - do:
         index:
             index: test
             type: test
             id: 1
-            body: { "foo": "aaa" }
+            body: {
+              "foo": "aaa",
+              "date": "2017/01/01",
+              "dates": ["2017/01/01", "2017/02/01", "2017/03/01"]
+            }
     - do:
         indices.refresh: {}
 
@@ -34,6 +44,7 @@ setup:
                                 x: "bbb"
 
     - match: { hits.hits.0.fields.bar.0: "aaabbb"}
+
 ---
 "Scripted Field Doing Compare":
     - do:
@@ -60,6 +71,7 @@ setup:
                                 x: "bbb"
 
     - match: { hits.hits.0.fields.bar.0: false}
+
 ---
 "Scripted Field with a null safe dereference (non-null)":
     - do:
@@ -89,6 +101,35 @@ setup:
 
     - match: { hits.hits.0.fields.bar.0: 5}
 
+---
+"Access a date":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            inline: "doc.date.date.dayOfWeek"
+
+    - match: { hits.hits.0.fields.bar.0: 7}
+
+---
+"Access many dates":
+    - do:
+        search:
+            body:
+                script_fields:
+                    bar:
+                        script:
+                            inline: >
+                              StringBuilder b = new StringBuilder();
+                              for (def date : doc.dates.dates) {
+                                b.append(" ").append(date.getDayOfWeek());
+                              }
+                              return b.toString().trim()
+
+    - match: { hits.hits.0.fields.bar.0: "7 3 3"}
+
 ---
 "Scripted Field with script error":
     - do:
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml
new file mode 100644
index 00000000000..30cc9f7f479
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yaml
@@ -0,0 +1,368 @@
+setup:
+    - do:
+        indices.create:
+            index: test
+            body:
+                mappings:
+                    test:
+                        properties:
+                            binary:
+                                type: binary
+                                doc_values: true
+                            boolean:
+                                type: boolean
+                            date:
+                                type: date
+                            geo_point:
+                                type: geo_point
+                            ip:
+                                type: ip
+                            keyword:
+                                type: keyword
+                            long:
+                                type: long
+                            integer:
+                                type: integer
+                            short:
+                                type: short
+                            byte:
+                                type: byte
+                            double:
+                                type: double
+                            float:
+                                type: float
+                            half_float:
+                                type: half_float
+                            scaled_float:
+                                type: scaled_float
+                                scaling_factor: 100
+                            token_count:
+                                type: token_count
+                                analyzer: standard
+
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body:
+                binary: U29tZSBiaW5hcnkgYmxvYg==
+                boolean: true
+                date: 2017-01-01T12:11:12
+                geo_point: 41.12,-71.34
+                ip: 192.168.0.1
+                keyword: not split at all
+                long: 12348732141234
+                integer: 134134566
+                short: 1324
+                byte: 12
+                double: 3.14159265358979
+                float: 3.141592654
+                half_float: 3.140625
+                scaled_float: 3.14
+                token_count: count all these words please
+
+    - do:
+        indices.refresh: {}
+
+---
+"binary":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['binary'].get(0).utf8ToString()"
+    - match: { hits.hits.0.fields.field.0: "Some binary blob" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['binary'].value.utf8ToString()"
+    - match: { hits.hits.0.fields.field.0: "Some binary blob" }
+
+---
+"boolean":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['boolean'].get(0)"
+    - match: { hits.hits.0.fields.field.0: true }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['boolean'].value"
+    - match: { hits.hits.0.fields.field.0: true }
+
+---
+"date":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['date'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['date'].value"
+    - match: { hits.hits.0.fields.field.0: 1483272672000 }
+
+---
+"geo_point":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['geo_point'].get(0)"
+    - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 }
+    - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['geo_point'].value"
+    - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 }
+    - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 }
+
+---
+"ip":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['ip'].get(0)"
+    - match: { hits.hits.0.fields.field.0: "192.168.0.1" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['ip'].value"
+    - match: { hits.hits.0.fields.field.0: "192.168.0.1" }
+
+---
+"keyword":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['keyword'].get(0)"
+    - match: { hits.hits.0.fields.field.0: "not split at all" }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['keyword'].value"
+    - match: { hits.hits.0.fields.field.0: "not split at all" }
+
+---
+"long":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['long'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 12348732141234 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['long'].value"
+    - match: { hits.hits.0.fields.field.0: 12348732141234 }
+
+---
+"integer":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['integer'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 134134566 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['integer'].value"
+    - match: { hits.hits.0.fields.field.0: 134134566 }
+
+---
+"short":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['short'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 1324 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['short'].value"
+    - match: { hits.hits.0.fields.field.0: 1324 }
+
+---
+"byte":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['byte'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 12 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['byte'].value"
+    - match: { hits.hits.0.fields.field.0: 12 }
+
+---
+"double":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['double'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.14159265358979 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['double'].value"
+    - match: { hits.hits.0.fields.field.0: 3.14159265358979 }
+
+---
+"float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.1415927410125732 } # this ends up as a double
+
+---
+"half_float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['half_float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.140625 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['half_float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.140625 }
+
+---
+"scaled_float":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['scaled_float'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 3.14 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['scaled_float'].value"
+    - match: { hits.hits.0.fields.field.0: 3.14 }
+
+---
+"token_count":
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['token_count'].get(0)"
+    - match: { hits.hits.0.fields.field.0: 5 }
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    field:
+                        script:
+                            inline: "doc['token_count'].value"
+    - match: { hits.hits.0.fields.field.0: 5 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
deleted file mode 100644
index d6eb5660728..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.client.ElasticsearchClient;
-
-@Deprecated
-public class MultiPercolateAction extends Action<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
-
-    public static final MultiPercolateAction INSTANCE = new MultiPercolateAction();
-    public static final String NAME = "indices:data/read/mpercolate";
-
-    private MultiPercolateAction() {
-        super(NAME);
-    }
-
-    @Override
-    public MultiPercolateResponse newResponse() {
-        return new MultiPercolateResponse();
-    }
-
-    @Override
-    public MultiPercolateRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new MultiPercolateRequestBuilder(client, this);
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
deleted file mode 100644
index c95b767d42f..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.CompositeIndicesRequest;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
-
-/**
- * A multi percolate request that encapsulates multiple {@link PercolateRequest} instances in a single api call.
- *
- * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class MultiPercolateRequest extends ActionRequest implements CompositeIndicesRequest {
-
-    private String[] indices;
-    private String documentType;
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
-    private List<PercolateRequest> requests = new ArrayList<>();
-
-    /**
-     * Embeds a percolate request to this multi percolate request
-     */
-    public MultiPercolateRequest add(PercolateRequestBuilder requestBuilder) {
-        return add(requestBuilder.request());
-    }
-
-    /**
-     * Embeds a percolate request to this multi percolate request
-     */
-    public MultiPercolateRequest add(PercolateRequest request) {
-        if (request.indices() == null && indices != null) {
-            request.indices(indices);
-        }
-        if (request.documentType() == null && documentType != null) {
-            request.documentType(documentType);
-        }
-        if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && indicesOptions != IndicesOptions.strictExpandOpenAndForbidClosed()) {
-            request.indicesOptions(indicesOptions);
-        }
-        requests.add(request);
-        return this;
-    }
-
-    /**
-     * Embeds a percolate request which request body is defined as raw bytes to this multi percolate request
-     */
-    public MultiPercolateRequest add(byte[] data, int from, int length) throws Exception {
-        return add(new BytesArray(data, from, length), true);
-    }
-
-    /**
-     * Embeds a percolate request which request body is defined as raw bytes to this multi percolate request
-     */
-    public MultiPercolateRequest add(BytesReference data, boolean allowExplicitIndex) throws IOException {
-        XContent xContent = XContentFactory.xContent(data);
-        int from = 0;
-        int length = data.length();
-        byte marker = xContent.streamSeparator();
-        while (true) {
-            int nextMarker = findNextMarker(marker, from, data, length);
-            if (nextMarker == -1) {
-                break;
-            }
-            // support first line with \n
-            if (nextMarker == 0) {
-                from = nextMarker + 1;
-                continue;
-            }
-
-            PercolateRequest percolateRequest = new PercolateRequest();
-            if (indices != null) {
-                percolateRequest.indices(indices);
-            }
-            if (documentType != null) {
-                percolateRequest.documentType(documentType);
-            }
-            if (indicesOptions != IndicesOptions.strictExpandOpenAndForbidClosed()) {
-                percolateRequest.indicesOptions(indicesOptions);
-            }
-
-            // now parse the action
-            if (nextMarker - from > 0) {
-                try (XContentParser parser = xContent.createParser(data.slice(from, nextMarker - from))) {
-                    // Move to START_OBJECT, if token is null, its an empty data
-                    XContentParser.Token token = parser.nextToken();
-                    if (token != null) {
-                        // Top level json object
-                        assert token == XContentParser.Token.START_OBJECT;
-                        token = parser.nextToken();
-                        if (token != XContentParser.Token.FIELD_NAME) {
-                            throw new ElasticsearchParseException("Expected field");
-                        }
-                        token = parser.nextToken();
-                        if (token != XContentParser.Token.START_OBJECT) {
-                            throw new ElasticsearchParseException("expected start object");
-                        }
-                        String percolateAction = parser.currentName();
-                        if ("percolate".equals(percolateAction)) {
-                            parsePercolateAction(parser, percolateRequest, allowExplicitIndex);
-                        } else if ("count".equals(percolateAction)) {
-                            percolateRequest.onlyCount(true);
-                            parsePercolateAction(parser, percolateRequest, allowExplicitIndex);
-                        } else {
-                            throw new ElasticsearchParseException("[{}] isn't a supported percolate operation", percolateAction);
-                        }
-                    }
-                }
-            }
-
-            // move pointers
-            from = nextMarker + 1;
-
-            // now for the body
-            nextMarker = findNextMarker(marker, from, data, length);
-            if (nextMarker == -1) {
-                break;
-            }
-
-            percolateRequest.source(data.slice(from, nextMarker - from));
-            // move pointers
-            from = nextMarker + 1;
-
-            add(percolateRequest);
-        }
-
-        return this;
-    }
-
-    private void parsePercolateAction(XContentParser parser, PercolateRequest percolateRequest, boolean allowExplicitIndex) throws IOException {
-        String globalIndex = indices != null && indices.length > 0 ? indices[0] : null;
-
-        Map<String, Object> header = parser.map();
-
-        if (header.containsKey("id")) {
-            GetRequest getRequest = new GetRequest(globalIndex);
-            percolateRequest.getRequest(getRequest);
-            for (Map.Entry<String, Object> entry : header.entrySet()) {
-                Object value = entry.getValue();
-                if ("id".equals(entry.getKey())) {
-                    getRequest.id(nodeStringValue(value, null));
-                    header.put("id", entry.getValue());
-                } else if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) {
-                    if (!allowExplicitIndex) {
-                        throw new IllegalArgumentException("explicit index in multi percolate is not allowed");
-                    }
-                    getRequest.index(nodeStringValue(value, null));
-                } else if ("type".equals(entry.getKey())) {
-                    getRequest.type(nodeStringValue(value, null));
-                } else if ("preference".equals(entry.getKey())) {
-                    getRequest.preference(nodeStringValue(value, null));
-                } else if ("routing".equals(entry.getKey())) {
-                    getRequest.routing(nodeStringValue(value, null));
-                } else if ("percolate_index".equals(entry.getKey()) || "percolate_indices".equals(entry.getKey()) || "percolateIndex".equals(entry.getKey()) || "percolateIndices".equals(entry.getKey())) {
-                    percolateRequest.indices(nodeStringArrayValue(value));
-                } else if ("percolate_type".equals(entry.getKey()) || "percolateType".equals(entry.getKey())) {
-                    percolateRequest.documentType(nodeStringValue(value, null));
-                } else if ("percolate_preference".equals(entry.getKey()) || "percolatePreference".equals(entry.getKey())) {
-                    percolateRequest.preference(nodeStringValue(value, null));
-                } else if ("percolate_routing".equals(entry.getKey()) || "percolateRouting".equals(entry.getKey())) {
-                    percolateRequest.routing(nodeStringValue(value, null));
-                }
-            }
-
-            // Setting values based on get request, if needed...
-            if ((percolateRequest.indices() == null || percolateRequest.indices().length == 0) && getRequest.index() != null) {
-                percolateRequest.indices(getRequest.index());
-            }
-            if (percolateRequest.documentType() == null && getRequest.type() != null) {
-                percolateRequest.documentType(getRequest.type());
-            }
-            if (percolateRequest.routing() == null && getRequest.routing() != null) {
-                percolateRequest.routing(getRequest.routing());
-            }
-            if (percolateRequest.preference() == null && getRequest.preference() != null) {
-                percolateRequest.preference(getRequest.preference());
-            }
-        } else {
-            for (Map.Entry<String, Object> entry : header.entrySet()) {
-                Object value = entry.getValue();
-                if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) {
-                    if (!allowExplicitIndex) {
-                        throw new IllegalArgumentException("explicit index in multi percolate is not allowed");
-                    }
-                    percolateRequest.indices(nodeStringArrayValue(value));
-                } else if ("type".equals(entry.getKey())) {
-                    percolateRequest.documentType(nodeStringValue(value, null));
-                } else if ("preference".equals(entry.getKey())) {
-                    percolateRequest.preference(nodeStringValue(value, null));
-                } else if ("routing".equals(entry.getKey())) {
-                    percolateRequest.routing(nodeStringValue(value, null));
-                }
-            }
-        }
-        percolateRequest.indicesOptions(IndicesOptions.fromMap(header, indicesOptions));
-    }
-
-    private int findNextMarker(byte marker, int from, BytesReference data, int length) {
-        for (int i = from; i < length; i++) {
-            if (data.get(i) == marker) {
-                return i;
-            }
-        }
-        return -1;
-    }
-
-    /**
-     * @return The list of already set percolate requests.
-     */
-    public List<PercolateRequest> requests() {
-        return this.requests;
-    }
-
-    /**
-     * @return Returns the {@link IndicesOptions} that is used as default for all percolate requests.
-     */
-    public IndicesOptions indicesOptions() {
-        return indicesOptions;
-    }
-
-    /**
-     * Sets the {@link IndicesOptions} for all percolate request that don't have this set.
-     *
-     * Warning: This should be set before adding any percolate requests. Setting this after adding percolate requests
-     * will have no effect on any percolate requests already added.
-     */
-    public MultiPercolateRequest indicesOptions(IndicesOptions indicesOptions) {
-        this.indicesOptions = indicesOptions;
-        return this;
-    }
-
-    /**
-     * @return The default indices for all percolate request.
-     */
-    public String[] indices() {
-        return indices;
-    }
-
-    /**
-     * Sets the default indices for any percolate request that doesn't have indices defined.
-     *
-     * Warning: This should be set before adding any percolate requests. Setting this after adding percolate requests
-     * will have no effect on any percolate requests already added.
-     */
-    public MultiPercolateRequest indices(String... indices) {
-        this.indices = indices;
-        return this;
-    }
-
-    /**
-     * @return Sets the default type for all percolate requests
-     */
-    public String documentType() {
-        return documentType;
-    }
-
-    /**
-     * Sets the default document type for any percolate request that doesn't have a document type set.
-     *
-     * Warning: This should be set before adding any percolate requests. Setting this after adding percolate requests
-     * will have no effect on any percolate requests already added.
-     */
-    public MultiPercolateRequest documentType(String type) {
-        this.documentType = type;
-        return this;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (requests.isEmpty()) {
-            validationException = addValidationError("no requests added", validationException);
-        }
-        for (int i = 0; i < requests.size(); i++) {
-            ActionRequestValidationException ex = requests.get(i).validate();
-            if (ex != null) {
-                if (validationException == null) {
-                    validationException = new ActionRequestValidationException();
-                }
-                validationException.addValidationErrors(ex.validationErrors());
-            }
-        }
-
-        return validationException;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        indices = in.readStringArray();
-        documentType = in.readOptionalString();
-        indicesOptions = IndicesOptions.readIndicesOptions(in);
-        int size = in.readVInt();
-        for (int i = 0; i < size; i++) {
-            PercolateRequest request = new PercolateRequest();
-            request.readFrom(in);
-            requests.add(request);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArrayNullable(indices);
-        out.writeOptionalString(documentType);
-        indicesOptions.writeIndicesOptions(out);
-        out.writeVInt(requests.size());
-        for (PercolateRequest request : requests) {
-            request.writeTo(out);
-        }
-    }
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java
deleted file mode 100644
index 8613b8b07bd..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.ElasticsearchClient;
-
-/**
- * A builder for to ease the use of defining a {@link MultiPercolateRequest} instance.
- *
- * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class MultiPercolateRequestBuilder extends ActionRequestBuilder<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
-
-    public MultiPercolateRequestBuilder(ElasticsearchClient client, MultiPercolateAction action) {
-        super(client, action, new MultiPercolateRequest());
-    }
-
-    /**
-     * Bundles the specified percolate request to the multi percolate request.
-     */
-    public MultiPercolateRequestBuilder add(PercolateRequest percolateRequest) {
-        request.add(percolateRequest);
-        return this;
-    }
-
-    /**
-     * Bundles the specified percolate request build to the multi percolate request.
-     */
-    public MultiPercolateRequestBuilder add(PercolateRequestBuilder percolateRequestBuilder) {
-        request.add(percolateRequestBuilder);
-        return this;
-    }
-
-    /**
-     * Specifies how to globally ignore indices that are not available and how to deal with wildcard indices expressions.
-     * <p>
-     * Invoke this method before invoking {@link #add(PercolateRequestBuilder)}.
-     */
-    public MultiPercolateRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
-        request.indicesOptions(indicesOptions);
-        return this;
-    }
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
deleted file mode 100644
index afc5b7ab6c7..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Iterator;
-
-/**
- * Represents the response of a multi percolate request.
- *
- * Each item represents the response of a percolator request and the order of the items is in the same order as the
- * percolator requests were defined in the multi percolate request.
- *
- * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class MultiPercolateResponse extends ActionResponse implements Iterable<MultiPercolateResponse.Item>, ToXContent {
-
-    private Item[] items;
-
-    MultiPercolateResponse(Item[] items) {
-        this.items = items;
-    }
-
-    MultiPercolateResponse() {
-        this.items = new Item[0];
-    }
-
-    @Override
-    public Iterator<Item> iterator() {
-        return Arrays.stream(items).iterator();
-    }
-
-    /**
-     * Same as {@link #getItems()}
-     */
-    public Item[] items() {
-        return items;
-    }
-
-    /**
-     * @return the percolate responses as items.
-     */
-    public Item[] getItems() {
-        return items;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(Fields.RESPONSES);
-        for (MultiPercolateResponse.Item item : items) {
-            builder.startObject();
-            if (item.isFailure()) {
-                ElasticsearchException.renderException(builder, params, item.getFailure());
-            } else {
-                item.getResponse().toXContent(builder, params);
-            }
-            builder.endObject();
-        }
-        builder.endArray();
-        return builder;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(items.length);
-        for (Item item : items) {
-            item.writeTo(out);
-        }
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        int size = in.readVInt();
-        items = new Item[size];
-        for (int i = 0; i < items.length; i++) {
-            items[i] = new Item();
-            items[i].readFrom(in);
-        }
-    }
-
-    /**
-     * Encapsulates a single percolator response which may contain an error or the actual percolator response itself.
-     */
-    public static class Item implements Streamable {
-
-        private PercolateResponse response;
-        private Exception exception;
-
-        Item(PercolateResponse response) {
-            this.response = response;
-        }
-
-        Item(Exception exception) {
-            this.exception = exception;
-        }
-
-        Item() {
-        }
-
-
-        /**
-         * @return The percolator response or <code>null</code> if there was error.
-         */
-        @Nullable
-        public PercolateResponse getResponse() {
-            return response;
-        }
-
-        /**
-         * @return An error description if there was an error or <code>null</code> if the percolate request was successful
-         */
-        @Nullable
-        public String getErrorMessage() {
-            return exception == null ? null : exception.getMessage();
-        }
-
-        /**
-         * @return <code>true</code> if the percolator request that this item represents failed otherwise
-         * <code>false</code> is returned.
-         */
-        public boolean isFailure() {
-            return exception != null;
-        }
-
-        public Exception getFailure() {
-            return exception;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            if (in.readBoolean()) {
-                response = new PercolateResponse();
-                response.readFrom(in);
-            } else {
-                exception = in.readException();
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            if (response != null) {
-                out.writeBoolean(true);
-                response.writeTo(out);
-            } else {
-                out.writeBoolean(false);
-                out.writeException(exception);
-            }
-        }
-    }
-
-    static final class Fields {
-        static final String RESPONSES = "responses";
-        static final String ERROR = "error";
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
deleted file mode 100644
index cebca9ed825..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.client.ElasticsearchClient;
-
-@Deprecated
-public class PercolateAction extends Action<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
-
-    public static final PercolateAction INSTANCE = new PercolateAction();
-    public static final String NAME = "indices:data/read/percolate";
-
-    private PercolateAction() {
-        super(NAME);
-    }
-
-    @Override
-    public PercolateResponse newResponse() {
-        return new PercolateResponse();
-    }
-
-    @Override
-    public PercolateRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new PercolateRequestBuilder(client, this);
-    }
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 40218e50a4f..f5978a2dbd1 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
 
@@ -90,8 +91,8 @@ final class PercolateQuery extends Query implements Accountable {
                     if (result == docId) {
                         if (twoPhaseIterator.matches()) {
                             if (needsScores) {
-                                QueryStore.Leaf percolatorQueries = queryStore.getQueries(leafReaderContext);
-                                Query query = percolatorQueries.getQuery(docId);
+                                CheckedFunction<Integer, Query, IOException> percolatorQueries = queryStore.getQueries(leafReaderContext);
+                                Query query = percolatorQueries.apply(docId);
                                 Explanation detail = percolatorIndexSearcher.explain(query, 0);
                                 return Explanation.match(scorer.score(), "PercolateQuery", detail);
                             } else {
@@ -120,7 +121,7 @@ final class PercolateQuery extends Query implements Accountable {
                     return null;
                 }
 
-                final QueryStore.Leaf queries = queryStore.getQueries(leafReaderContext);
+                final CheckedFunction<Integer, Query, IOException> queries = queryStore.getQueries(leafReaderContext);
                 if (needsScores) {
                     return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
@@ -128,7 +129,7 @@ final class PercolateQuery extends Query implements Accountable {
 
                         @Override
                         boolean matchDocId(int docId) throws IOException {
-                            Query query = percolatorQueries.getQuery(docId);
+                            Query query = percolatorQueries.apply(docId);
                             if (query != null) {
                                 TopDocs topDocs = percolatorIndexSearcher.search(query, 1);
                                 if (topDocs.totalHits > 0) {
@@ -166,7 +167,7 @@ final class PercolateQuery extends Query implements Accountable {
                             if (verifiedDocsBits.get(docId)) {
                                 return true;
                             }
-                            Query query = percolatorQueries.getQuery(docId);
+                            Query query = percolatorQueries.apply(docId);
                             return query != null && Lucene.exists(percolatorIndexSearcher, query);
                         }
                     };
@@ -224,26 +225,18 @@ final class PercolateQuery extends Query implements Accountable {
     }
 
     @FunctionalInterface
-    public interface QueryStore {
-
-        Leaf getQueries(LeafReaderContext ctx) throws IOException;
-
-        @FunctionalInterface
-        interface Leaf {
-
-            Query getQuery(int docId) throws IOException;
-
-        }
-
+    interface QueryStore {
+        CheckedFunction<Integer, Query, IOException> getQueries(LeafReaderContext ctx) throws IOException;
     }
 
     abstract static class BaseScorer extends Scorer {
 
         final Scorer approximation;
-        final QueryStore.Leaf percolatorQueries;
+        final CheckedFunction<Integer, Query, IOException> percolatorQueries;
         final IndexSearcher percolatorIndexSearcher;
 
-        BaseScorer(Weight weight, Scorer approximation, QueryStore.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) {
+        BaseScorer(Weight weight, Scorer approximation, CheckedFunction<Integer, Query, IOException> percolatorQueries,
+                   IndexSearcher percolatorIndexSearcher) {
             super(weight);
             this.approximation = approximation;
             this.percolatorQueries = percolatorQueries;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 24243081f23..97a65a29ada 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -95,6 +95,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     private final String field;
     private final String documentType;
     private final BytesReference document;
+    private final XContentType documentXContentType;
 
     private final String indexedDocumentIndex;
     private final String indexedDocumentType;
@@ -103,7 +104,16 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     private final String indexedDocumentPreference;
     private final Long indexedDocumentVersion;
 
+    /**
+     * @deprecated use {@link #PercolateQueryBuilder(String, String, BytesReference, XContentType)} with the document content type to avoid
+     * autodetection
+     */
+    @Deprecated
     public PercolateQueryBuilder(String field, String documentType, BytesReference document) {
+        this(field, documentType, document, XContentFactory.xContentType(document));
+    }
+
+    public PercolateQueryBuilder(String field, String documentType, BytesReference document, XContentType documentXContentType) {
         if (field == null) {
             throw new IllegalArgumentException("[field] is a required argument");
         }
@@ -116,6 +126,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         this.field = field;
         this.documentType = documentType;
         this.document = document;
+        this.documentXContentType = Objects.requireNonNull(documentXContentType);
         indexedDocumentIndex = null;
         indexedDocumentType = null;
         indexedDocumentId = null;
@@ -151,6 +162,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         this.indexedDocumentPreference = indexedDocumentPreference;
         this.indexedDocumentVersion = indexedDocumentVersion;
         this.document = null;
+        this.documentXContentType = null;
     }
 
     /**
@@ -171,6 +183,15 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             indexedDocumentVersion = null;
         }
         document = in.readOptionalBytesReference();
+        if (document != null) {
+            if (in.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+                documentXContentType = XContentType.readFrom(in);
+            } else {
+                documentXContentType = XContentFactory.xContentType(document);
+            }
+        } else {
+            documentXContentType = null;
+        }
     }
 
     @Override
@@ -189,6 +210,9 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             out.writeBoolean(false);
         }
         out.writeOptionalBytesReference(document);
+        if (document != null && out.getVersion().after(Version.V_5_3_0_UNRELEASED)) { // TODO update to onOrAfter after backporting
+            documentXContentType.writeTo(out);
+        }
     }
 
     @Override
@@ -197,16 +221,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         builder.field(DOCUMENT_TYPE_FIELD.getPreferredName(), documentType);
         builder.field(QUERY_FIELD.getPreferredName(), field);
         if (document != null) {
-            XContentType contentType = XContentFactory.xContentType(document);
-            if (contentType == builder.contentType()) {
-                builder.rawField(DOCUMENT_FIELD.getPreferredName(), document);
-            } else {
-                try (XContentParser parser = XContentFactory.xContent(contentType).createParser(document)) {
-                    parser.nextToken();
-                    builder.field(DOCUMENT_FIELD.getPreferredName());
-                    builder.copyCurrentStructure(parser);
-                }
-            }
+            builder.rawField(DOCUMENT_FIELD.getPreferredName(), document);
         }
         if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
             if (indexedDocumentIndex != null) {
@@ -256,7 +271,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, DOCUMENT_FIELD)) {
+                if (DOCUMENT_FIELD.match(currentFieldName)) {
                     try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
                         builder.copyCurrentStructure(parser);
                         builder.flush();
@@ -267,25 +282,25 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                             "] query does not support [" + token + "]");
                 }
             } else if (token.isValue()) {
-                if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
+                if (QUERY_FIELD.match(currentFieldName)) {
                     field = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, DOCUMENT_TYPE_FIELD)) {
+                } else if (DOCUMENT_TYPE_FIELD.match(currentFieldName)) {
                     documentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_INDEX)) {
+                } else if (INDEXED_DOCUMENT_FIELD_INDEX.match(currentFieldName)) {
                     indexedDocumentIndex = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_TYPE)) {
+                } else if (INDEXED_DOCUMENT_FIELD_TYPE.match(currentFieldName)) {
                     indexedDocumentType = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ID)) {
+                } else if (INDEXED_DOCUMENT_FIELD_ID.match(currentFieldName)) {
                     indexedDocumentId = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ROUTING)) {
+                } else if (INDEXED_DOCUMENT_FIELD_ROUTING.match(currentFieldName)) {
                     indexedDocumentRouting = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_PREFERENCE)) {
+                } else if (INDEXED_DOCUMENT_FIELD_PREFERENCE.match(currentFieldName)) {
                     indexedDocumentPreference = parser.text();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_VERSION)) {
+                } else if (INDEXED_DOCUMENT_FIELD_VERSION.match(currentFieldName)) {
                     indexedDocumentVersion = parser.longValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+                } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName)) {
                     boost = parser.floatValue();
-                } else if (parseContext.getParseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+                } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName)) {
                     queryName = parser.text();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[" + PercolateQueryBuilder.NAME +
@@ -304,7 +319,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
 
         PercolateQueryBuilder queryBuilder;
         if (source != null) {
-            queryBuilder = new PercolateQueryBuilder(field, documentType, source);
+            queryBuilder = new PercolateQueryBuilder(field, documentType, source, XContentType.JSON);
         } else if (indexedDocumentId != null) {
             queryBuilder = new PercolateQueryBuilder(field, documentType, indexedDocumentIndex, indexedDocumentType,
                     indexedDocumentId, indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
@@ -360,7 +375,8 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                 "indexed document [" + indexedDocumentIndex + "/" + indexedDocumentType + "/" + indexedDocumentId + "] source disabled"
             );
         }
-        return new PercolateQueryBuilder(field, documentType, getResponse.getSourceAsBytesRef());
+        final BytesReference source = getResponse.getSourceAsBytesRef();
+        return new PercolateQueryBuilder(field, documentType, source, XContentFactory.xContentType(source));
     }
 
     @Override
@@ -380,7 +396,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType);
         DocumentMapper docMapper = docMapperForType.getDocumentMapper();
 
-        ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document));
+        ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document, documentXContentType));
 
         FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer();
         // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when
@@ -445,6 +461,11 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         return document;
     }
 
+    //pkg-private for testing
+    XContentType getXContentType() {
+        return documentXContentType;
+    }
+
     static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
         RAMDirectory ramDirectory = new RAMDirectory();
         try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) {
@@ -486,7 +507,8 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                     BytesRef qbSource = binaryDocValues.get(docId);
                     if (qbSource.length > 0) {
                         XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent();
-                        try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) {
+                        try (XContentParser sourceParser = xContent.createParser(context.getXContentRegistry(), qbSource.bytes,
+                                qbSource.offset, qbSource.length)) {
                             return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
                         }
                     } else {
@@ -509,7 +531,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                     throw new IllegalStateException("No source found for document with docid [" + docId + "]");
                 }
 
-                try (XContentParser sourceParser = XContentHelper.createParser(visitor.source)) {
+                try (XContentParser sourceParser = XContentHelper.createParser(context.getXContentRegistry(), visitor.source)) {
                     String currentFieldName = null;
                     XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
                     if (token != XContentParser.Token.START_OBJECT) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
deleted file mode 100644
index bc449ea932d..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ElasticsearchGenerationException;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-import java.util.Map;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-
-/**
- * A request to execute a percolate operation.
- *
- * @deprecated Instead use search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class PercolateRequest extends ActionRequest implements IndicesRequest.Replaceable {
-
-    protected String[] indices;
-    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
-    private String documentType;
-    private String routing;
-    private String preference;
-    private boolean onlyCount;
-
-    private GetRequest getRequest;
-    private BytesReference source;
-
-    public String[] indices() {
-        return indices;
-    }
-
-    public final PercolateRequest indices(String... indices) {
-        this.indices = indices;
-        return this;
-    }
-
-    public IndicesOptions indicesOptions() {
-        return indicesOptions;
-    }
-
-    public PercolateRequest indicesOptions(IndicesOptions indicesOptions) {
-        this.indicesOptions = indicesOptions;
-        return this;
-    }
-
-
-    /**
-     * Getter for {@link #documentType(String)}
-     */
-    public String documentType() {
-        return documentType;
-    }
-
-    /**
-     * Sets the type of the document to percolate. This is important as it selects the mapping to be used to parse
-     * the document.
-     */
-    public PercolateRequest documentType(String type) {
-        this.documentType = type;
-        return this;
-    }
-
-    /**
-     * Getter for {@link #routing(String)}
-     */
-    public String routing() {
-        return routing;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public PercolateRequest routing(String routing) {
-        this.routing = routing;
-        return this;
-    }
-
-    /**
-     * Getter for {@link #preference(String)}
-     */
-    public String preference() {
-        return preference;
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
-     */
-    public PercolateRequest preference(String preference) {
-        this.preference = preference;
-        return this;
-    }
-
-    /**
-     * Getter for {@link #getRequest(GetRequest)}
-     */
-    public GetRequest getRequest() {
-        return getRequest;
-    }
-
-    /**
-     * This defines where to fetch the document to be percolated from, which is an alternative of defining the document
-     * to percolate in the request body.
-     *
-     * If this defined than this will override the document specified in the request body.
-     */
-    public PercolateRequest getRequest(GetRequest getRequest) {
-        this.getRequest = getRequest;
-        return this;
-    }
-
-    /**
-     * @return The request body in its raw form.
-     */
-    public BytesReference source() {
-        return source;
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(Map document) throws ElasticsearchGenerationException {
-        return source(document, Requests.CONTENT_TYPE);
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    @SuppressWarnings("unchecked")
-    public PercolateRequest source(Map document, XContentType contentType) throws ElasticsearchGenerationException {
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-            builder.map(document);
-            return source(builder);
-        } catch (IOException e) {
-            throw new ElasticsearchGenerationException("Failed to generate [" + document + "]", e);
-        }
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(String document) {
-        this.source = new BytesArray(document);
-        return this;
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(XContentBuilder documentBuilder) {
-        source = documentBuilder.bytes();
-        return this;
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(byte[] document) {
-        return source(document, 0, document.length);
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(byte[] source, int offset, int length) {
-        return source(new BytesArray(source, offset, length));
-    }
-
-    /**
-     * Raw version of {@link #source(PercolateSourceBuilder)}
-     */
-    public PercolateRequest source(BytesReference source) {
-        this.source = source;
-        return this;
-    }
-
-    /**
-     * Sets the request body definition for this percolate request as raw bytes.
-     *
-     * This is the preferred way to set the request body.
-     */
-    public PercolateRequest source(PercolateSourceBuilder sourceBuilder) {
-        this.source = sourceBuilder.buildAsBytes(Requests.CONTENT_TYPE);
-        return this;
-    }
-
-    /**
-     * Getter for {@link #onlyCount(boolean)}
-     */
-    public boolean onlyCount() {
-        return onlyCount;
-    }
-
-    /**
-     * Sets whether this percolate request should only count the number of percolator queries that matches with
-     * the document being percolated and don't keep track of the actual queries that have matched.
-     */
-    public PercolateRequest onlyCount(boolean onlyCount) {
-        this.onlyCount = onlyCount;
-        return this;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (documentType == null) {
-            validationException = addValidationError("type is missing", validationException);
-        }
-        if (source == null && getRequest == null) {
-            validationException = addValidationError("source or get is missing", validationException);
-        }
-        if (getRequest != null && getRequest.storedFields() != null) {
-            validationException = addValidationError("get stored fields option isn't supported via percolate request", validationException);
-        }
-        return validationException;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        indices = in.readStringArray();
-        indicesOptions = IndicesOptions.readIndicesOptions(in);
-        documentType = in.readString();
-        routing = in.readOptionalString();
-        preference = in.readOptionalString();
-        source = in.readBytesReference();
-        if (in.readBoolean()) {
-            getRequest = new GetRequest();
-            getRequest.readFrom(in);
-        }
-        onlyCount = in.readBoolean();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArrayNullable(indices);
-        indicesOptions.writeIndicesOptions(out);
-        out.writeString(documentType);
-        out.writeOptionalString(routing);
-        out.writeOptionalString(preference);
-        out.writeBytesReference(source);
-        if (getRequest != null) {
-            out.writeBoolean(true);
-            getRequest.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
-        out.writeBoolean(onlyCount);
-    }
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
deleted file mode 100644
index e73a3267fee..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.sort.SortBuilder;
-
-import java.util.Map;
-
-/**
- * A builder the easy to use of defining a percolate request.
- *
- * @deprecated Instead use search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
-
-    private PercolateSourceBuilder sourceBuilder;
-
-    public PercolateRequestBuilder(ElasticsearchClient client, PercolateAction action) {
-        super(client, action, new PercolateRequest());
-    }
-
-    public PercolateRequestBuilder setIndices(String... indices) {
-        request.indices(indices);
-        return this;
-    }
-
-    public PercolateRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
-        request.indicesOptions(indicesOptions);
-        return this;
-    }
-
-    /**
-     * Sets the type of the document to percolate. This is important as it selects the mapping to be used to parse
-     * the document.
-     */
-    public PercolateRequestBuilder setDocumentType(String type) {
-        request.documentType(type);
-        return this;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the search will be executed on.
-     */
-    public PercolateRequestBuilder setRouting(String routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    /**
-     * List of routing values to control the shards the search will be executed on.
-     */
-    public PercolateRequestBuilder setRouting(String... routings) {
-        request.routing(Strings.arrayToCommaDelimitedString(routings));
-        return this;
-    }
-
-    /**
-     * Sets the preference to execute the search. Defaults to randomize across shards. Can be set to
-     * <tt>_local</tt> to prefer local shards, <tt>_primary</tt> to execute only on primary shards, or
-     * a custom value, which guarantees that the same order will be used across different requests.
-     */
-    public PercolateRequestBuilder setPreference(String preference) {
-        request.preference(preference);
-        return this;
-    }
-
-    /**
-     * Enables percolating an existing document. Instead of specifying the source of the document to percolate, define
-     * a get request that will fetch a document and use its source.
-     */
-    public PercolateRequestBuilder setGetRequest(GetRequest getRequest) {
-        request.getRequest(getRequest);
-        return this;
-    }
-
-    /**
-     * Whether only to return total count and don't keep track of the matches (Count percolation).
-     */
-    public PercolateRequestBuilder setOnlyCount(boolean onlyCount) {
-        request.onlyCount(onlyCount);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setSize(int)}}
-     */
-    public PercolateRequestBuilder setSize(int size) {
-        sourceBuilder().setSize(size);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setSort(boolean)}}
-     */
-    public PercolateRequestBuilder setSortByScore(boolean sort) {
-        sourceBuilder().setSort(sort);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#addSort(SortBuilder)}
-     */
-    public PercolateRequestBuilder addSort(SortBuilder<?> sort) {
-        sourceBuilder().addSort(sort);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setSort(boolean)}}
-     */
-    public PercolateRequestBuilder setScore(boolean score) {
-        sourceBuilder().setTrackScores(score);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setDoc(PercolateSourceBuilder.DocBuilder)}
-     */
-    public PercolateRequestBuilder setPercolateDoc(PercolateSourceBuilder.DocBuilder docBuilder) {
-        sourceBuilder().setDoc(docBuilder);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setQueryBuilder(QueryBuilder)}
-     */
-    public PercolateRequestBuilder setPercolateQuery(QueryBuilder queryBuilder) {
-        sourceBuilder().setQueryBuilder(queryBuilder);
-        return this;
-    }
-
-    /**
-     * Delegates to {@link PercolateSourceBuilder#setHighlightBuilder(HighlightBuilder)}
-     */
-    public PercolateRequestBuilder setHighlightBuilder(HighlightBuilder highlightBuilder) {
-        sourceBuilder().setHighlightBuilder(highlightBuilder);
-        return this;
-    }
-
-    /**
-     * Delegates to
-     * {@link PercolateSourceBuilder#addAggregation(AggregationBuilder)}
-     */
-    public PercolateRequestBuilder addAggregation(AggregationBuilder aggregationBuilder) {
-        sourceBuilder().addAggregation(aggregationBuilder);
-        return this;
-    }
-
-    /**
-     * Delegates to
-     * {@link PercolateSourceBuilder#addAggregation(PipelineAggregationBuilder)}
-     */
-    public PercolateRequestBuilder addAggregation(PipelineAggregationBuilder aggregationBuilder) {
-        sourceBuilder().addAggregation(aggregationBuilder);
-        return this;
-    }
-
-    /**
-     * Sets the percolate request definition directly on the request. This will
-     * overwrite any definitions set by any of the delegate methods.
-     */
-    public PercolateRequestBuilder setSource(PercolateSourceBuilder source) {
-        sourceBuilder = source;
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(Map<String, Object> source) {
-        request.source(source);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(Map<String, Object> source, XContentType contentType) {
-        request.source(source, contentType);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(String source) {
-        request.source(source);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(XContentBuilder sourceBuilder) {
-        request.source(sourceBuilder);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(BytesReference source) {
-        request.source(source);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(byte[] source) {
-        request.source(source);
-        return this;
-    }
-
-    /**
-     * Raw variant of {@link #setSource(PercolateSourceBuilder)}
-     */
-    public PercolateRequestBuilder setSource(byte[] source, int offset, int length) {
-        request.source(source, offset, length);
-        return this;
-    }
-
-    private PercolateSourceBuilder sourceBuilder() {
-        if (sourceBuilder == null) {
-            sourceBuilder = new PercolateSourceBuilder();
-        }
-        return sourceBuilder;
-    }
-
-    @Override
-    public PercolateRequest request() {
-        if (sourceBuilder != null) {
-            request.source(sourceBuilder);
-        }
-        return request;
-    }
-
-    @Override
-    protected PercolateRequest beforeExecute(PercolateRequest request) {
-        if (sourceBuilder != null) {
-            request.source(sourceBuilder);
-        }
-        return request;
-    }
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
deleted file mode 100644
index 500d5416d52..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.support.broadcast.BroadcastResponse;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Encapsulates the response of a percolator request.
- *
- * @deprecated Instead use search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class PercolateResponse extends BroadcastResponse implements Iterable<PercolateResponse.Match>, ToXContent {
-
-    public static final Match[] EMPTY = new Match[0];
-    // PercolateQuery emits this score if no 'query' is defined in the percolate request
-    public static final float NO_SCORE = 0.0f;
-
-    private long tookInMillis;
-    private Match[] matches;
-    private long count;
-    private InternalAggregations aggregations;
-
-    PercolateResponse(int totalShards, int successfulShards, int failedShards, List<ShardOperationFailedException> shardFailures,
-                             Match[] matches, long count, long tookInMillis, InternalAggregations aggregations) {
-        super(totalShards, successfulShards, failedShards, shardFailures);
-        if (tookInMillis < 0) {
-            throw new IllegalArgumentException("tookInMillis must be positive but was: " + tookInMillis);
-        }
-        this.tookInMillis = tookInMillis;
-        this.matches = matches;
-        this.count = count;
-        this.aggregations = aggregations;
-    }
-
-    PercolateResponse() {
-    }
-
-    /**
-     * How long the percolate took.
-     */
-    public TimeValue getTook() {
-        return new TimeValue(tookInMillis);
-    }
-
-    /**
-     * How long the percolate took in milliseconds.
-     */
-    public long getTookInMillis() {
-        return tookInMillis;
-    }
-
-    /**
-     * @return The queries that match with the document being percolated. This can return <code>null</code> if th.
-     */
-    public Match[] getMatches() {
-        return this.matches;
-    }
-
-    /**
-     * @return The total number of queries that have matched with the document being percolated.
-     */
-    public long getCount() {
-        return count;
-    }
-
-    /**
-     * @return Any aggregations that has been executed on the query metadata. This can return <code>null</code>.
-     */
-    public InternalAggregations getAggregations() {
-        return aggregations;
-    }
-
-    @Override
-    public Iterator<Match> iterator() {
-        return Arrays.asList(matches).iterator();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.TOOK, tookInMillis);
-        RestActions.buildBroadcastShardsHeader(builder, params, this);
-
-        builder.field(Fields.TOTAL, count);
-        if (matches != null) {
-            builder.startArray(Fields.MATCHES);
-            boolean justIds = "ids".equals(params.param("percolate_format"));
-            if (justIds) {
-                for (PercolateResponse.Match match : matches) {
-                    builder.value(match.getId());
-                }
-            } else {
-                for (PercolateResponse.Match match : matches) {
-                    builder.startObject();
-                    builder.field(Fields._INDEX, match.getIndex());
-                    builder.field(Fields._ID, match.getId());
-                    float score = match.getScore();
-                    if (score != NO_SCORE) {
-                        builder.field(Fields._SCORE, match.getScore());
-                    }
-                    if (match.getHighlightFields().isEmpty() == false) {
-                        builder.startObject(Fields.HIGHLIGHT);
-                        for (HighlightField field : match.getHighlightFields().values()) {
-                            field.toXContent(builder, params);
-                        }
-                        builder.endObject();
-                    }
-                    builder.endObject();
-                }
-            }
-            builder.endArray();
-        }
-        if (aggregations != null) {
-            aggregations.toXContent(builder, params);
-        }
-        return builder;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        tookInMillis = in.readVLong();
-        count = in.readVLong();
-        int size = in.readVInt();
-        if (size != -1) {
-            matches = new Match[size];
-            for (int i = 0; i < size; i++) {
-                matches[i] = new Match();
-                matches[i].readFrom(in);
-            }
-        }
-        aggregations = InternalAggregations.readOptionalAggregations(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVLong(tookInMillis);
-        out.writeVLong(count);
-        if (matches == null) {
-            out.writeVInt(-1);
-        } else {
-            out.writeVInt(matches.length);
-            for (Match match : matches) {
-                match.writeTo(out);
-            }
-        }
-        out.writeOptionalStreamable(aggregations);
-    }
-
-    /**
-     * Represents a query that has matched with the document that was percolated.
-     */
-    public static class Match implements Streamable {
-
-        private Text index;
-        private Text id;
-        private float score;
-        private Map<String, HighlightField> hl;
-
-        /**
-         * Constructor only for internal usage.
-         */
-        public Match(Text index, Text id, float score, Map<String, HighlightField> hl) {
-            this.id = id;
-            this.score = score;
-            this.index = index;
-            this.hl = hl;
-        }
-
-        /**
-         * Constructor only for internal usage.
-         */
-        public Match(Text index, Text id, float score) {
-            this.id = id;
-            this.score = score;
-            this.index = index;
-        }
-
-        Match() {
-        }
-
-        /**
-         * @return The index that the matched percolator query resides in.
-         */
-        public Text getIndex() {
-            return index;
-        }
-
-        /**
-         * @return The id of the matched percolator query.
-         */
-        public Text getId() {
-            return id;
-        }
-
-        /**
-         * @return If in the percolate request a query was specified this returns the score representing how well that
-         * query matched on the metadata associated with the matching query otherwise {@link Float#NaN} is returned.
-         */
-        public float getScore() {
-            return score;
-        }
-
-        /**
-         * @return If highlighting was specified in the percolate request the this returns highlight snippets for each
-         * matching field in the document being percolated based on this query otherwise <code>null</code> is returned.
-         */
-        @Nullable
-        public Map<String, HighlightField> getHighlightFields() {
-            return hl;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            id = in.readText();
-            index = in.readText();
-            score = in.readFloat();
-            int size = in.readVInt();
-            if (size > 0) {
-                hl = new HashMap<>(size);
-                for (int j = 0; j < size; j++) {
-                    hl.put(in.readString(), HighlightField.readHighlightField(in));
-                }
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeText(id);
-            out.writeText(index);
-            out.writeFloat(score);
-            if (hl != null) {
-                out.writeVInt(hl.size());
-                for (Map.Entry<String, HighlightField> entry : hl.entrySet()) {
-                    out.writeString(entry.getKey());
-                    entry.getValue().writeTo(out);
-                }
-            } else {
-                out.writeVInt(0);
-            }
-        }
-    }
-
-    static final class Fields {
-        static final String TOOK = "took";
-        static final String TOTAL = "total";
-        static final String MATCHES = "matches";
-        static final String _INDEX = "_index";
-        static final String _ID = "_id";
-        static final String _SCORE = "_score";
-        static final String HIGHLIGHT = "highlight";
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
deleted file mode 100644
index 4019f91300c..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ElasticsearchGenerationException;
-import org.elasticsearch.action.support.ToXContentToBytes;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.search.sort.ScoreSortBuilder;
-import org.elasticsearch.search.sort.SortBuilder;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Builder to create the percolate request body.
- *
- * @deprecated Instead use search API with {@link PercolateQueryBuilder}
- */
-@Deprecated
-public class PercolateSourceBuilder extends ToXContentToBytes {
-
-    private DocBuilder docBuilder;
-    private QueryBuilder queryBuilder;
-    private Integer size;
-    private List<SortBuilder<?>> sorts;
-    private Boolean trackScores;
-    private HighlightBuilder highlightBuilder;
-    private List<AggregationBuilder> aggregationBuilders;
-    private List<PipelineAggregationBuilder> pipelineAggregationBuilders;
-
-    /**
-     * Sets the document to run the percolate queries against.
-     */
-    public PercolateSourceBuilder setDoc(DocBuilder docBuilder) {
-        this.docBuilder = docBuilder;
-        return this;
-    }
-
-    /**
-     * Sets a query to reduce the number of percolate queries to be evaluated and score the queries that match based
-     * on this query.
-     */
-    public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) {
-        this.queryBuilder = queryBuilder;
-        return this;
-    }
-
-    /**
-     * Limits the maximum number of percolate query matches to be returned.
-     */
-    public PercolateSourceBuilder setSize(int size) {
-        this.size = size;
-        return this;
-    }
-
-    /**
-     * Similar as {@link #setTrackScores(boolean)}, but whether to sort by the score descending.
-     */
-    public PercolateSourceBuilder setSort(boolean sort) {
-        if (sort) {
-            addSort(new ScoreSortBuilder());
-        } else {
-            this.sorts = null;
-        }
-        return this;
-    }
-
-    /**
-     * Adds a sort builder. Only sorting by score desc is supported.
-     *
-     * By default the matching percolator queries are returned in an undefined order.
-     */
-    public PercolateSourceBuilder addSort(SortBuilder<?> sort) {
-        if (sorts == null) {
-            sorts = new ArrayList<>();
-        }
-        sorts.add(sort);
-        return this;
-    }
-
-    /**
-     * Whether to compute a score for each match and include it in the response. The score is based on
-     * {@link #setQueryBuilder(QueryBuilder)}.
-     */
-    public PercolateSourceBuilder setTrackScores(boolean trackScores) {
-        this.trackScores = trackScores;
-        return this;
-    }
-
-    /**
-     * Enables highlighting for the percolate document. Per matched percolate query highlight the percolate document.
-     */
-    public PercolateSourceBuilder setHighlightBuilder(HighlightBuilder highlightBuilder) {
-        this.highlightBuilder = highlightBuilder;
-        return this;
-    }
-
-    /**
-     * Add an aggregation definition.
-     */
-    public PercolateSourceBuilder addAggregation(AggregationBuilder aggregationBuilder) {
-        if (aggregationBuilders == null) {
-            aggregationBuilders = new ArrayList<>();
-        }
-        aggregationBuilders.add(aggregationBuilder);
-        return this;
-    }
-
-    /**
-     * Add an aggregation definition.
-     */
-    public PercolateSourceBuilder addAggregation(PipelineAggregationBuilder aggregationBuilder) {
-        if (pipelineAggregationBuilders == null) {
-            pipelineAggregationBuilders = new ArrayList<>();
-        }
-        pipelineAggregationBuilders.add(aggregationBuilder);
-        return this;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        if (docBuilder != null) {
-            docBuilder.toXContent(builder, params);
-        }
-        if (queryBuilder != null) {
-            builder.field("query");
-            queryBuilder.toXContent(builder, params);
-        }
-        if (size != null) {
-            builder.field("size", size);
-        }
-        if (sorts != null) {
-            builder.startArray("sort");
-            for (SortBuilder<?> sort : sorts) {
-                sort.toXContent(builder, params);
-            }
-            builder.endArray();
-        }
-        if (trackScores != null) {
-            builder.field("track_scores", trackScores);
-        }
-        if (highlightBuilder != null) {
-            builder.field(SearchSourceBuilder.HIGHLIGHT_FIELD.getPreferredName(), highlightBuilder);
-        }
-        if (aggregationBuilders != null || pipelineAggregationBuilders != null) {
-            builder.field("aggregations");
-            builder.startObject();
-            if (aggregationBuilders != null) {
-                for (AggregationBuilder aggregation : aggregationBuilders) {
-                    aggregation.toXContent(builder, params);
-                }
-            }
-            if (pipelineAggregationBuilders != null) {
-                for (PipelineAggregationBuilder aggregation : pipelineAggregationBuilders) {
-                    aggregation.toXContent(builder, params);
-                }
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-        return builder;
-    }
-
-    /**
-     * @return A new {@link DocBuilder} instance.
-     */
-    public static DocBuilder docBuilder() {
-        return new DocBuilder();
-    }
-
-    /**
-     * A builder for defining the document to be percolated in various ways.
-     */
-    public static class DocBuilder implements ToXContent {
-
-        private BytesReference doc;
-
-        /**
-         * Sets the document to be percolated.
-         */
-        public DocBuilder setDoc(BytesReference doc) {
-            this.doc = doc;
-            return this;
-        }
-
-        /**
-         * Sets the document to be percolated.
-         */
-        public DocBuilder setDoc(String field, Object value) {
-            Map<String, Object> values = new HashMap<>(2);
-            values.put(field, value);
-            setDoc(values);
-            return this;
-        }
-
-        /**
-         * Sets the document to be percolated.
-         */
-        public DocBuilder setDoc(String doc) {
-            this.doc = new BytesArray(doc);
-            return this;
-        }
-
-        /**
-         * Sets the document to be percolated.
-         */
-        public DocBuilder setDoc(XContentBuilder doc) {
-            this.doc = doc.bytes();
-            return this;
-        }
-
-        /**
-         * Sets the document to be percolated.
-         */
-        public DocBuilder setDoc(Map doc) {
-            return setDoc(doc, Requests.CONTENT_TYPE);
-        }
-
-        @SuppressWarnings("unchecked")
-        public DocBuilder setDoc(Map doc, XContentType contentType) {
-            try {
-                return setDoc(XContentFactory.contentBuilder(contentType).map(doc));
-            } catch (IOException e) {
-                throw new ElasticsearchGenerationException("Failed to generate [" + doc + "]", e);
-            }
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return builder.rawField("doc", doc);
-        }
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index ecc037089ab..e1511c216ae 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -377,12 +377,18 @@ public class PercolatorFieldMapper extends FieldMapper {
             RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder;
             if (rangeQueryBuilder.from() instanceof String) {
                 String from = (String) rangeQueryBuilder.from();
-                String to = (String) rangeQueryBuilder.to();
-                if (from.contains("now") || to.contains("now")) {
+                if (from.contains("now")) {
                     throw new IllegalArgumentException("percolator queries containing time range queries based on the " +
                             "current time is unsupported");
                 }
             }
+            if (rangeQueryBuilder.to() instanceof String) {
+                String to = (String) rangeQueryBuilder.to();
+                if (to.contains("now")) {
+                    throw new IllegalArgumentException("percolator queries containing time range queries based on the " +
+                        "current time is unsupported");
+                }
+            }
         } else if (queryBuilder instanceof HasChildQueryBuilder) {
             throw new IllegalArgumentException("the [has_child] query is unsupported inside a percolator query");
         } else if (queryBuilder instanceof HasParentQueryBuilder) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 2afa2c92ed1..db0745a045e 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -86,7 +86,7 @@ public final class PercolatorHighlightSubFetchPhase extends HighlightPhase {
             try {
                 LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
                 int segmentDocId = hit.docId() - ctx.docBase;
-                query = queryStore.getQueries(ctx).getQuery(segmentDocId);
+                query = queryStore.getQueries(ctx).apply(segmentDocId);
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index d314de3b05f..d09599a7af4 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -19,26 +19,21 @@
 
 package org.elasticsearch.percolator;
 
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
-import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.singletonList;
 
-public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlugin, SearchPlugin {
+public class PercolatorPlugin extends Plugin implements MapperPlugin, SearchPlugin {
 
     private final Settings settings;
 
@@ -46,17 +41,6 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlug
         this.settings = settings;
     }
 
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Arrays.asList(new ActionHandler<>(PercolateAction.INSTANCE, TransportPercolateAction.class),
-                new ActionHandler<>(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class));
-    }
-
-    @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestPercolateAction.class, RestMultiPercolateAction.class);
-    }
-
     @Override
     public List<QuerySpec<?>> getQueries() {
         return singletonList(new QuerySpec<>(PercolateQueryBuilder.NAME, PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent));
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index 9564601627f..dce0e3dd2a3 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -380,7 +380,7 @@ public final class QueryAnalyzer {
 
         private final Query unsupportedQuery;
 
-        public UnsupportedQueryException(Query unsupportedQuery) {
+        UnsupportedQueryException(Query unsupportedQuery) {
             super(LoggerMessageFormat.format("no query terms can be extracted from query [{}]", unsupportedQuery));
             this.unsupportedQuery = unsupportedQuery;
         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
deleted file mode 100644
index 193e1ec6e6d..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestToXContentListener;
-
-import java.io.IOException;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-@Deprecated
-public class RestMultiPercolateAction extends BaseRestHandler {
-
-    private final boolean allowExplicitIndex;
-
-    @Inject
-    public RestMultiPercolateAction(Settings settings, RestController controller) {
-        super(settings);
-        controller.registerHandler(POST, "/_mpercolate", this);
-        controller.registerHandler(POST, "/{index}/_mpercolate", this);
-        controller.registerHandler(POST, "/{index}/{type}/_mpercolate", this);
-
-        controller.registerHandler(GET, "/_mpercolate", this);
-        controller.registerHandler(GET, "/{index}/_mpercolate", this);
-        controller.registerHandler(GET, "/{index}/{type}/_mpercolate", this);
-
-        this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
-    }
-
-    @Override
-    public RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) throws IOException {
-        MultiPercolateRequest multiPercolateRequest = new MultiPercolateRequest();
-        multiPercolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, multiPercolateRequest.indicesOptions()));
-        multiPercolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
-        multiPercolateRequest.documentType(restRequest.param("type"));
-        multiPercolateRequest.add(restRequest.contentOrSourceParam(), allowExplicitIndex);
-        return channel -> client.execute(MultiPercolateAction.INSTANCE, multiPercolateRequest, new RestToXContentListener<>(channel));
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
deleted file mode 100644
index 55c17c8715a..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.VersionType;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.RestActions;
-import org.elasticsearch.rest.action.RestToXContentListener;
-
-import java.io.IOException;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-@Deprecated
-public class RestPercolateAction extends BaseRestHandler {
-
-    @Inject
-    public RestPercolateAction(Settings settings, RestController controller) {
-        super(settings);
-        controller.registerHandler(GET, "/{index}/{type}/_percolate", this);
-        controller.registerHandler(POST, "/{index}/{type}/_percolate", this);
-
-        RestPercolateExistingDocHandler existingDocHandler = new RestPercolateExistingDocHandler(settings);
-        controller.registerHandler(GET, "/{index}/{type}/{id}/_percolate", existingDocHandler);
-        controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate", existingDocHandler);
-
-        RestCountPercolateDocHandler countHandler = new RestCountPercolateDocHandler(settings);
-        controller.registerHandler(GET, "/{index}/{type}/_percolate/count", countHandler);
-        controller.registerHandler(POST, "/{index}/{type}/_percolate/count", countHandler);
-
-        RestCountPercolateExistingDocHandler countExistingDocHandler = new RestCountPercolateExistingDocHandler(settings);
-        controller.registerHandler(GET, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
-        controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
-    }
-
-    private RestChannelConsumer parseDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, NodeClient client) {
-        percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
-        percolateRequest.documentType(restRequest.param("type"));
-        percolateRequest.routing(restRequest.param("routing"));
-        percolateRequest.preference(restRequest.param("preference"));
-        percolateRequest.source(restRequest.contentOrSourceParam());
-
-        percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        return channel -> executePercolate(client, percolateRequest, channel);
-    }
-
-    private RestChannelConsumer parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, NodeClient client) {
-        String index = restRequest.param("index");
-        String type = restRequest.param("type");
-        percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("percolate_index", index)));
-        percolateRequest.documentType(restRequest.param("percolate_type", type));
-
-        GetRequest getRequest = new GetRequest(index, type,
-                restRequest.param("id"));
-        getRequest.routing(restRequest.param("routing"));
-        getRequest.preference(restRequest.param("preference"));
-        getRequest.refresh(restRequest.paramAsBoolean("refresh", getRequest.refresh()));
-        getRequest.realtime(restRequest.paramAsBoolean("realtime", getRequest.realtime()));
-        getRequest.version(RestActions.parseVersion(restRequest));
-        getRequest.versionType(VersionType.fromString(restRequest.param("version_type"), getRequest.versionType()));
-
-        percolateRequest.getRequest(getRequest);
-        percolateRequest.routing(restRequest.param("percolate_routing"));
-        percolateRequest.preference(restRequest.param("percolate_preference"));
-        percolateRequest.source(restRequest.contentOrSourceParam());
-
-        percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        return channel -> executePercolate(client, percolateRequest, channel);
-    }
-
-    private void executePercolate(final NodeClient client, final PercolateRequest percolateRequest, final RestChannel restChannel) {
-        client.execute(PercolateAction.INSTANCE, percolateRequest, new RestToXContentListener<>(restChannel));
-    }
-
-    @Override
-    public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
-        PercolateRequest percolateRequest = new PercolateRequest();
-        return parseDocPercolate(percolateRequest, restRequest, client);
-    }
-
-    private final class RestCountPercolateDocHandler extends BaseRestHandler {
-
-        private RestCountPercolateDocHandler(Settings settings) {
-            super(settings);
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
-            PercolateRequest percolateRequest = new PercolateRequest();
-            percolateRequest.onlyCount(true);
-            return parseDocPercolate(percolateRequest, restRequest, client);
-        }
-
-    }
-
-    private final class RestPercolateExistingDocHandler extends BaseRestHandler {
-
-        RestPercolateExistingDocHandler(Settings settings) {
-            super(settings);
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
-            PercolateRequest percolateRequest = new PercolateRequest();
-            return parseExistingDocPercolate(percolateRequest, restRequest, client);
-        }
-
-    }
-
-    private final class RestCountPercolateExistingDocHandler extends BaseRestHandler {
-
-        RestCountPercolateExistingDocHandler(Settings settings) {
-            super(settings);
-        }
-
-        @Override
-        public RestChannelConsumer prepareRequest(RestRequest restRequest, final NodeClient client) throws IOException {
-            PercolateRequest percolateRequest = new PercolateRequest();
-            percolateRequest.onlyCount(true);
-            return parseExistingDocPercolate(percolateRequest, restRequest, client);
-        }
-
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
deleted file mode 100644
index 8b290d4c9c4..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.MultiGetItemResponse;
-import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.search.MultiSearchRequest;
-import org.elasticsearch.action.search.MultiSearchResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Deprecated
-public class TransportMultiPercolateAction extends HandledTransportAction<MultiPercolateRequest, MultiPercolateResponse> {
-
-    private final Client client;
-    private final ParseFieldMatcher parseFieldMatcher;
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public TransportMultiPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                         ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                         Client client, SearchRequestParsers searchRequestParsers) {
-        super(settings, MultiPercolateAction.NAME, threadPool, transportService, actionFilters,
-              indexNameExpressionResolver, MultiPercolateRequest::new);
-        this.client = client;
-        this.searchRequestParsers = searchRequestParsers;
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
-    }
-
-    @Override
-    protected void doExecute(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener) {
-        List<Tuple<Integer, GetRequest>> getRequests = new ArrayList<>();
-        for (int i = 0; i < request.requests().size(); i++) {
-            GetRequest getRequest = request.requests().get(i).getRequest();
-            if (getRequest != null) {
-                getRequests.add(new Tuple<>(i, getRequest));
-            }
-        }
-        if (getRequests.isEmpty()) {
-            innerDoExecute(request, listener, Collections.emptyMap(), new HashMap<>());
-        } else {
-            MultiGetRequest multiGetRequest = new MultiGetRequest();
-            for (Tuple<Integer, GetRequest> tuple : getRequests) {
-                GetRequest getRequest = tuple.v2();
-                multiGetRequest.add(new MultiGetRequest.Item(getRequest.index(), getRequest.type(), getRequest.id()));
-            }
-            client.multiGet(multiGetRequest, new ActionListener<MultiGetResponse>() {
-                @Override
-                public void onResponse(MultiGetResponse response) {
-                    Map<Integer, BytesReference> getResponseSources = new HashMap<>(response.getResponses().length);
-                    Map<Integer, MultiPercolateResponse.Item> preFailures = new HashMap<>();
-                    for (int i = 0; i < response.getResponses().length; i++) {
-                        MultiGetItemResponse itemResponse = response.getResponses()[i];
-                        int originalSlot = getRequests.get(i).v1();
-                        if (itemResponse.isFailed()) {
-                            preFailures.put(originalSlot, new MultiPercolateResponse.Item(itemResponse.getFailure().getFailure()));
-                        } else {
-                            if (itemResponse.getResponse().isExists()) {
-                                getResponseSources.put(originalSlot, itemResponse.getResponse().getSourceAsBytesRef());
-                            } else {
-                                GetRequest getRequest = getRequests.get(i).v2();
-                                preFailures.put(originalSlot, new MultiPercolateResponse.Item(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist", getRequest.index(), getRequest.type(), getRequest.id())));
-                            }
-                        }
-                    }
-                    innerDoExecute(request, listener, getResponseSources, preFailures);
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    listener.onFailure(e);
-                }
-            });
-        }
-    }
-
-    private void innerDoExecute(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener, Map<Integer, BytesReference> getResponseSources, Map<Integer, MultiPercolateResponse.Item> preFailures) {
-        try {
-            MultiSearchRequest multiSearchRequest = createMultiSearchRequest(request, getResponseSources, preFailures);
-            if (multiSearchRequest.requests().isEmpty()) {
-                // we may failed to turn all percolate requests into search requests,
-                // in that case just return the response...
-                listener.onResponse(
-                        createMultiPercolateResponse(new MultiSearchResponse(new MultiSearchResponse.Item[0]), request, preFailures)
-                );
-            } else {
-                client.multiSearch(multiSearchRequest, new ActionListener<MultiSearchResponse>() {
-                    @Override
-                    public void onResponse(MultiSearchResponse response) {
-                        try {
-                            listener.onResponse(createMultiPercolateResponse(response, request, preFailures));
-                        } catch (Exception e) {
-                            onFailure(e);
-                        }
-                    }
-
-                    @Override
-                    public void onFailure(Exception e) {
-                        listener.onFailure(e);
-                    }
-                });
-            }
-        } catch (Exception e) {
-            listener.onFailure(e);
-        }
-    }
-
-    private MultiSearchRequest createMultiSearchRequest(MultiPercolateRequest multiPercolateRequest, Map<Integer, BytesReference> getResponseSources, Map<Integer, MultiPercolateResponse.Item> preFailures) throws IOException {
-        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
-        multiSearchRequest.indicesOptions(multiPercolateRequest.indicesOptions());
-
-        for (int i  = 0; i < multiPercolateRequest.requests().size(); i++) {
-            if (preFailures.keySet().contains(i)) {
-                continue;
-            }
-
-            PercolateRequest percolateRequest = multiPercolateRequest.requests().get(i);
-            BytesReference docSource = getResponseSources.get(i);
-            try {
-                SearchRequest searchRequest = TransportPercolateAction.createSearchRequest(
-                    percolateRequest, docSource, searchRequestParsers.queryParsers,
-                    searchRequestParsers.aggParsers, searchRequestParsers.searchExtParsers, parseFieldMatcher);
-                multiSearchRequest.add(searchRequest);
-            } catch (Exception e) {
-                preFailures.put(i, new MultiPercolateResponse.Item(e));
-            }
-        }
-
-        return multiSearchRequest;
-    }
-
-    private MultiPercolateResponse createMultiPercolateResponse(MultiSearchResponse multiSearchResponse, MultiPercolateRequest request, Map<Integer, MultiPercolateResponse.Item> preFailures) {
-        int searchResponseIndex = 0;
-        MultiPercolateResponse.Item[] percolateItems = new MultiPercolateResponse.Item[request.requests().size()];
-        for (int i = 0; i < percolateItems.length; i++) {
-            if (preFailures.keySet().contains(i)) {
-                percolateItems[i] = preFailures.get(i);
-            } else {
-                MultiSearchResponse.Item searchItem = multiSearchResponse.getResponses()[searchResponseIndex++];
-                if (searchItem.isFailure()) {
-                    percolateItems[i] = new MultiPercolateResponse.Item(searchItem.getFailure());
-                } else {
-                    PercolateRequest percolateRequest = request.requests().get(i);
-                    percolateItems[i] = new MultiPercolateResponse.Item(TransportPercolateAction.createPercolateResponse(searchItem.getResponse(), percolateRequest.onlyCount()));
-                }
-            }
-        }
-        return new MultiPercolateResponse(percolateItems);
-    }
-
-}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
deleted file mode 100644
index 88aea9469da..00000000000
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.DefaultShardOperationFailedException;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchExtRegistry;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-@Deprecated
-public class TransportPercolateAction extends HandledTransportAction<PercolateRequest, PercolateResponse> {
-
-    private final Client client;
-    private final ParseFieldMatcher parseFieldMatcher;
-    private final SearchRequestParsers searchRequestParsers;
-
-    @Inject
-    public TransportPercolateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                    ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                                    Client client, SearchRequestParsers searchRequestParsers) {
-        super(settings, PercolateAction.NAME, threadPool, transportService, actionFilters,
-                indexNameExpressionResolver, PercolateRequest::new);
-        this.client = client;
-        this.searchRequestParsers = searchRequestParsers;
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
-    }
-
-    @Override
-    protected void doExecute(PercolateRequest request, ActionListener<PercolateResponse> listener) {
-        if (request.getRequest() != null) {
-            client.get(request.getRequest(), new ActionListener<GetResponse>() {
-                @Override
-                public void onResponse(GetResponse getResponse) {
-                    if (getResponse.isExists()) {
-                        innerDoExecute(request, getResponse.getSourceAsBytesRef(), listener);
-                    } else {
-                        onFailure(new ResourceNotFoundException("percolate document [{}/{}/{}] doesn't exist",
-                                request.getRequest().index(), request.getRequest().type(), request.getRequest().id()));
-                    }
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    listener.onFailure(e);
-                }
-            });
-        } else {
-            innerDoExecute(request, null, listener);
-        }
-    }
-
-    private void innerDoExecute(PercolateRequest request, BytesReference docSource, ActionListener<PercolateResponse> listener) {
-        SearchRequest searchRequest;
-        try {
-            searchRequest = createSearchRequest(request, docSource, searchRequestParsers.queryParsers,
-                searchRequestParsers.aggParsers, searchRequestParsers.searchExtParsers, parseFieldMatcher);
-        } catch (IOException e) {
-            listener.onFailure(e);
-            return;
-        }
-        client.search(searchRequest, new ActionListener<SearchResponse>() {
-            @Override
-            public void onResponse(SearchResponse searchResponse) {
-                try {
-                    listener.onResponse(createPercolateResponse(searchResponse, request.onlyCount()));
-                } catch (Exception e) {
-                    onFailure(e);
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                listener.onFailure(e);
-            }
-        });
-    }
-
-    public static SearchRequest createSearchRequest(PercolateRequest percolateRequest, BytesReference documentSource,
-                                                    IndicesQueriesRegistry queryRegistry, AggregatorParsers aggParsers,
-                                                    SearchExtRegistry searchExtRegistry, ParseFieldMatcher parseFieldMatcher)
-            throws IOException {
-        SearchRequest searchRequest = new SearchRequest();
-        if (percolateRequest.indices() != null) {
-            searchRequest.indices(percolateRequest.indices());
-        }
-        searchRequest.indicesOptions(percolateRequest.indicesOptions());
-        searchRequest.routing(percolateRequest.routing());
-        searchRequest.preference(percolateRequest.preference());
-
-        BytesReference querySource = null;
-        XContentBuilder searchSource = XContentFactory.jsonBuilder().startObject();
-        if (percolateRequest.source() != null && percolateRequest.source().length() > 0) {
-            try (XContentParser parser = XContentHelper.createParser(percolateRequest.source())) {
-                String currentFieldName = null;
-                XContentParser.Token token = parser.nextToken();
-                if (token != XContentParser.Token.START_OBJECT) {
-                    throw new IllegalArgumentException("Unknown token [" + token+ "]");
-                }
-
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.START_OBJECT) {
-                        if ("doc".equals(currentFieldName)) {
-                            XContentBuilder builder = XContentFactory.jsonBuilder();
-                            builder.copyCurrentStructure(parser);
-                            builder.flush();
-                            documentSource = builder.bytes();
-                        } else if ("query".equals(currentFieldName) || "filter".equals(currentFieldName)) {
-                            XContentBuilder builder = XContentFactory.jsonBuilder();
-                            builder.copyCurrentStructure(parser);
-                            builder.flush();
-                            querySource = builder.bytes();
-                        } else if ("sort".equals(currentFieldName)) {
-                            searchSource.field("sort");
-                            searchSource.copyCurrentStructure(parser);
-                        } else if ("aggregations".equals(currentFieldName)) {
-                            searchSource.field("aggregations");
-                            searchSource.copyCurrentStructure(parser);
-                        } else if ("highlight".equals(currentFieldName)) {
-                            searchSource.field("highlight");
-                            searchSource.copyCurrentStructure(parser);
-                        } else {
-                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
-                        }
-                    } else if (token == XContentParser.Token.START_ARRAY) {
-                        if ("sort".equals(currentFieldName)) {
-                            searchSource.field("sort");
-                            searchSource.copyCurrentStructure(parser);
-                        } else {
-                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
-                        }
-                    } else if (token.isValue()) {
-                        if ("size".equals(currentFieldName)) {
-                            if (percolateRequest.onlyCount()) {
-                                throw new IllegalArgumentException("Cannot set size if onlyCount == true");
-                            }
-                            searchSource.field("size", parser.intValue());
-                        } else if ("sort".equals(currentFieldName)) {
-                            searchSource.field("sort", parser.text());
-                        } else if ("track_scores".equals(currentFieldName) || "trackScores".equals(currentFieldName)) {
-                            searchSource.field("track_scores", parser.booleanValue());
-                        } else {
-                            throw new IllegalArgumentException("Unknown field [" + currentFieldName+ "]");
-                        }
-                    } else {
-                        throw new IllegalArgumentException("Unknown token [" + token + "]");
-                    }
-                }
-            }
-        }
-
-        if (percolateRequest.onlyCount()) {
-            searchSource.field("size", 0);
-        }
-
-        PercolateQueryBuilder percolateQueryBuilder =
-                new PercolateQueryBuilder("query", percolateRequest.documentType(), documentSource);
-        if (querySource != null) {
-            try (XContentParser parser = XContentHelper.createParser(querySource)) {
-                QueryParseContext queryParseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-                boolQueryBuilder.must(queryParseContext.parseInnerQueryBuilder());
-                boolQueryBuilder.filter(percolateQueryBuilder);
-                searchSource.field("query", boolQueryBuilder);
-            }
-        } else {
-            // wrapping in a constant score query with boost 0 for bwc reason.
-            // percolator api didn't emit scores before and never included scores
-            // for how well percolator queries matched with the document being percolated
-            searchSource.field("query", new ConstantScoreQueryBuilder(percolateQueryBuilder).boost(0f));
-        }
-
-        searchSource.endObject();
-        searchSource.flush();
-        BytesReference source = searchSource.bytes();
-        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(source)) {
-            QueryParseContext context = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-            searchSourceBuilder.parseXContent(context, aggParsers, null, searchExtRegistry);
-            searchRequest.source(searchSourceBuilder);
-            return searchRequest;
-        }
-    }
-
-    public static PercolateResponse createPercolateResponse(SearchResponse searchResponse, boolean onlyCount) {
-        SearchHits hits = searchResponse.getHits();
-        PercolateResponse.Match[] matches;
-        if (onlyCount) {
-            matches = null;
-        } else {
-            matches = new PercolateResponse.Match[hits.getHits().length];
-            for (int i = 0; i < hits.getHits().length; i++) {
-                SearchHit hit = hits.getHits()[i];
-                matches[i] = new PercolateResponse.Match(new Text(hit.getIndex()),
-                        new Text(hit.getId()), hit.getScore(), hit.getHighlightFields());
-            }
-        }
-
-        List<ShardOperationFailedException> shardFailures = new ArrayList<>(searchResponse.getShardFailures().length);
-        for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) {
-            shardFailures.add(new DefaultShardOperationFailedException(shardSearchFailure.index(), shardSearchFailure.shardId(),
-                    shardSearchFailure.getCause()));
-        }
-
-        return new PercolateResponse(
-            searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), searchResponse.getFailedShards(), shardFailures,
-                matches, hits.getTotalHits(), searchResponse.getTookInMillis(), (InternalAggregations) searchResponse.getAggregations()
-        );
-    }
-
-}
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
index 1526823369f..65005e957ac 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -60,6 +60,7 @@ import org.apache.lucene.search.spans.SpanNotQuery;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -384,13 +385,13 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
                 @Override
                 public Scorer scorer(LeafReaderContext context) throws IOException {
                     DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
-                    PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context);
+                    CheckedFunction<Integer, Query, IOException> leaf = queryStore.getQueries(context);
                     FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) {
 
                         @Override
                         protected boolean match(int doc) {
                             try {
-                                Query query = leaf.getQuery(doc);
+                                Query query = leaf.apply(doc);
                                 float score = memoryIndex.search(query);
                                 if (score != 0f) {
                                     if (needsScores) {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
deleted file mode 100644
index 0a359376f7b..00000000000
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.query.Operator;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
-import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
-import static org.elasticsearch.percolator.PercolatorTestUtil.prepareMultiPercolate;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class MultiPercolatorIT extends ESIntegTestCase {
-
-    private static final String INDEX_NAME = "queries";
-    private static final String TYPE_NAME = "query";
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-    public void testBasics() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text"));
-        ensureGreen();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        MultiPercolateResponse response = prepareMultiPercolate(client())
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject())))
-                .add(preparePercolate(client()) // non existing doc, so error element
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5")))
-                .execute().actionGet();
-
-        MultiPercolateResponse.Item item = response.getItems()[0];
-        assertMatchCount(item.getResponse(), 2L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(2));
-        assertThat(item.getErrorMessage(), nullValue());
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
-
-        item = response.getItems()[1];
-        assertThat(item.getErrorMessage(), nullValue());
-
-        assertMatchCount(item.getResponse(), 2L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        item = response.getItems()[2];
-        assertThat(item.getErrorMessage(), nullValue());
-        assertMatchCount(item.getResponse(), 4L);
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
-
-        item = response.getItems()[3];
-        assertThat(item.getErrorMessage(), nullValue());
-        assertMatchCount(item.getResponse(), 1L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContaining("4"));
-
-        item = response.getItems()[4];
-        assertThat(item.getResponse(), nullValue());
-        assertThat(item.getErrorMessage(), notNullValue());
-        assertThat(item.getErrorMessage(), containsString("[" + INDEX_NAME + "/type/5] doesn't exist"));
-    }
-
-    public void testWithRouting() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text"));
-        ensureGreen();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setRouting("a")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setRouting("a")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setRouting("a")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                                .must(matchQuery("field1", "b"))
-                                .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setRouting("a")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        MultiPercolateResponse response = prepareMultiPercolate(client())
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setRouting("a")
-                        .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setRouting("a")
-                        .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setRouting("a")
-                        .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject())))
-                .add(preparePercolate(client())
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setRouting("a")
-                        .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject())))
-                .add(preparePercolate(client()) // non existing doc, so error element
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setRouting("a")
-                        .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5")))
-                .execute().actionGet();
-
-        MultiPercolateResponse.Item item = response.getItems()[0];
-        assertMatchCount(item.getResponse(), 2L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(2));
-        assertThat(item.getErrorMessage(), nullValue());
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
-
-        item = response.getItems()[1];
-        assertThat(item.getErrorMessage(), nullValue());
-
-        assertMatchCount(item.getResponse(), 2L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        item = response.getItems()[2];
-        assertThat(item.getErrorMessage(), nullValue());
-        assertMatchCount(item.getResponse(), 4L);
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
-
-        item = response.getItems()[3];
-        assertThat(item.getErrorMessage(), nullValue());
-        assertMatchCount(item.getResponse(), 1L);
-        assertThat(item.getResponse().getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(item.getResponse().getMatches(), INDEX_NAME), arrayContaining("4"));
-
-        item = response.getItems()[4];
-        assertThat(item.getResponse(), nullValue());
-        assertThat(item.getErrorMessage(), notNullValue());
-        assertThat(item.getErrorMessage(), containsString("[" + INDEX_NAME + "/type/5] doesn't exist"));
-    }
-
-    public void testExistingDocsOnly() throws Exception {
-        prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-
-        int numQueries = randomIntBetween(50, 100);
-        logger.info("--> register a queries");
-        for (int i = 0; i < numQueries; i++) {
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                    .execute().actionGet();
-        }
-
-        client().prepareIndex(INDEX_NAME, "type", "1")
-                .setSource(jsonBuilder().startObject().field("field", "a").endObject())
-                .execute().actionGet();
-        refresh();
-
-        MultiPercolateRequestBuilder builder = prepareMultiPercolate(client());
-        int numPercolateRequest = randomIntBetween(50, 100);
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
-                            .setIndices(INDEX_NAME).setDocumentType("type")
-                            .setSize(numQueries)
-            );
-        }
-
-        MultiPercolateResponse response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest));
-        for (MultiPercolateResponse.Item item : response) {
-            assertThat(item.isFailure(), equalTo(false));
-            assertMatchCount(item.getResponse(), numQueries);
-            assertThat(item.getResponse().getMatches().length, equalTo(numQueries));
-        }
-
-        // Non existing doc
-        builder = prepareMultiPercolate(client());
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
-                            .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
-
-            );
-        }
-
-        response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest));
-        for (MultiPercolateResponse.Item item : response) {
-            assertThat(item.isFailure(), equalTo(true));
-            assertThat(item.getErrorMessage(), containsString("doesn't exist"));
-            assertThat(item.getResponse(), nullValue());
-        }
-
-        // One existing doc
-        builder = prepareMultiPercolate(client());
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
-                            .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
-            );
-        }
-        builder.add(
-                preparePercolate(client())
-                        .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
-                        .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
-        );
-
-        response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest + 1));
-        assertThat(response.items()[numPercolateRequest].isFailure(), equalTo(false));
-        assertMatchCount(response.items()[numPercolateRequest].getResponse(), numQueries);
-        assertThat(response.items()[numPercolateRequest].getResponse().getMatches().length, equalTo(numQueries));
-    }
-
-    public void testWithDocsOnly() throws Exception {
-        prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-        ensureGreen();
-
-        int numQueries = randomIntBetween(50, 100);
-        logger.info("--> register a queries");
-        for (int i = 0; i < numQueries; i++) {
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                    .execute().actionGet();
-        }
-        refresh();
-
-        MultiPercolateRequestBuilder builder = prepareMultiPercolate(client());
-        int numPercolateRequest = randomIntBetween(50, 100);
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setIndices(INDEX_NAME).setDocumentType("type")
-                            .setSize(numQueries)
-                            .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "a").endObject())));
-        }
-
-        MultiPercolateResponse response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest));
-        for (MultiPercolateResponse.Item item : response) {
-            assertThat(item.isFailure(), equalTo(false));
-            assertMatchCount(item.getResponse(), numQueries);
-            assertThat(item.getResponse().getMatches().length, equalTo(numQueries));
-        }
-
-        // All illegal json
-        builder = prepareMultiPercolate(client());
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setIndices(INDEX_NAME).setDocumentType("type")
-                            .setSource("illegal json"));
-        }
-
-        response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest));
-        for (MultiPercolateResponse.Item item : response) {
-            assertThat(item.isFailure(), equalTo(true));
-            assertThat(item.getFailure(), notNullValue());
-        }
-
-        // one valid request
-        builder = prepareMultiPercolate(client());
-        for (int i = 0; i < numPercolateRequest; i++) {
-            builder.add(
-                    preparePercolate(client())
-                            .setIndices(INDEX_NAME).setDocumentType("type")
-                            .setSource("illegal json"));
-        }
-        builder.add(
-                preparePercolate(client())
-                        .setSize(numQueries)
-                        .setIndices(INDEX_NAME).setDocumentType("type")
-                        .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "a").endObject())));
-
-        response = builder.execute().actionGet();
-        assertThat(response.items().length, equalTo(numPercolateRequest + 1));
-        assertThat(response.items()[numPercolateRequest].isFailure(), equalTo(false));
-        assertMatchCount(response.items()[numPercolateRequest].getResponse(), numQueries);
-        assertThat(response.items()[numPercolateRequest].getResponse().getMatches().length, equalTo(numQueries));
-    }
-
-    public void testNestedMultiPercolation() throws IOException {
-        initNestedIndexAndPercolation();
-        MultiPercolateRequestBuilder mpercolate= prepareMultiPercolate(client());
-        mpercolate.add(preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
-        mpercolate.add(preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
-        MultiPercolateResponse response = mpercolate.get();
-        assertEquals(response.getItems()[0].getResponse().getMatches().length, 0);
-        assertEquals(response.getItems()[1].getResponse().getMatches().length, 1);
-        assertEquals(response.getItems()[1].getResponse().getMatches()[0].getId().string(), "Q");
-    }
-
-    void initNestedIndexAndPercolation() throws IOException {
-        XContentBuilder mapping = XContentFactory.jsonBuilder();
-        mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
-                .startObject("employee").field("type", "nested").startObject("properties")
-                .startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
-                .endObject();
-
-        assertAcked(client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("company", mapping));
-        ensureGreen(INDEX_NAME);
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
-                .field("query", QueryBuilders.nestedQuery("employee", QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)).endObject()).get();
-
-        refresh();
-
-    }
-
-    XContentBuilder getMatchingNestedDoc() throws IOException {
-        XContentBuilder doc = XContentFactory.jsonBuilder();
-        doc.startObject().field("companyname", "stark").startArray("employee")
-                .startObject().field("name", "virginia potts").endObject()
-                .startObject().field("name", "tony stark").endObject()
-                .endArray().endObject();
-        return doc;
-    }
-
-    XContentBuilder getNotMatchingNestedDoc() throws IOException {
-        XContentBuilder doc = XContentFactory.jsonBuilder();
-        doc.startObject().field("companyname", "notstark").startArray("employee")
-                .startObject().field("name", "virginia stark").endObject()
-                .startObject().field("name", "tony potts").endObject()
-                .endArray().endObject();
-        return doc;
-    }
-
-}
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
deleted file mode 100644
index 8e3eeebc622..00000000000
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.StreamsUtils;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class MultiPercolatorRequestTests extends ESTestCase {
-    public void testParseBulkRequests() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/percolator/mpercolate1.json");
-        MultiPercolateRequest request = new MultiPercolateRequest().add(data, 0, data.length);
-
-        assertThat(request.requests().size(), equalTo(8));
-        PercolateRequest percolateRequest = request.requests().get(0);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index1"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.strictExpandOpenAndForbidClosed()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        Map sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value1").map()));
-
-        percolateRequest = request.requests().get(1);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index2"));
-        assertThat(percolateRequest.indices()[1], equalTo("my-index3"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.fromOptions(true, true, true, false, IndicesOptions.strictExpandOpenAndForbidClosed())));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value2").map()));
-
-        percolateRequest = request.requests().get(2);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index4"));
-        assertThat(percolateRequest.indices()[1], equalTo("my-index5"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.fromOptions(false, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
-        assertThat(percolateRequest.onlyCount(), equalTo(true));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value3").map()));
-
-        percolateRequest = request.requests().get(3);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index6"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.fromOptions(false, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), notNullValue());
-        assertThat(percolateRequest.getRequest().id(), equalTo("1"));
-        assertThat(percolateRequest.getRequest().type(), equalTo("my-type1"));
-        assertThat(percolateRequest.getRequest().index(), equalTo("my-index6"));
-        assertThat(percolateRequest.getRequest().routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.getRequest().preference(), equalTo("_local"));
-
-        percolateRequest = request.requests().get(4);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index7"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.strictExpandOpenAndForbidClosed()));
-        assertThat(percolateRequest.onlyCount(), equalTo(true));
-        assertThat(percolateRequest.getRequest(), notNullValue());
-        assertThat(percolateRequest.getRequest().id(), equalTo("2"));
-        assertThat(percolateRequest.getRequest().type(), equalTo("my-type1"));
-        assertThat(percolateRequest.getRequest().index(), equalTo("my-index7"));
-        assertThat(percolateRequest.getRequest().routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.getRequest().preference(), equalTo("_local"));
-
-        percolateRequest = request.requests().get(5);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index8"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("primary"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.strictExpandOpenAndForbidClosed()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value4").map()));
-
-        percolateRequest = request.requests().get(6);
-        assertThat(percolateRequest.indices()[0], equalTo("percolate-index1"));
-        assertThat(percolateRequest.documentType(), equalTo("other-type"));
-        assertThat(percolateRequest.routing(), equalTo("percolate-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.getRequest(), notNullValue());
-        assertThat(percolateRequest.getRequest().indices()[0], equalTo("my-index9"));
-        assertThat(percolateRequest.getRequest().type(), equalTo("my-type1"));
-        assertThat(percolateRequest.getRequest().routing(), nullValue());
-        assertThat(percolateRequest.getRequest().preference(), nullValue());
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.strictExpandOpenAndForbidClosed()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), nullValue());
-
-        percolateRequest = request.requests().get(7);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index10"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), nullValue());
-        assertThat(percolateRequest.preference(), nullValue());
-        assertThat(percolateRequest.getRequest(), notNullValue());
-        assertThat(percolateRequest.getRequest().indices()[0], equalTo("my-index10"));
-        assertThat(percolateRequest.getRequest().type(), equalTo("my-type1"));
-        assertThat(percolateRequest.getRequest().routing(), nullValue());
-        assertThat(percolateRequest.getRequest().preference(), nullValue());
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.fromOptions(false, false, true, false, IndicesOptions.strictExpandOpenAndForbidClosed())));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), nullValue());
-    }
-
-    public void testParseBulkRequestsDefaults() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/percolator/mpercolate2.json");
-        MultiPercolateRequest request = new MultiPercolateRequest();
-        request.indices("my-index1").documentType("my-type1").indicesOptions(IndicesOptions.lenientExpandOpen());
-        request.add(data, 0, data.length);
-
-        assertThat(request.requests().size(), equalTo(3));
-        PercolateRequest percolateRequest = request.requests().get(0);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index1"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        Map sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value1").map()));
-
-        percolateRequest = request.requests().get(1);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index1"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.routing(), equalTo("my-routing-1"));
-        assertThat(percolateRequest.preference(), equalTo("_local"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value2").map()));
-
-        percolateRequest = request.requests().get(2);
-        assertThat(percolateRequest.indices()[0], equalTo("my-index1"));
-        assertThat(percolateRequest.documentType(), equalTo("my-type1"));
-        assertThat(percolateRequest.indicesOptions(), equalTo(IndicesOptions.lenientExpandOpen()));
-        assertThat(percolateRequest.onlyCount(), equalTo(false));
-        assertThat(percolateRequest.getRequest(), nullValue());
-        assertThat(percolateRequest.source(), notNullValue());
-        sourceMap = createParser(JsonXContent.jsonXContent, percolateRequest.source()).map();
-        assertThat(sourceMap.get("doc"), equalTo((Object) MapBuilder.newMapBuilder().put("field1", "value3").map()));
-    }
-
-}
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index e1589cba14b..23f3fc77c20 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -27,14 +27,18 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
@@ -48,6 +52,7 @@ import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -107,7 +112,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
             return new PercolateQueryBuilder(queryField, docType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
                     indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
         } else {
-            return new PercolateQueryBuilder(queryField, docType, documentSource);
+            return new PercolateQueryBuilder(queryField, docType, documentSource, XContentType.JSON);
         }
     }
 
@@ -155,7 +160,8 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> pqb.toQuery(createShardContext()));
         assertThat(e.getMessage(), equalTo("query builder must be rewritten first"));
         QueryBuilder rewrite = pqb.rewrite(createShardContext());
-        PercolateQueryBuilder geoShapeQueryBuilder = new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource);
+        PercolateQueryBuilder geoShapeQueryBuilder =
+            new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource, XContentType.JSON);
         assertEquals(geoShapeQueryBuilder, rewrite);
     }
 
@@ -176,14 +182,16 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
 
     public void testRequiredParameters() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            new PercolateQueryBuilder(null, null, new BytesArray("{}"));
+            new PercolateQueryBuilder(null, null, new BytesArray("{}"), XContentType.JSON);
         });
         assertThat(e.getMessage(), equalTo("[field] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}")));
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}"), XContentType.JSON));
         assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", "_document_type", null));
+        e = expectThrows(IllegalArgumentException.class,
+            () -> new PercolateQueryBuilder("_field", "_document_type", null, null));
         assertThat(e.getMessage(), equalTo("[document] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
@@ -226,7 +234,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         }
 
         Analyzer analyzer = new WhitespaceAnalyzer();
-        ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null);
+        ParsedDocument parsedDocument = new ParsedDocument(null, null, "_id", "_type", null, docs, null, null, null);
         IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument);
         assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs));
 
@@ -239,6 +247,26 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT));
     }
 
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("P4AAAAAFZmllbGQEdHlwZQAAAAAAAA57ImZvbyI6ImJhciJ9AAAAAA==");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+            Version.V_5_0_3_UNRELEASED, Version.V_5_1_1_UNRELEASED, Version.V_5_1_2_UNRELEASED, Version.V_5_2_0_UNRELEASED);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            PercolateQueryBuilder queryBuilder = new PercolateQueryBuilder(in);
+            assertEquals("type", queryBuilder.getDocumentType());
+            assertEquals("field", queryBuilder.getField());
+            assertEquals("{\"foo\":\"bar\"}", queryBuilder.getDocument().utf8ToString());
+            assertEquals(XContentType.JSON, queryBuilder.getXContentType());
+
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.setVersion(version);
+                queryBuilder.writeTo(out);
+                assertArrayEquals(data, out.bytes().toBytesRef().bytes);
+            }
+        }
+    }
+
     private static BytesReference randomSource() {
         try {
             XContentBuilder xContent = XContentFactory.jsonBuilder();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
deleted file mode 100644
index 3577745380c..00000000000
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
-import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class PercolatorAggregationsIT extends ESIntegTestCase {
-
-    private static final String INDEX_NAME = "queries";
-    private static final String TYPE_NAME = "query";
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-    // Just test the integration with facets and aggregations, not the facet and aggregation functionality!
-    public void testAggregations() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text", "field2", "type=keyword"));
-        ensureGreen();
-
-        int numQueries = scaledRandomIntBetween(250, 500);
-        int numUniqueQueries = between(1, numQueries / 2);
-        String[] values = new String[numUniqueQueries];
-        for (int i = 0; i < values.length; i++) {
-            values[i] = "value" + i;
-        }
-        int[] expectedCount = new int[numUniqueQueries];
-
-        logger.info("--> registering {} queries", numQueries);
-        for (int i = 0; i < numQueries; i++) {
-            String value = values[i % numUniqueQueries];
-            expectedCount[i % numUniqueQueries]++;
-            QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", "b").endObject()).execute()
-                    .actionGet();
-        }
-        refresh();
-
-        for (int i = 0; i < numQueries; i++) {
-            String value = values[i % numUniqueQueries];
-            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
-                    .setIndices(INDEX_NAME)
-                    .setDocumentType("type")
-                    .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
-
-            SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
-            percolateRequestBuilder.addAggregation(AggregationBuilders.terms("a").field("field2").collectMode(aggCollectionMode));
-
-            if (randomBoolean()) {
-                percolateRequestBuilder.setPercolateQuery(matchAllQuery());
-            }
-
-            boolean countOnly = randomBoolean();
-            if (countOnly) {
-                percolateRequestBuilder.setOnlyCount(countOnly);
-            } else {
-                // can only set size if we also keep track of matches (i.e. countOnly == false)
-                if (randomBoolean()) {
-                    percolateRequestBuilder.setScore(true).setSize(expectedCount[i % numUniqueQueries]);
-                } else {
-                    percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
-                }
-            }
-
-            PercolateResponse response = percolateRequestBuilder.execute().actionGet();
-            assertMatchCount(response, expectedCount[i % numUniqueQueries]);
-            if (!countOnly) {
-                assertThat(response.getMatches(), arrayWithSize(expectedCount[i % numUniqueQueries]));
-            }
-
-            List<Aggregation> aggregations = response.getAggregations().asList();
-            assertThat(aggregations.size(), equalTo(1));
-            assertThat(aggregations.get(0).getName(), equalTo("a"));
-            List<Terms.Bucket> buckets = new ArrayList<>(((Terms) aggregations.get(0)).getBuckets());
-            assertThat(buckets.size(), equalTo(1));
-            assertThat(buckets.get(0).getKeyAsString(), equalTo("b"));
-            assertThat(buckets.get(0).getDocCount(), equalTo((long) expectedCount[i % values.length]));
-        }
-    }
-
-    // Just test the integration with facets and aggregations, not the facet and aggregation functionality!
-    public void testAggregationsAndPipelineAggregations() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text", "field2", "type=keyword"));
-        ensureGreen();
-
-        int numQueries = scaledRandomIntBetween(250, 500);
-        int numUniqueQueries = between(1, numQueries / 2);
-        String[] values = new String[numUniqueQueries];
-        for (int i = 0; i < values.length; i++) {
-            values[i] = "value" + i;
-        }
-        int[] expectedCount = new int[numUniqueQueries];
-
-        logger.info("--> registering {} queries", numQueries);
-        for (int i = 0; i < numQueries; i++) {
-            String value = values[i % numUniqueQueries];
-            expectedCount[i % numUniqueQueries]++;
-            QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", "b").endObject()).execute()
-                    .actionGet();
-        }
-        refresh();
-
-        for (int i = 0; i < numQueries; i++) {
-            String value = values[i % numUniqueQueries];
-            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
-                    .setIndices(INDEX_NAME)
-                    .setDocumentType("type")
-                    .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
-
-            SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
-            percolateRequestBuilder.addAggregation(AggregationBuilders.terms("a").field("field2").collectMode(aggCollectionMode));
-
-            if (randomBoolean()) {
-                percolateRequestBuilder.setPercolateQuery(matchAllQuery());
-            }
-
-            boolean countOnly = randomBoolean();
-            if (countOnly) {
-                percolateRequestBuilder.setOnlyCount(countOnly);
-            } else {
-                // can only set size if we also keep track of matches (i.e. countOnly == false)
-                if (randomBoolean()) {
-                    percolateRequestBuilder.setScore(true).setSize(expectedCount[i % numUniqueQueries]);
-                } else {
-                    percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
-                }
-            }
-
-            percolateRequestBuilder.addAggregation(PipelineAggregatorBuilders.maxBucket("max_a", "a>_count"));
-
-            PercolateResponse response = percolateRequestBuilder.execute().actionGet();
-            assertMatchCount(response, expectedCount[i % numUniqueQueries]);
-            if (!countOnly) {
-                assertThat(response.getMatches(), arrayWithSize(expectedCount[i % numUniqueQueries]));
-            }
-
-            Aggregations aggregations = response.getAggregations();
-            assertThat(aggregations.asList().size(), equalTo(2));
-            Terms terms = aggregations.get("a");
-            assertThat(terms, notNullValue());
-            assertThat(terms.getName(), equalTo("a"));
-            List<Terms.Bucket> buckets = new ArrayList<>(terms.getBuckets());
-            assertThat(buckets.size(), equalTo(1));
-            assertThat(buckets.get(0).getKeyAsString(), equalTo("b"));
-            assertThat(buckets.get(0).getDocCount(), equalTo((long) expectedCount[i % values.length]));
-
-            InternalBucketMetricValue maxA = aggregations.get("max_a");
-            assertThat(maxA, notNullValue());
-            assertThat(maxA.getName(), equalTo("max_a"));
-            assertThat(maxA.value(), equalTo((double) expectedCount[i % values.length]));
-            assertThat(maxA.keys(), equalTo(new String[] { "b" }));
-        }
-    }
-
-    public void testSignificantAggs() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .execute().actionGet();
-        ensureGreen();
-        PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value").endObject()))
-                .addAggregation(AggregationBuilders.significantTerms("a").field("field2"));
-        PercolateResponse response = percolateRequestBuilder.get();
-        assertNoFailures(response);
-    }
-
-    public void testSingleShardAggregations() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME).setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 1))
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text", "field2", "type=keyword"));
-        ensureGreen();
-
-        int numQueries = scaledRandomIntBetween(250, 500);
-
-        logger.info("--> registering {} queries", numQueries);
-        for (int i = 0; i < numQueries; i++) {
-            String value = "value0";
-            QueryBuilder queryBuilder = matchQuery("field1", value);
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", queryBuilder).field("field2", i % 3 == 0 ? "b" : "a").endObject())
-                    .execute()
-                    .actionGet();
-        }
-        refresh();
-
-        for (int i = 0; i < numQueries; i++) {
-            String value = "value0";
-            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
-                    .setIndices(INDEX_NAME)
-                    .setDocumentType("type")
-                    .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()));
-
-            SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values());
-            percolateRequestBuilder.addAggregation(AggregationBuilders.terms("terms").field("field2").collectMode(aggCollectionMode)
-                    .order(Order.term(true)).shardSize(2).size(1));
-
-            if (randomBoolean()) {
-                percolateRequestBuilder.setPercolateQuery(matchAllQuery());
-            }
-
-            boolean countOnly = randomBoolean();
-            if (countOnly) {
-                percolateRequestBuilder.setOnlyCount(countOnly);
-            } else {
-                // can only set size if we also keep track of matches (i.e. countOnly == false)
-                if (randomBoolean()) {
-                    percolateRequestBuilder.setScore(true).setSize(numQueries);
-                } else {
-                    percolateRequestBuilder.setSortByScore(true).setSize(numQueries);
-                }
-            }
-
-            percolateRequestBuilder.addAggregation(PipelineAggregatorBuilders.maxBucket("max_terms", "terms>_count"));
-
-            PercolateResponse response = percolateRequestBuilder.execute().actionGet();
-            assertMatchCount(response, numQueries);
-            if (!countOnly) {
-                assertThat(response.getMatches(), arrayWithSize(numQueries));
-            }
-
-            Aggregations aggregations = response.getAggregations();
-            assertThat(aggregations.asList().size(), equalTo(2));
-            Terms terms = aggregations.get("terms");
-            assertThat(terms, notNullValue());
-            assertThat(terms.getName(), equalTo("terms"));
-            List<Terms.Bucket> buckets = new ArrayList<>(terms.getBuckets());
-            assertThat(buckets.size(), equalTo(1));
-            assertThat(buckets.get(0).getKeyAsString(), equalTo("a"));
-
-            InternalBucketMetricValue maxA = aggregations.get("max_terms");
-            assertThat(maxA, notNullValue());
-            assertThat(maxA.getName(), equalTo("max_terms"));
-            assertThat(maxA.keys(), equalTo(new String[] { "a" }));
-        }
-    }
-}
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
index db1bbf13c83..28b3e6a2fdd 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java
@@ -21,10 +21,8 @@ package org.elasticsearch.percolator;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +32,7 @@ public class PercolatorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index ec1e44344e5..d47e0a46144 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -332,12 +332,9 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(typeName)
             .startObject("properties").startObject(fieldName).field("type", "percolator").field("index", "no").endObject().endObject()
             .endObject().endObject().string();
-        try {
-            mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
-            fail("MapperParsingException expected");
-        } catch (MapperParsingException e) {
-            assertThat(e.getMessage(), equalTo("Mapping definition for [" + fieldName + "] has unsupported parameters:  [index : no]"));
-        }
+        MapperParsingException e = expectThrows(MapperParsingException.class, () ->
+            mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true));
+        assertThat(e.getMessage(), containsString("Mapping definition for [" + fieldName + "] has unsupported parameters:  [index : no]"));
     }
 
     // multiple percolator fields are allowed in the mapping, but only one field can be used at index time.
@@ -445,6 +442,53 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                 }
         );
         assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        e = expectThrows(MapperParsingException.class, () -> {
+                mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                    jsonBuilder().startObject()
+                        .field(fieldName, rangeQuery("date_field").from("now"))
+                        .endObject().bytes());
+            }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        e = expectThrows(MapperParsingException.class, () -> {
+                mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                    jsonBuilder().startObject()
+                        .field(fieldName, rangeQuery("date_field").to("now"))
+                        .endObject().bytes());
+            }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+    }
+
+    // https://github.com/elastic/elasticsearch/issues/22355
+    public void testVerifyRangeQueryWithNullBounds() throws Exception {
+        addQueryMapping();
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+                mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                    jsonBuilder().startObject()
+                        .field(fieldName, rangeQuery("date_field").from("now").to(null))
+                        .endObject().bytes());
+            }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        e = expectThrows(MapperParsingException.class, () -> {
+                mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                    jsonBuilder().startObject()
+                        .field(fieldName, rangeQuery("date_field").from(null).to("now"))
+                        .endObject().bytes());
+            }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+
+        // No validation failures:
+        mapperService.documentMapper(typeName).parse("test", typeName, "1",
+            jsonBuilder().startObject()
+                .field(fieldName, rangeQuery("date_field").from("2016-01-01").to(null))
+                .endObject().bytes());
+        mapperService.documentMapper(typeName).parse("test", typeName, "1",
+            jsonBuilder().startObject()
+                .field(fieldName, rangeQuery("date_field").from(null).to("2016-01-01"))
+                .endObject().bytes());
     }
 
     public void testUnsupportedQueries() {
@@ -521,7 +565,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
 
         ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
                 XContentFactory.jsonBuilder().startObject()
-                        .rawField(fieldName, new BytesArray(query.string()))
+                        .rawField(fieldName, new BytesArray(query.string()), query.contentType())
                         .endObject().bytes());
         BytesRef querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         Map<String, Object> parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2();
@@ -548,7 +592,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
 
         doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
                 XContentFactory.jsonBuilder().startObject()
-                        .rawField(fieldName, new BytesArray(query.string()))
+                        .rawField(fieldName, new BytesArray(query.string()), query.contentType())
                         .endObject().bytes());
         querySource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         parsedQuery = XContentHelper.convertToMap(new BytesArray(querySource), true).v2();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
deleted file mode 100644
index 70c7f651922..00000000000
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ /dev/null
@@ -1,1839 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.percolator;
-
-import com.vividsolutions.jts.geom.Coordinate;
-
-import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.ResourceNotFoundException;
-import org.elasticsearch.action.admin.indices.alias.Alias;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
-import org.elasticsearch.common.lucene.search.function.CombineFunction;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Settings.Builder;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.engine.VersionConflictEngineException;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.query.Operator;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.QueryShardException;
-import org.elasticsearch.index.query.functionscore.WeightBuilder;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.MockScriptPlugin;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.function.Function;
-
-import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
-import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
-import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
-import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.emptyArray;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
-
-public class PercolatorIT extends ESIntegTestCase {
-
-    private static final String INDEX_NAME = "queries";
-    private static final String TYPE_NAME = "query";
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Collections.singleton(PercolatorPlugin.class);
-    }
-
-    public void testSimple1() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-        ensureGreen();
-
-        logger.info("--> Add dummy doc");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1").setSource("field1", "value").execute().actionGet();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Percolate doc with field1=b");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
-
-        logger.info("--> Percolate doc with field1=c");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        logger.info("--> Percolate doc with field1=b c");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 4L);
-        assertThat(response.getMatches(), arrayWithSize(4));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
-
-        logger.info("--> Percolate doc with field1=d");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("4"));
-
-        logger.info("--> Percolate non existing doc");
-        try {
-            preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                    .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5"))
-                    .execute().actionGet();
-            fail("Exception should have been thrown");
-        } catch (ResourceNotFoundException e) {
-            assertThat(e.getMessage(), equalTo("percolate document [queries/type/5] doesn't exist"));
-        }
-    }
-
-    public void testSimple2() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping("type1", "field1", "type=long", "field2", "type=text")
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-        );
-        ensureGreen();
-
-        // introduce the doc
-        XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                .field("field1", 1)
-                .field("field2", "value")
-                .endObject().endObject();
-
-        PercolateResponse response = preparePercolate(client()).setSource(doc)
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .execute().actionGet();
-        assertMatchCount(response, 0L);
-        assertThat(response.getMatches(), emptyArray());
-
-        // add first query...
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "test1")
-                .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field2", "value")).endObject())
-                .execute().actionGet();
-        refresh();
-
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
-                .setSource(doc).execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("test1"));
-
-        // add second query...
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "test2")
-                .setSource(XContentFactory.jsonBuilder().startObject().field("query", termQuery("field1", 1)).endObject())
-                .execute().actionGet();
-        refresh();
-
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(doc)
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("test1", "test2"));
-
-
-        client().prepareDelete(INDEX_NAME, TYPE_NAME, "test2").execute().actionGet();
-        refresh();
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(doc).execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("test1"));
-    }
-
-    public void testPercolateQueriesWithRouting() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .setSettings(Settings.builder().put("index.number_of_shards", 2))
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type", "field1", "type=text")
-                .execute().actionGet();
-        ensureGreen();
-
-        logger.info("--> register a queries");
-        for (int i = 1; i <= 100; i++) {
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                    .setRouting(Integer.toString(i % 2))
-                    .execute().actionGet();
-        }
-        refresh();
-
-        logger.info("--> Percolate doc with no routing");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
-                .setSize(100)
-                .execute().actionGet();
-        assertMatchCount(response, 100L);
-        assertThat(response.getMatches(), arrayWithSize(100));
-
-        logger.info("--> Percolate doc with routing=0");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
-                        .setSize(100)
-                        .setRouting("0")
-                        .execute().actionGet();
-        assertMatchCount(response, 50L);
-        assertThat(response.getMatches(), arrayWithSize(50));
-
-        logger.info("--> Percolate doc with routing=1");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
-                .setSize(100)
-                .setRouting("1")
-                .execute().actionGet();
-        assertMatchCount(response, 50L);
-        assertThat(response.getMatches(), arrayWithSize(50));
-    }
-
-    public void storePercolateQueriesOnRecreatedIndex() throws Exception {
-        prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, "test", "1").setSource("field1", "value1").execute().actionGet();
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "kuku1")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        cluster().wipeIndices("test");
-        createIndex("test");
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, "test", "1").setSource("field1", "value1").execute().actionGet();
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "kuku2")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-    }
-
-    // see #2814
-    public void testPercolateCustomAnalyzer() throws Exception {
-        Builder builder = Settings.builder();
-        builder.put("index.analysis.analyzer.lwhitespacecomma.tokenizer", "whitespacecomma");
-        builder.putArray("index.analysis.analyzer.lwhitespacecomma.filter", "lowercase");
-        builder.put("index.analysis.tokenizer.whitespacecomma.type", "pattern");
-        builder.put("index.analysis.tokenizer.whitespacecomma.pattern", "(,|\\s+)");
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("doc")
-                .startObject("properties")
-                .startObject("filingcategory").field("type", "text").field("analyzer", "lwhitespacecomma").endObject()
-                .endObject()
-                .endObject().endObject();
-
-        assertAcked(prepareCreate(INDEX_NAME).setSettings(builder)
-                .addMapping("doc", mapping)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-        );
-        ensureGreen();
-
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject()
-                        .field("source", "productizer")
-                        .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("filingcategory:s")))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-        refresh();
-
-        PercolateResponse percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("doc")
-                .setSource(jsonBuilder().startObject()
-                        .startObject("doc").field("filingcategory", "s").endObject()
-                        .field("query", termQuery("source", "productizer"))
-                        .endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-
-    }
-
-    public void testCreateIndexAndThenRegisterPercolator() throws Exception {
-        prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type1", "field1", "type=text")
-                .get();
-        ensureGreen();
-
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .execute().actionGet();
-        refresh();
-        SearchResponse countResponse = client().prepareSearch().setSize(0)
-                .setQuery(matchAllQuery()).setTypes(TYPE_NAME)
-                .execute().actionGet();
-        assertThat(countResponse.getHits().totalHits(), equalTo(1L));
-
-
-        for (int i = 0; i < 10; i++) {
-            PercolateResponse percolate = preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("type1")
-                    .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
-                    .execute().actionGet();
-            assertMatchCount(percolate, 1L);
-            assertThat(percolate.getMatches(), arrayWithSize(1));
-        }
-
-        for (int i = 0; i < 10; i++) {
-            PercolateResponse percolate = preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("type1")
-                    .setPreference("_local")
-                    .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
-                    .execute().actionGet();
-            assertMatchCount(percolate, 1L);
-            assertThat(percolate.getMatches(), arrayWithSize(1));
-        }
-
-
-        logger.info("--> delete the index");
-        client().admin().indices().prepareDelete(INDEX_NAME).execute().actionGet();
-        logger.info("--> make sure percolated queries for it have been deleted as well");
-        countResponse = client().prepareSearch().setSize(0)
-                .setQuery(matchAllQuery()).setTypes(TYPE_NAME)
-                .execute().actionGet();
-        assertHitCount(countResponse, 0L);
-    }
-
-    public void testMultiplePercolators() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("type1", "field1", "type=text")
-        );
-        ensureGreen();
-
-        logger.info("--> register a query 1");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        logger.info("--> register a query 2");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "bubu")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "green")
-                        .field("query", termQuery("field1", "value2"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        PercolateResponse percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("kuku"));
-
-        percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value2").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("bubu"));
-
-    }
-
-    public void testDynamicAddingRemovingQueries() throws Exception {
-        assertAcked(
-                prepareCreate(INDEX_NAME)
-                        .addMapping("type1", "field1", "type=text")
-                        .addMapping(TYPE_NAME, "query", "type=percolator")
-        );
-        ensureGreen();
-
-        logger.info("--> register a query 1");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "kuku")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "blue")
-                        .field("query", termQuery("field1", "value1"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        PercolateResponse percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("kuku"));
-
-        logger.info("--> register a query 2");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "bubu")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "green")
-                        .field("query", termQuery("field1", "value2"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value2").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("bubu"));
-
-        logger.info("--> register a query 3");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "susu")
-                .setSource(jsonBuilder().startObject()
-                        .field("color", "red")
-                        .field("query", termQuery("field1", "value2"))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-
-        PercolateSourceBuilder sourceBuilder = new PercolateSourceBuilder()
-                .setDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value2").endObject()))
-                .setQueryBuilder(termQuery("color", "red"));
-        percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(sourceBuilder)
-                .execute().actionGet();
-        assertMatchCount(percolate, 1L);
-        assertThat(percolate.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("susu"));
-
-        logger.info("--> deleting query 1");
-        client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
-
-        percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type1")
-                .setSource(jsonBuilder().startObject().startObject("doc").startObject("type1")
-                        .field("field1", "value1")
-                        .endObject().endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(percolate, 0L);
-        assertThat(percolate.getMatches(), emptyArray());
-    }
-
-    public void testPercolatingExistingDocs() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        logger.info("--> Adding docs");
-        client().prepareIndex(INDEX_NAME, "type", "1").setSource("field1", "b").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "3").setSource("field1", "b c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "4").setSource("field1", "d").execute().actionGet();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Percolate existing doc with id 1");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
-
-        logger.info("--> Percolate existing doc with id 2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        logger.info("--> Percolate existing doc with id 3");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3"))
-                .execute().actionGet();
-        assertMatchCount(response, 4L);
-        assertThat(response.getMatches(), arrayWithSize(4));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
-
-        logger.info("--> Percolate existing doc with id 4");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4"))
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("4"));
-    }
-
-    public void testPercolatingExistingDocs_routing() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .execute().actionGet();
-        ensureGreen();
-
-        logger.info("--> Adding docs");
-        client().prepareIndex(INDEX_NAME, "type", "1").setSource("field1", "b").setRouting("4").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").setRouting("3").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "3").setSource("field1", "b c").setRouting("2").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "4").setSource("field1", "d").setRouting("1").execute().actionGet();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Percolate existing doc with id 1");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1").routing("4"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
-
-        logger.info("--> Percolate existing doc with id 2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").routing("3"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        logger.info("--> Percolate existing doc with id 3");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3").routing("2"))
-                .execute().actionGet();
-        assertMatchCount(response, 4L);
-        assertThat(response.getMatches(), arrayWithSize(4));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
-
-        logger.info("--> Percolate existing doc with id 4");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4").routing("1"))
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("4"));
-    }
-
-    public void testPercolatingExistingDocs_versionCheck() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        logger.info("--> Adding docs");
-        client().prepareIndex(INDEX_NAME, "type", "1").setSource("field1", "b").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "3").setSource("field1", "b c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "4").setSource("field1", "d").execute().actionGet();
-
-        logger.info("--> registering queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Percolate existing doc with id 2 and version 1");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(1L))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-
-        logger.info("--> Percolate existing doc with id 2 and version 2");
-        try {
-            preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("type")
-                    .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(2L))
-                    .execute().actionGet();
-            fail("Error should have been thrown");
-        } catch (VersionConflictEngineException e) {
-        }
-
-        logger.info("--> Index doc with id for the second time");
-        client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").execute().actionGet();
-
-        logger.info("--> Percolate existing doc with id 2 and version 2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(2L))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
-    }
-
-    public void testPercolateMultipleIndicesAndAliases() throws Exception {
-        prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-        prepareCreate(INDEX_NAME + "2").addMapping(TYPE_NAME, "query", "type=percolator").get();
-        ensureGreen();
-
-        logger.info("--> registering queries");
-        for (int i = 1; i <= 10; i++) {
-            String index = i % 2 == 0 ? INDEX_NAME : INDEX_NAME + "2";
-            client().prepareIndex(index, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                    .execute().actionGet();
-        }
-        refresh();
-
-        logger.info("--> Percolate doc to index test1");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-
-        logger.info("--> Percolate doc to index test2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME + "2").setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-
-        logger.info("--> Percolate doc to index test1 and test2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME, INDEX_NAME + "2").setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 10L);
-        assertThat(response.getMatches(), arrayWithSize(10));
-
-        logger.info("--> Percolate doc to index test2 and test3, with ignore missing");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME , INDEX_NAME + "3").setDocumentType("type")
-                .setIndicesOptions(IndicesOptions.lenientExpandOpen())
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-
-        logger.info("--> Adding aliases");
-        IndicesAliasesResponse aliasesResponse = client().admin().indices().prepareAliases()
-                .addAlias(INDEX_NAME, "my-alias1")
-                .addAlias(INDEX_NAME + "2", "my-alias1")
-                .addAlias(INDEX_NAME + "2", "my-alias2")
-                .setTimeout(TimeValue.timeValueHours(10))
-                .execute().actionGet();
-        assertTrue(aliasesResponse.isAcknowledged());
-
-        logger.info("--> Percolate doc to my-alias1");
-        response = preparePercolate(client())
-                .setIndices("my-alias1").setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 10L);
-        assertThat(response.getMatches(), arrayWithSize(10));
-        for (PercolateResponse.Match match : response) {
-            assertThat(match.getIndex().string(), anyOf(equalTo(INDEX_NAME), equalTo(INDEX_NAME + "2")));
-        }
-
-        logger.info("--> Percolate doc to my-alias2");
-        response = preparePercolate(client())
-                .setIndices("my-alias2").setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        for (PercolateResponse.Match match : response) {
-            assertThat(match.getIndex().string(), equalTo(INDEX_NAME + "2"));
-        }
-    }
-
-    public void testPercolateWithAliasFilter() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                        .addMapping(TYPE_NAME, "query", "type=percolator")
-                        .addMapping("my-type", "a", "type=keyword")
-                        .addAlias(new Alias("a").filter(QueryBuilders.termQuery("a", "a")))
-                        .addAlias(new Alias("b").filter(QueryBuilders.termQuery("a", "b")))
-                        .addAlias(new Alias("c").filter(QueryBuilders.termQuery("a", "c")))
-        );
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "a").endObject())
-                .get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("a", "b").endObject())
-                .get();
-        refresh();
-
-        // Specifying only the document to percolate and no filter, sorting or aggs, the queries are retrieved from
-        // memory directly. Otherwise we need to retrieve those queries from lucene to be able to execute filters,
-        // aggregations and sorting on top of them. So this test a different code execution path.
-        PercolateResponse response = preparePercolate(client())
-                .setIndices("a")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(1L));
-        assertThat(response.getMatches()[0].getId().string(), equalTo("1"));
-
-        response = preparePercolate(client())
-                .setIndices("b")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(1L));
-        assertThat(response.getMatches()[0].getId().string(), equalTo("2"));
-
-
-        response = preparePercolate(client())
-                .setIndices("c")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(0L));
-
-        // Testing that the alias filter and the filter specified while percolating are both taken into account.
-        response = preparePercolate(client())
-                .setIndices("a")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .setPercolateQuery(QueryBuilders.matchAllQuery())
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(1L));
-        assertThat(response.getMatches()[0].getId().string(), equalTo("1"));
-
-        response = preparePercolate(client())
-                .setIndices("b")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .setPercolateQuery(QueryBuilders.matchAllQuery())
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(1L));
-        assertThat(response.getMatches()[0].getId().string(), equalTo("2"));
-
-
-        response = preparePercolate(client())
-                .setIndices("c")
-                .setDocumentType("my-type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .setPercolateQuery(QueryBuilders.matchAllQuery())
-                .get();
-        assertNoFailures(response);
-        assertThat(response.getCount(), equalTo(0L));
-    }
-
-    public void testCountPercolation() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        logger.info("--> Add dummy doc");
-        client().prepareIndex(INDEX_NAME, "type", "1").setSource("field1", "value").execute().actionGet();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Count percolate doc with field1=b");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate doc with field1=c");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate doc with field1=b c");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 4L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate doc with field1=d");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate non existing doc");
-        try {
-            preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                    .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5"))
-                    .execute().actionGet();
-            fail("Exception should have been thrown");
-        } catch (ResourceNotFoundException e) {
-            assertThat(e.getMessage(), equalTo("percolate document [" + INDEX_NAME + "/type/5] doesn't exist"));
-        }
-    }
-
-    public void testCountPercolatingExistingDocs() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        logger.info("--> Adding docs");
-        client().prepareIndex(INDEX_NAME, "type", "1").setSource("field1", "b").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "3").setSource("field1", "b c").execute().actionGet();
-        client().prepareIndex(INDEX_NAME, "type", "4").setSource("field1", "d").execute().actionGet();
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                        .must(matchQuery("field1", "b"))
-                        .must(matchQuery("field1", "c"))
-                ).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Count percolate existing doc with id 1");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate existing doc with id 2");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate existing doc with id 3");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3"))
-                .execute().actionGet();
-        assertMatchCount(response, 4L);
-        assertThat(response.getMatches(), nullValue());
-
-        logger.info("--> Count percolate existing doc with id 4");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4"))
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), nullValue());
-    }
-
-    public void testPercolateSizingWithQueryAndFilter() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        int numLevels = randomIntBetween(1, 25);
-        long numQueriesPerLevel = randomIntBetween(10, 250);
-        long totalQueries = numLevels * numQueriesPerLevel;
-        logger.info("--> register {} queries", totalQueries);
-        for (int level = 1; level <= numLevels; level++) {
-            for (int query = 1; query <= numQueriesPerLevel; query++) {
-                client().prepareIndex(INDEX_NAME, TYPE_NAME, level + "-" + query)
-                        .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", level).endObject())
-                        .execute().actionGet();
-            }
-        }
-        refresh();
-
-        boolean onlyCount = randomBoolean();
-        PercolateRequestBuilder builder = preparePercolate(client())
-            .setIndices(INDEX_NAME).setDocumentType("my-type")
-            .setOnlyCount(onlyCount)
-            .setPercolateDoc(docBuilder().setDoc("field", "value"));
-        if (!onlyCount) {
-            builder.setSize((int) totalQueries);
-        }
-        PercolateResponse response = builder.execute().actionGet();
-        assertMatchCount(response, totalQueries);
-        if (!onlyCount) {
-            assertThat(response.getMatches().length, equalTo((int) totalQueries));
-        }
-
-        int size = randomIntBetween(0, (int) totalQueries - 1);
-        builder = preparePercolate(client())
-            .setIndices(INDEX_NAME).setDocumentType("my-type")
-            .setOnlyCount(onlyCount)
-            .setPercolateDoc(docBuilder().setDoc("field", "value"));
-        if (!onlyCount) {
-            builder.setSize(size);
-        }
-        response = builder.execute().actionGet();
-        assertMatchCount(response, totalQueries);
-        if (!onlyCount) {
-            assertThat(response.getMatches().length, equalTo(size));
-        }
-
-        // The query / filter capabilities are NOT in realtime
-        refresh();
-
-        int runs = randomIntBetween(3, 16);
-        for (int i = 0; i < runs; i++) {
-            onlyCount = randomBoolean();
-            builder = preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setOnlyCount(onlyCount)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
-            if (!onlyCount) {
-                builder.setSize((int) numQueriesPerLevel);
-            }
-            response = builder.execute().actionGet();
-            assertMatchCount(response, numQueriesPerLevel);
-            if (!onlyCount) {
-                assertThat(response.getMatches().length, equalTo((int) numQueriesPerLevel));
-            }
-        }
-
-        for (int i = 0; i < runs; i++) {
-            onlyCount = randomBoolean();
-            builder = preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setOnlyCount(onlyCount)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
-            if (!onlyCount) {
-                builder.setSize((int) numQueriesPerLevel);
-            }
-            response = builder.execute().actionGet();
-            assertMatchCount(response, numQueriesPerLevel);
-            if (!onlyCount) {
-                assertThat(response.getMatches().length, equalTo((int) numQueriesPerLevel));
-            }
-        }
-
-        for (int i = 0; i < runs; i++) {
-            onlyCount = randomBoolean();
-            size = randomIntBetween(0, (int) numQueriesPerLevel - 1);
-            builder = preparePercolate(client())
-                    .setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setOnlyCount(onlyCount)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(termQuery("level", 1 + randomInt(numLevels - 1)));
-            if (!onlyCount) {
-                builder.setSize(size);
-            }
-            response = builder.execute().actionGet();
-            assertMatchCount(response, numQueriesPerLevel);
-            if (!onlyCount) {
-                assertThat(response.getMatches().length, equalTo(size));
-            }
-        }
-    }
-
-    public void testPercolateScoreAndSorting() throws Exception {
-        prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        // Add a dummy doc, that shouldn't never interfere with percolate operations.
-        client().prepareIndex(INDEX_NAME, "my-type", "1").setSource("field", "value").execute().actionGet();
-
-        Map<Integer, NavigableSet<Integer>> controlMap = new HashMap<>();
-        long numQueries = randomIntBetween(100, 250);
-        logger.info("--> register {} queries", numQueries);
-        for (int i = 0; i < numQueries; i++) {
-            int value = randomInt(10);
-            client().prepareIndex(INDEX_NAME, TYPE_NAME, Integer.toString(i))
-                    .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", i).field("field1", value).endObject())
-                    .execute().actionGet();
-            if (!controlMap.containsKey(value)) {
-                controlMap.put(value, new TreeSet<Integer>());
-            }
-            controlMap.get(value).add(i);
-        }
-        List<Integer> usedValues = new ArrayList<>(controlMap.keySet());
-        refresh();
-
-        // Only retrieve the score
-        int runs = randomInt(27);
-        for (int i = 0; i < runs; i++) {
-            int size = randomIntBetween(1, 50);
-            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setScore(true)
-                    .setSize(size)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level")))
-                    .execute().actionGet();
-            assertMatchCount(response, numQueries);
-            assertThat(response.getMatches().length, equalTo(size));
-            for (int j = 0; j < response.getMatches().length; j++) {
-                String id = response.getMatches()[j].getId().string();
-                assertThat(Integer.valueOf(id), equalTo((int) response.getMatches()[j].getScore()));
-            }
-        }
-
-        // Sort the queries by the score
-        for (int i = 0; i < runs; i++) {
-            int size = randomIntBetween(1, 10);
-            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setSortByScore(true)
-                    .setSize(size)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level")))
-                    .execute().actionGet();
-            assertMatchCount(response, numQueries);
-            assertThat(response.getMatches().length, equalTo(size));
-
-            int expectedId = (int) (numQueries - 1);
-            for (PercolateResponse.Match match : response) {
-                assertThat(match.getId().string(), equalTo(Integer.toString(expectedId)));
-                assertThat(match.getScore(), equalTo((float) expectedId));
-                assertThat(match.getIndex().string(), equalTo(INDEX_NAME));
-                expectedId--;
-            }
-        }
-
-
-        for (int i = 0; i < runs; i++) {
-            int value = usedValues.get(randomInt(usedValues.size() - 1));
-            NavigableSet<Integer> levels = controlMap.get(value);
-            int size = randomIntBetween(1, levels.size());
-            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
-                    .setSortByScore(true)
-                    .setSize(size)
-                    .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                    .setPercolateQuery(
-                            QueryBuilders.functionScoreQuery(matchQuery("field1", value), fieldValueFactorFunction("level"))
-                                    .boostMode(
-                                    CombineFunction.REPLACE))
-                    .execute().actionGet();
-
-            assertMatchCount(response, levels.size());
-            assertThat(response.getMatches().length, equalTo(Math.min(levels.size(), size)));
-            Iterator<Integer> levelIterator = levels.descendingIterator();
-            for (PercolateResponse.Match match : response) {
-                int controlLevel = levelIterator.next();
-                assertThat(match.getId().string(), equalTo(Integer.toString(controlLevel)));
-                assertThat(match.getScore(), equalTo((float) controlLevel));
-                assertThat(match.getIndex().string(), equalTo(INDEX_NAME));
-            }
-        }
-    }
-
-    public void testPercolateSortingWithNoSize() throws Exception {
-        prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 1).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("level", 2).endObject())
-                .execute().actionGet();
-        refresh();
-
-        PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
-                .setSortByScore(true)
-                .setSize(2)
-                .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level")))
-                .execute().actionGet();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches()[0].getId().string(), equalTo("2"));
-        assertThat(response.getMatches()[0].getScore(), equalTo(2f));
-        assertThat(response.getMatches()[1].getId().string(), equalTo("1"));
-        assertThat(response.getMatches()[1].getScore(), equalTo(1f));
-    }
-
-    public void testPercolateOnEmptyIndex() throws Exception {
-        prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
-                .setSortByScore(true)
-                .setSize(2)
-                .setPercolateDoc(docBuilder().setDoc("field", "value"))
-                .setPercolateQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("level").missing(0.0)))
-                .execute().actionGet();
-        assertMatchCount(response, 0L);
-    }
-
-    public void testPercolatorWithHighlighting() throws Exception {
-        StringBuilder fieldMapping = new StringBuilder("type=text")
-                .append(",store=").append(randomBoolean());
-        if (randomBoolean()) {
-            fieldMapping.append(",term_vector=with_positions_offsets");
-        } else if (randomBoolean()) {
-            fieldMapping.append(",index_options=offsets");
-        }
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping("type", "field1", fieldMapping.toString())
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-        );
-
-        logger.info("--> register a queries");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "3")
-                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "4")
-                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject())
-                .execute().actionGet();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "5")
-                .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> Percolate doc with field1=The quick brown fox jumps over the lazy dog");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
-                .setHighlightBuilder(new HighlightBuilder().field("field1"))
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        PercolateResponse.Match[] matches = response.getMatches();
-        Arrays.sort(matches, (a, b) -> a.getId().compareTo(b.getId()));
-
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
-
-        logger.info("--> Query percolate doc with field1=The quick brown fox jumps over the lazy dog");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
-                .setHighlightBuilder(new HighlightBuilder().field("field1"))
-                .setPercolateQuery(matchAllQuery())
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        matches = response.getMatches();
-        Arrays.sort(matches, new Comparator<PercolateResponse.Match>() {
-            @Override
-            public int compare(PercolateResponse.Match a, PercolateResponse.Match b) {
-                return a.getId().compareTo(b.getId());
-            }
-        });
-
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
-
-        logger.info("--> Query percolate with score for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
-                .setHighlightBuilder(new HighlightBuilder().field("field1"))
-                .setPercolateQuery(functionScoreQuery(new WeightBuilder().setWeight(5.5f)))
-                .setScore(true)
-                .execute().actionGet();
-        assertNoFailures(response);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        matches = response.getMatches();
-        Arrays.sort(matches, new Comparator<PercolateResponse.Match>() {
-            @Override
-            public int compare(PercolateResponse.Match a, PercolateResponse.Match b) {
-                return a.getId().compareTo(b.getId());
-            }
-        });
-
-        assertThat(matches[0].getScore(), equalTo(5.5f));
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
-        assertThat(matches[1].getScore(), equalTo(5.5f));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
-        assertThat(matches[2].getScore(), equalTo(5.5f));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getScore(), equalTo(5.5f));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
-        assertThat(matches[4].getScore(), equalTo(5.5f));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
-
-        logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
-                .setHighlightBuilder(new HighlightBuilder().field("field1"))
-                .setPercolateQuery(functionScoreQuery(new WeightBuilder().setWeight(5.5f)))
-                .setSortByScore(true)
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        matches = response.getMatches();
-        Arrays.sort(matches, new Comparator<PercolateResponse.Match>() {
-            @Override
-            public int compare(PercolateResponse.Match a, PercolateResponse.Match b) {
-                return a.getId().compareTo(b.getId());
-            }
-        });
-
-        assertThat(matches[0].getScore(), equalTo(5.5f));
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
-        assertThat(matches[1].getScore(), equalTo(5.5f));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
-        assertThat(matches[2].getScore(), equalTo(5.5f));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getScore(), equalTo(5.5f));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
-        assertThat(matches[4].getScore(), equalTo(5.5f));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
-
-        logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
-                .setHighlightBuilder(new HighlightBuilder().field("field1").highlightQuery(QueryBuilders.matchQuery("field1", "jumps")))
-                .setPercolateQuery(functionScoreQuery(new WeightBuilder().setWeight(5.5f)))
-                .setSortByScore(true)
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        matches = response.getMatches();
-        Arrays.sort(matches, new Comparator<PercolateResponse.Match>() {
-            @Override
-            public int compare(PercolateResponse.Match a, PercolateResponse.Match b) {
-                return a.getId().compareTo(b.getId());
-            }
-        });
-
-        assertThat(matches[0].getScore(), equalTo(5.5f));
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[1].getScore(), equalTo(5.5f));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[2].getScore(), equalTo(5.5f));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getScore(), equalTo(5.5f));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[4].getScore(), equalTo(5.5f));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-
-        // Highlighting an existing doc
-        client().prepareIndex(INDEX_NAME, "type", "1")
-                .setSource(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject())
-                .get();
-        refresh();
-
-        logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSize(5)
-                .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
-                .setHighlightBuilder(new HighlightBuilder().field("field1"))
-                .setPercolateQuery(functionScoreQuery(new WeightBuilder().setWeight(5.5f)))
-                .setSortByScore(true)
-                .execute().actionGet();
-        assertMatchCount(response, 5L);
-        assertThat(response.getMatches(), arrayWithSize(5));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4", "5"));
-
-        matches = response.getMatches();
-        Arrays.sort(matches, new Comparator<PercolateResponse.Match>() {
-            @Override
-            public int compare(PercolateResponse.Match a, PercolateResponse.Match b) {
-                return a.getId().compareTo(b.getId());
-            }
-        });
-
-        assertThat(matches[0].getScore(), equalTo(5.5f));
-        assertThat(matches[0].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick <em>brown</em> <em>fox</em> jumps over the lazy dog"));
-        assertThat(matches[1].getScore(), equalTo(5.5f));
-        assertThat(matches[1].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the <em>lazy</em> <em>dog</em>"));
-        assertThat(matches[2].getScore(), equalTo(5.5f));
-        assertThat(matches[2].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox <em>jumps</em> over the lazy dog"));
-        assertThat(matches[3].getScore(), equalTo(5.5f));
-        assertThat(matches[3].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown fox jumps over the lazy <em>dog</em>"));
-        assertThat(matches[4].getScore(), equalTo(5.5f));
-        assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
-    }
-
-    public void testPercolateNonMatchingConstantScoreQuery() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("doc", "message", "type=text"));
-        ensureGreen();
-
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject()
-                        .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery()
-                                .must(QueryBuilders.queryStringQuery("root"))
-                                .must(QueryBuilders.termQuery("message", "tree"))))
-                        .endObject())
-                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
-                .get();
-        refresh();
-
-        PercolateResponse percolate = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("doc")
-                .setSource(jsonBuilder().startObject()
-                        .startObject("doc").field("message", "A new bonsai tree ").endObject()
-                        .endObject())
-                .execute().actionGet();
-        assertNoFailures(percolate);
-        assertMatchCount(percolate, 0L);
-    }
-
-    public void testNestedPercolation() throws IOException {
-        initNestedIndexAndPercolation();
-        PercolateResponse response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
-        assertEquals(response.getMatches().length, 0);
-        response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
-        assertEquals(response.getMatches().length, 1);
-        assertEquals(response.getMatches()[0].getId().string(), "Q");
-    }
-
-    public void testNonNestedDocumentDoesNotTriggerAssertion() throws IOException {
-        initNestedIndexAndPercolation();
-        XContentBuilder doc = jsonBuilder();
-        doc.startObject();
-        doc.field("some_unnested_field", "value");
-        doc.endObject();
-        PercolateResponse response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(doc)).setIndices(INDEX_NAME).setDocumentType("company").get();
-        assertNoFailures(response);
-    }
-
-    public void testNestedPercolationOnExistingDoc() throws IOException {
-        initNestedIndexAndPercolation();
-        client().prepareIndex(INDEX_NAME, "company", "notmatching").setSource(getNotMatchingNestedDoc()).get();
-        client().prepareIndex(INDEX_NAME, "company", "matching").setSource(getMatchingNestedDoc()).get();
-        refresh();
-        PercolateResponse response = preparePercolate(client()).setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("notmatching")).setDocumentType("company").setIndices(INDEX_NAME).get();
-        assertEquals(response.getMatches().length, 0);
-        response = preparePercolate(client()).setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("matching")).setDocumentType("company").setIndices(INDEX_NAME).get();
-        assertEquals(response.getMatches().length, 1);
-        assertEquals(response.getMatches()[0].getId().string(), "Q");
-    }
-
-    public void testDontReportDeletedPercolatorDocs() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .execute().actionGet();
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .get();
-        refresh();
-
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "value").endObject()))
-                .setPercolateQuery(QueryBuilders.matchAllQuery())
-                .get();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1"));
-    }
-
-    public void testAddQueryWithNoMapping() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        try {
-            client().prepareIndex(INDEX_NAME, TYPE_NAME)
-                    .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "value")).endObject())
-                    .get();
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getRootCause(), instanceOf(QueryShardException.class));
-        }
-
-        try {
-            client().prepareIndex(INDEX_NAME, TYPE_NAME)
-                    .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(0).to(1)).endObject())
-                    .get();
-            fail();
-        } catch (MapperParsingException e) {
-            assertThat(e.getRootCause(), instanceOf(QueryShardException.class));
-        }
-    }
-
-    void initNestedIndexAndPercolation() throws IOException {
-        XContentBuilder mapping = XContentFactory.jsonBuilder();
-        mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
-                .startObject("employee").field("type", "nested").startObject("properties")
-                .startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
-                .endObject();
-
-        assertAcked(client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping("company", mapping)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-        );
-        ensureGreen(INDEX_NAME);
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "Q").setSource(jsonBuilder().startObject()
-                .field("query", QueryBuilders.nestedQuery("employee", QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)).endObject()).get();
-
-        refresh();
-
-    }
-
-    XContentBuilder getMatchingNestedDoc() throws IOException {
-        XContentBuilder doc = XContentFactory.jsonBuilder();
-        doc.startObject().field("companyname", "stark").startArray("employee")
-                .startObject().field("name", "virginia potts").endObject()
-                .startObject().field("name", "tony stark").endObject()
-                .endArray().endObject();
-        return doc;
-    }
-
-    XContentBuilder getNotMatchingNestedDoc() throws IOException {
-        XContentBuilder doc = XContentFactory.jsonBuilder();
-        doc.startObject().field("companyname", "notstark").startArray("employee")
-                .startObject().field("name", "virginia stark").endObject()
-                .startObject().field("name", "tony potts").endObject()
-                .endArray().endObject();
-        return doc;
-    }
-
-    public void testNestedDocFilter() throws IOException {
-        String mapping = "{\n" +
-                "    \"doc\": {\n" +
-                "      \"properties\": {\n" +
-                "        \"name\": {\"type\":\"text\"},\n" +
-                "        \"persons\": {\n" +
-                "          \"type\": \"nested\"\n," +
-                "          \"properties\" : {\"foo\" : {\"type\" : \"text\"}}" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  }";
-        String doc = "{\n" +
-                "    \"name\": \"obama\",\n" +
-                "    \"persons\": [\n" +
-                "      {\n" +
-                "        \"foo\": \"bar\"\n" +
-                "      }\n" +
-                "    ]\n" +
-                "  }";
-        String q1 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must\": {\n" +
-                "        \"match\": {\n" +
-                "          \"name\": \"obama\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        String q2 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must_not\": {\n" +
-                "        \"match\": {\n" +
-                "          \"name\": \"obama\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        String q3 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must\": {\n" +
-                "        \"match\": {\n" +
-                "          \"persons.foo\": \"bar\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        String q4 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must_not\": {\n" +
-                "        \"match\": {\n" +
-                "          \"persons.foo\": \"bar\"\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        String q5 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must\": {\n" +
-                "        \"nested\": {\n" +
-                "          \"path\": \"persons\",\n" +
-                "          \"query\": {\n" +
-                "            \"match\": {\n" +
-                "              \"persons.foo\": \"bar\"\n" +
-                "            }\n" +
-                "          }\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        String q6 = "{\n" +
-                "  \"query\": {\n" +
-                "    \"bool\": {\n" +
-                "      \"must_not\": {\n" +
-                "        \"nested\": {\n" +
-                "          \"path\": \"persons\",\n" +
-                "          \"query\": {\n" +
-                "            \"match\": {\n" +
-                "              \"persons.foo\": \"bar\"\n" +
-                "            }\n" +
-                "          }\n" +
-                "        }\n" +
-                "      }\n" +
-                "    }\n" +
-                "  },\n" +
-                "\"text\":\"foo\""+
-                "}";
-        assertAcked(client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("doc", mapping));
-        ensureGreen(INDEX_NAME);
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q1).setId("q1").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q2).setId("q2").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q3).setId("q3").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q4).setId("q4").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q5).setId("q5").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q6).setId("q6").get();
-        refresh();
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("doc")
-                .setPercolateDoc(docBuilder().setDoc(doc))
-                .get();
-        assertMatchCount(response, 3L);
-        Set<String> expectedIds = new HashSet<>();
-        expectedIds.add("q1");
-        expectedIds.add("q4");
-        expectedIds.add("q5");
-        for (PercolateResponse.Match match : response.getMatches()) {
-            assertTrue(expectedIds.remove(match.getId().string()));
-        }
-        assertTrue(expectedIds.isEmpty());
-        response = preparePercolate(client()).setOnlyCount(true)
-                .setIndices(INDEX_NAME).setDocumentType("doc")
-                .setPercolateDoc(docBuilder().setDoc(doc))
-                .get();
-        assertMatchCount(response, 3L);
-        response = preparePercolate(client()).setScore(randomBoolean()).setSortByScore(randomBoolean()).setOnlyCount(randomBoolean()).setPercolateQuery(QueryBuilders.termQuery("text", "foo"))
-                .setIndices(INDEX_NAME).setDocumentType("doc")
-                .setPercolateDoc(docBuilder().setDoc(doc))
-                .get();
-        assertMatchCount(response, 3L);
-    }
-
-    public void testMapUnmappedFieldAsString() throws IOException{
-        // If index.percolator.map_unmapped_fields_as_string is set to true, unmapped field is mapped as an analyzed string.
-        Settings.Builder settings = Settings.builder()
-                .put(indexSettings())
-                .put("index.percolator.map_unmapped_fields_as_string", true);
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .setSettings(settings));
-        client().prepareIndex(INDEX_NAME, TYPE_NAME)
-                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()).get();
-        refresh();
-        logger.info("--> Percolate doc with field1=value");
-        PercolateResponse response1 = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value").endObject()))
-                .execute().actionGet();
-        assertMatchCount(response1, 1L);
-        assertThat(response1.getMatches(), arrayWithSize(1));
-    }
-
-    public void testGeoShapeWithMapUnmappedFieldAsString() throws Exception {
-        // If index.percolator.map_unmapped_fields_as_string is set to true, unmapped field is mapped as an analyzed string.
-        Settings.Builder settings = Settings.builder()
-            .put(indexSettings())
-            .put("index.percolator.map_unmapped_fields_as_string", true);
-        assertAcked(prepareCreate(INDEX_NAME)
-            .setSettings(settings)
-            .addMapping(TYPE_NAME, "query", "type=percolator")
-            .addMapping("type", "location", "type=geo_shape"));
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-            .setSource(jsonBuilder().startObject().field("query", geoShapeQuery("location", ShapeBuilders.newEnvelope(new Coordinate(0d, 50d), new Coordinate(2d, 40d)))).endObject())
-            .get();
-        refresh();
-
-        PercolateResponse response1 = preparePercolate(client())
-            .setIndices(INDEX_NAME).setDocumentType("type")
-            .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject()
-                .startObject("location")
-                    .field("type", "point")
-                    .field("coordinates", Arrays.asList(1.44207d, 43.59959d))
-                .endObject()
-                .endObject()))
-            .execute().actionGet();
-        assertMatchCount(response1, 1L);
-        assertThat(response1.getMatches().length, equalTo(1));
-        assertThat(response1.getMatches()[0].getId().string(), equalTo("1"));
-    }
-
-    public void testFailParentChild() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("child", "_parent", "type=parent").addMapping("parent"));
-        Exception e = expectThrows(MapperParsingException.class, () -> client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", hasChildQuery("child", matchAllQuery(), ScoreMode.None)).endObject())
-                .get());
-        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
-        assertThat(e.getCause().getMessage(), equalTo("the [has_child] query is unsupported inside a percolator query"));
-    }
-
-    public void testPercolateDocumentWithParentField() throws Exception {
-        assertAcked(prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .addMapping("child", "_parent", "type=parent").addMapping("parent"));
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        // Just percolating a document that has a _parent field in its mapping should just work:
-        PercolateResponse response = preparePercolate(client())
-                .setDocumentType("parent")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("field", "value"))
-                .get();
-        assertMatchCount(response, 1);
-        assertThat(response.getMatches()[0].getId().string(), equalTo("1"));
-    }
-
-    public void testFilterByNow() throws Exception {
-        prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).field("created", "2015-07-10T14:41:54+0000").endObject())
-                .get();
-        refresh();
-
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME)
-                .setDocumentType("type")
-                .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
-                .setPercolateQuery(rangeQuery("created").lte("now"))
-                .get();
-        assertMatchCount(response, 1);
-    }
-}
-
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index 665b9926a58..07fc9aa2fe6 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -19,6 +19,8 @@
 package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
@@ -26,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
@@ -48,6 +51,8 @@ import java.util.Map;
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
+import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -58,16 +63,13 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
-import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
-import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
-import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
-import static org.hamcrest.Matchers.arrayWithSize;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.core.IsNull.notNullValue;
 
 public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
@@ -87,19 +89,17 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
     public void testPercolateScriptQuery() throws IOException {
         client().admin().indices().prepareCreate("index").addMapping("type", "query", "type=percolator").get();
-        ensureGreen();
         client().prepareIndex("index", "type", "1")
             .setSource(jsonBuilder().startObject().field("query", QueryBuilders.scriptQuery(
                 new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1==1", Collections.emptyMap()))).endObject())
             .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
             .execute().actionGet();
-        PercolateResponse response = preparePercolate(client())
-            .setIndices("index").setDocumentType("type")
-            .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
-            .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(response.getMatches(), arrayWithSize(1));
-        assertThat(convertFromTextArray(response.getMatches(), "index"), arrayContainingInAnyOrder("1"));
+        SearchResponse response = client().prepareSearch("index")
+            .setQuery(new PercolateQueryBuilder("query", "type", jsonBuilder().startObject().field("field1", "b").endObject().bytes(),
+                XContentType.JSON))
+            .get();
+        assertHitCount(response, 1);
+        assertSearchHits(response, "1");
     }
 
     public void testPercolatorQuery() throws Exception {
@@ -124,7 +124,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         BytesReference source = jsonBuilder().startObject().endObject().bytes();
         logger.info("percolating empty doc");
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@@ -132,7 +132,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         source = jsonBuilder().startObject().field("field1", "value").endObject().bytes();
         logger.info("percolating doc with 1 field");
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 2);
@@ -142,7 +142,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes();
         logger.info("percolating doc with 2 fields");
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 3);
@@ -201,7 +201,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         // Test long range:
         BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("3"));
@@ -209,7 +209,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         source = jsonBuilder().startObject().field("field1", 11).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@@ -217,7 +217,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         // Test double range:
         source = jsonBuilder().startObject().field("field2", 12).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("6"));
@@ -225,7 +225,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         source = jsonBuilder().startObject().field("field2", 11).endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("4"));
@@ -233,7 +233,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         // Test IP range:
         source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("9"));
@@ -241,7 +241,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes();
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("7"));
@@ -265,7 +265,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                         .must(matchQuery("field2", "value"))
                 ).endObject()).get();
 
-        client().prepareIndex("test", "type", "1").setSource("{}").get();
+        client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get();
         client().prepareIndex("test", "type", "2").setSource("field1", "value").get();
         client().prepareIndex("test", "type", "3").setSource("field1", "value", "field2", "value").get();
         client().admin().indices().prepareRefresh().get();
@@ -307,7 +307,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
             .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
             .get();
 
-        client().prepareIndex("test", "type", "1").setSource("{}").get();
+        client().prepareIndex("test", "type", "1").setSource("{}", XContentType.JSON).get();
         client().admin().indices().prepareRefresh().get();
 
         logger.info("percolating empty doc with source disabled");
@@ -378,7 +378,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .field("field2", "the quick brown fox falls down into the well")
                 .endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source, XContentType.JSON))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 4);
@@ -425,7 +425,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .field("field1", "The quick brown fox jumps over the lazy dog")
                 .endObject().bytes();
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("query", "type", document))
+                .setQuery(new PercolateQueryBuilder("query", "type", document, XContentType.JSON))
                 .highlighter(new HighlightBuilder().field("field1"))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
@@ -459,7 +459,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         client().admin().indices().prepareRefresh().get();
 
         SearchResponse response = client().prepareSearch().setQuery(
-                new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"))
+                new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"), XContentType.JSON)
         ).get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
@@ -519,7 +519,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         BytesReference source = jsonBuilder().startObject().field("field", "value").endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source, XContentType.JSON))
                 .setIndices("test1")
                 .get();
         assertHitCount(response, 1);
@@ -528,7 +528,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         assertThat(response.getHits().getAt(0).index(), equalTo("test1"));
 
         response = client().prepareSearch()
-                .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source, XContentType.JSON))
                 .setIndices("test2")
                 .get();
         assertHitCount(response, 1);
@@ -578,7 +578,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                                     .startObject().field("name", "virginia potts").endObject()
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
-                            .endObject().bytes()))
+                            .endObject().bytes(), XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 1);
@@ -592,17 +592,96 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                                     .startObject().field("name", "virginia stark").endObject()
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
-                            .endObject().bytes()))
+                            .endObject().bytes(), XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
 
         response = client().prepareSearch()
                 .setQuery(new PercolateQueryBuilder("query", "employee",
-                        XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes()))
+                        XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes(),
+                    XContentType.JSON))
                 .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
     }
 
+    public void testPercolatorQueryViaMultiSearch() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+            .addMapping("type", "field1", "type=text")
+            .addMapping("queries", "query", "type=percolator")
+        );
+
+        client().prepareIndex("test", "queries", "1")
+            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "b")).field("a", "b").endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "2")
+            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "c")).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "3")
+            .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                .must(matchQuery("field1", "b"))
+                .must(matchQuery("field1", "c"))
+            ).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "queries", "4")
+            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+            .execute().actionGet();
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("field1", "c").endObject())
+            .execute().actionGet();
+        client().admin().indices().prepareRefresh().get();
+
+        MultiSearchResponse response = client().prepareMultiSearch()
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    jsonBuilder().startObject().field("field1", "b").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    yamlBuilder().startObject().field("field1", "c").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    smileBuilder().startObject().field("field1", "b c").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type",
+                    jsonBuilder().startObject().field("field1", "d").endObject().bytes(), XContentType.JSON)))
+            .add(client().prepareSearch("test")
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null)))
+            .add(client().prepareSearch("test") // non existing doc, so error element
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "2", null, null, null)))
+            .get();
+
+        MultiSearchResponse.Item item = response.getResponses()[0];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "1", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[1];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "2", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[2];
+        assertHitCount(item.getResponse(), 4L);
+        assertSearchHits(item.getResponse(), "1", "2", "3", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[3];
+        assertHitCount(item.getResponse(), 1L);
+        assertSearchHits(item.getResponse(), "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[4];
+        assertHitCount(item.getResponse(), 2L);
+        assertSearchHits(item.getResponse(), "2", "4");
+        assertThat(item.getFailureMessage(), nullValue());
+
+        item = response.getResponses()[5];
+        assertThat(item.getResponse(), nullValue());
+        assertThat(item.getFailureMessage(), notNullValue());
+        assertThat(item.getFailureMessage(), equalTo("all shards failed"));
+        assertThat(ExceptionsHelper.unwrapCause(item.getFailure().getCause()).getMessage(),
+            containsString("[test/type/2] couldn't be found"));
+    }
+
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
deleted file mode 100644
index 456c2508d8e..00000000000
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.percolator;
-
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.Strings;
-import org.junit.Assert;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.formatShardStatus;
-
-/** Static method pulled out of PercolatorIT, used by other tests */
-public class PercolatorTestUtil extends Assert {
-
-    public static PercolateRequestBuilder preparePercolate(ElasticsearchClient client) {
-        return new PercolateRequestBuilder(client, PercolateAction.INSTANCE);
-    }
-
-    public static MultiPercolateRequestBuilder prepareMultiPercolate(ElasticsearchClient client) {
-        return new MultiPercolateRequestBuilder(client, MultiPercolateAction.INSTANCE);
-    }
-
-    public static void assertMatchCount(PercolateResponse percolateResponse, long expectedHitCount) {
-        if (percolateResponse.getCount() != expectedHitCount) {
-            fail("Count is " + percolateResponse.getCount() + " but " + expectedHitCount + " was expected. " +
-                    formatShardStatus(percolateResponse));
-        }
-        assertVersionSerializable(percolateResponse);
-    }
-
-    public static String[] convertFromTextArray(PercolateResponse.Match[] matches, String index) {
-        if (matches.length == 0) {
-            return Strings.EMPTY_ARRAY;
-        }
-        String[] strings = new String[matches.length];
-        for (int i = 0; i < matches.length; i++) {
-            assertEquals(index, matches[i].getIndex().string());
-            strings[i] = matches[i].getId().string();
-        }
-        return strings;
-    }
-
-}
diff --git a/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate1.json b/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate1.json
deleted file mode 100644
index 44079390bfc..00000000000
--- a/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate1.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{"percolate" : {"index" : "my-index1", "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local", "ignore_unavailable" : false}}
-{"doc" : {"field1" : "value1"}}
-{"percolate" : {"indices" : ["my-index2", "my-index3"], "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local", "ignore_unavailable" : true}}
-{"doc" : {"field1" : "value2"}}
-{"count" : {"indices" : ["my-index4", "my-index5"], "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local", "expand_wildcards" : "open,closed"}}
-{"doc" : {"field1" : "value3"}}
-{"percolate" : {"id" : "1", "index" : "my-index6", "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local", "expand_wildcards" : ["open", "closed"]}}
-{}
-{"count" : {"id" : "2", "index" : "my-index7", "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local"}}
-{}
-{"percolate" : {"index" : "my-index8", "type" : "my-type1", "routing" : "my-routing-1", "preference" : "primary"}}
-{"doc" : {"field1" : "value4"}}
-{"percolate" : {"id" : "3", "index" : "my-index9", "type" : "my-type1", "percolate_index": "percolate-index1", "percolate_type": "other-type", "percolate_preference": "_local", "percolate_routing": "percolate-routing-1"}}
-{}
-{"percolate" : {"id" : "4", "index" : "my-index10", "type" : "my-type1", "allow_no_indices": false, "expand_wildcards" : ["open"]}}
-{}
diff --git a/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate2.json b/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate2.json
deleted file mode 100644
index fa676cf6182..00000000000
--- a/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate2.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{"percolate" : {"routing" : "my-routing-1", "preference" : "_local"}}
-{"doc" : {"field1" : "value1"}}
-{"percolate" : {"index" : "my-index1", "type" : "my-type1", "routing" : "my-routing-1", "preference" : "_local", "ignore_unavailable" : true}}
-{"doc" : {"field1" : "value2"}}
-{"percolate" : {}}
-{"doc" : {"field1" : "value3"}}
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yaml
new file mode 100644
index 00000000000..d8f2b3264e8
--- /dev/null
+++ b/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yaml
@@ -0,0 +1,50 @@
+---
+"Test percolator basics via rest":
+  - do:
+      indices.create:
+        index: queries_index
+        body:
+          mappings:
+            queries:
+              properties:
+                query:
+                  type: percolator
+            test_type:
+              properties:
+                foo:
+                  type: keyword
+
+  - do:
+      index:
+        index: queries_index
+        type: queries
+        id:   test_percolator
+        body:
+          query:
+            match_all: {}
+
+  - do:
+        indices.refresh: {}
+
+  - do:
+      search:
+        body:
+          - query:
+              percolate:
+                document_type: test_type
+                field: query
+                document:
+                  foo: bar
+  - match:  { hits.total:     1  }
+
+  - do:
+      msearch:
+        body:
+          - index: queries_index
+          - query:
+              percolate:
+                document_type: test_type
+                field: query
+                document:
+                  foo: bar
+  - match:  { responses.0.hits.total:     1  }
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/mpercolate/10_basic.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/mpercolate/10_basic.yaml
deleted file mode 100644
index ab75d6abe29..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/mpercolate/10_basic.yaml
+++ /dev/null
@@ -1,56 +0,0 @@
----
-"Basic multi-percolate":
-
-  - do:
-      indices.create:
-        index: percolator_index
-        body:
-          mappings:
-            queries:
-              properties:
-                query:
-                  type: percolator
-
-  - do:
-      index:
-        index:  percolator_index
-        type:   my_type
-        id:     1
-        body:   {foo: bar}
-
-  - do:
-      index:
-        index: percolator_index
-        type: queries
-        id:   test_percolator
-        body:
-          query:
-            match_all: {}
-
-  - do:
-        indices.refresh: {}
-
-  - do:
-      mpercolate:
-        body:
-          - percolate:
-              index:  percolator_index
-              type:   my_type
-          - doc:
-              foo: bar
-          - percolate:
-              index:  percolator_index1
-              type:   my_type
-          - doc:
-              foo: bar
-          - percolate:
-              index:  percolator_index
-              type:   my_type
-              id:     1
-          - {}
-
-  - match:  { responses.0.total:     1  }
-  - match:  { responses.1.error.root_cause.0.type: index_not_found_exception }
-  - match:  { responses.1.error.root_cause.0.reason: "/no.such.index/" }
-  - match:  { responses.1.error.root_cause.0.index: percolator_index1 }
-  - match:  { responses.2.total:     1  }
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/15_new.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/15_new.yaml
deleted file mode 100644
index 45532d95757..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/15_new.yaml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-"Basic percolation tests":
-
-  - do:
-      indices.create:
-        index: test_index
-        body:
-          mappings:
-            queries:
-              properties:
-                query:
-                  type: percolator
-
-  - do:
-      index:
-        index: test_index
-        type: queries
-        id:   test_percolator
-        body:
-          query:
-            match_all: {}
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      percolate:
-        index: test_index
-        type:  test_type
-        body:
-          doc:
-            foo: bar
-
-  - match: {'total': 1}
-  - match: {'matches': [{_index: test_index, _id: test_percolator}]}
-
-  - do:
-      count_percolate:
-        index: test_index
-        type:  test_type
-        body:
-          doc:
-            foo: bar
-
-  - is_false:  matches
-  - match: {'total': 1}
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
deleted file mode 100644
index 2d15e2cd222..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
----
-"Percolate existing documents":
-
-  - do:
-      indices.create:
-        index: percolator_index
-        body:
-          mappings:
-            queries:
-              properties:
-                query:
-                  type: percolator
-
-  - do:
-      index:
-        index: percolator_index
-        type: queries
-        id:   test_percolator
-        body:
-          query:
-            match_all: {}
-          tag: tag1
-
-  - do:
-      index:
-        index:  percolator_index
-        type:   test_type
-        id:     1
-        body:
-          foo: bar
-
-  - do:
-      indices.create:
-        index: my_index
-
-  - do:
-      index:
-        index:  my_index
-        type:   my_type
-        id:     1
-        body:
-          foo: bar
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      percolate:
-        index: percolator_index
-        type:  test_type
-        id: 1
-
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator}]}
-
-  - do:
-      percolate:
-        index: my_index
-        type: my_type
-        id: 1
-        percolate_index: percolator_index
-        percolate_type: test_type
-
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator}]}
-
-
-  - do:
-      index:
-        index:  my_index
-        type:   my_type
-        id:     1
-        body:
-          foo: bar
-
-  - do:
-      percolate:
-        index: my_index
-        type: my_type
-        id: 1
-        version: 2
-        percolate_index: percolator_index
-        percolate_type: test_type
-
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator}]}
-
-  - do:
-      catch:        conflict
-      percolate:
-        index: my_index
-        type: my_type
-        id: 1
-        version: 1
-        percolate_index: percolator_index
-        percolate_type: test_type
-
-  - do:
-      percolate:
-        index: percolator_index
-        type:  test_type
-        id: 1
-        body:
-          filter:
-            term:
-              tag: non_existing_tag
-
-  - match: {'matches': []}
-
-  - do:
-      percolate:
-        index: percolator_index
-        type:  test_type
-        id: 1
-        body:
-          filter:
-            term:
-              tag: tag1
-
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator, _score: 0.2876821}]}
-
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/17_empty.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/17_empty.yaml
deleted file mode 100644
index a6a56bb4636..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/17_empty.yaml
+++ /dev/null
@@ -1,37 +0,0 @@
----
-"Basic percolation tests on an empty cluster":
-
-  - do:
-      indices.create:
-        index: test_index
-        body:
-          mappings:
-            queries:
-              properties:
-                query:
-                  type: percolator
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      percolate:
-        index: test_index
-        type:  test_type
-        body:
-          doc:
-            foo: bar
-
-  - match: {'total': 0}
-  - match: {'matches': []}
-
-  - do:
-      count_percolate:
-        index: test_index
-        type:  test_type
-        body:
-          doc:
-            foo: bar
-
-  - is_false:  matches
-  - match: {'total': 0}
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml
deleted file mode 100644
index 8f3287ddd02..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-"Basic percolation highlight query test":
-
-  - do:
-      indices.create:
-        index: test_index
-        body:
-            mappings:
-                type_1:
-                    properties:
-                        foo:
-                            type: text
-                queries:
-                    properties:
-                      query:
-                        type: percolator
-
-  - do:
-      index:
-        index: test_index
-        type: queries
-        id:  test_percolator
-        body:
-          query:
-            match:
-              foo: bar
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      percolate:
-        index: test_index
-        type:  type_1
-        body:
-          doc:
-              foo: "bar foo"
-          size: 1
-          highlight:
-              fields:
-                foo:
-                    highlight_query:
-                      match:
-                          foo: foo
-
-  - match: {'total': 1}
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/19_nested.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/19_nested.yaml
deleted file mode 100644
index eff66846da6..00000000000
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/19_nested.yaml
+++ /dev/null
@@ -1,117 +0,0 @@
----
-setup:
-  - do:
-      indices.create:
-          index: nestedindex
-          body:
-              mappings:
-                  company:
-                      properties:
-                          companyname:
-                              type: text
-                          employee:
-                              type: nested
-                              properties:
-                                  name:
-                                      type: text
-                  queries:
-                      properties:
-                        query:
-                          type: percolator
-
-
-  - do:
-      indices.refresh: {}
-  - do:
-      index:
-          index: nestedindex
-          type:  "queries"
-          id:    query
-          body:  { "query": { "nested": { "path": "employee", "score_mode": "avg", "query": { "match": { "employee.name": { "query": "virginia potts", "operator": "and"} } } } } }
-  - do:
-      indices.refresh: {}
-
-
----
-"Basic percolation tests on nested doc":
-  - do:
-      percolate:
-        index: nestedindex
-        type:  company
-        body: { "doc": { "companyname": "stark", "employee": [ { "name": "virginia stark"}, { "name": "tony potts"} ] } }
-
-  - match: {'total': 0}
-
-  - do:
-      percolate:
-          index: nestedindex
-          type:  company
-          body: { "doc": { "companyname": "stark", "employee": [ { "name": "virginia potts"}, { "name": "tony stark"} ] } }
-
-  - match: {'total': 1}
-
----
-"Percolate existing docs":
-  - do:
-      index:
-          index: nestedindex
-          type:  company
-          id:    notmatching
-          body: { "companyname": "stark", "employee": [ { "name": "virginia stark"}, { "name": "tony potts"} ] }
-
-
-  - do:
-      index:
-          index: nestedindex
-          type:  company
-          id:    matching
-          body: { "companyname": "stark", "employee": [ { "name": "virginia potts"}, { "name": "tony stark"} ] }
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      percolate:
-          index: nestedindex
-          type:  company
-          id: notmatching
-
-  - match: {'total': 0}
-
-  - do:
-      percolate:
-          index: nestedindex
-          type:  company
-          id: matching
-
-  - match: {'total': 1}
-
----
-"Test multi percolate":
-
-
-  - do:
-      mpercolate:
-         body:
-            - percolate: {"index": "nestedindex", "type": "company"}
-            - doc: { "companyname": "stark", "employee": [ { "name": "virginia stark"}, { "name": "tony potts"} ] }
-            - percolate: {"index": "nestedindex", "type": "company"}
-            - doc: { "companyname": "stark", "employee": [ { "name": "virginia potts"}, { "name": "tony stark"} ] }
-
-  - match: {'responses.0.total': 0}
-  - match: {'responses.1.total': 1}
-
-  - do:
-      mpercolate:
-         body:
-            - percolate: {"index": "nestedindex", "type": "company"}
-            - doc: { "companyname": "stark", "employee": [ { "name": "virginia potts"}, {"name": "tony stark"} ] }
-            - percolate: {"index": "nestedindex", "type": "company"}
-            - doc: { "companyname": "stark", "employee": [ { "name": "virginia stark"}, { "name": "tony potts"} ] }
-
-
-  - match: {'responses.0.total': 1}
-  - match: {'responses.1.total': 0}
-
-
-
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index d9c00065824..0b222c1a3cd 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.test-with-dependencies'
+
 esplugin {
   description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
   classname 'org.elasticsearch.index.reindex.ReindexPlugin'
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
deleted file mode 100644
index 0328e606d9e..00000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.DocWriteResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
-import org.elasticsearch.action.bulk.BackoffPolicy;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.bulk.Retry;
-import org.elasticsearch.client.ParentTaskAssigningClient;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
-import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.unmodifiableList;
-import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
-import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
-import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
-import static org.elasticsearch.rest.RestStatus.CONFLICT;
-import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
-
-/**
- * Abstract base for scrolling across a search and executing bulk actions on all results. All package private methods are package private so
- * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block.
- */
-public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>> {
-    protected final Logger logger;
-    protected final WorkingBulkByScrollTask task;
-    protected final ThreadPool threadPool;
-    /**
-     * The request for this action. Named mainRequest because we create lots of <code>request</code> variables all representing child
-     * requests of this mainRequest.
-     */
-    protected final Request mainRequest;
-
-    private final AtomicLong startTime = new AtomicLong(-1);
-    private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
-
-    private final ParentTaskAssigningClient client;
-    private final ActionListener<BulkIndexByScrollResponse> listener;
-    private final Retry bulkRetry;
-    private final ScrollableHitSource scrollSource;
-
-    public AbstractAsyncBulkByScrollAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                                           ThreadPool threadPool, Request mainRequest, ActionListener<BulkIndexByScrollResponse> listener) {
-        this.task = task;
-        this.logger = logger;
-        this.client = client;
-        this.threadPool = threadPool;
-        this.mainRequest = mainRequest;
-        this.listener = listener;
-        BackoffPolicy backoffPolicy = buildBackoffPolicy();
-        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry));
-        scrollSource = buildScrollableResultSource(backoffPolicy);
-        /*
-         * Default to sorting by doc. We can't do this in the request itself because it is normal to *add* to the sorts rather than replace
-         * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't
-         * another sort.
-         */
-        List<SortBuilder<?>> sorts = mainRequest.getSearchRequest().source().sorts();
-        if (sorts == null || sorts.isEmpty()) {
-            mainRequest.getSearchRequest().source().sort(fieldSort("_doc"));
-        }
-        mainRequest.getSearchRequest().source().version(needsSourceDocumentVersions());
-    }
-
-    /**
-     * Does this operation need the versions of the source documents?
-     */
-    protected abstract boolean needsSourceDocumentVersions();
-
-    protected abstract BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs);
-
-    protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) {
-        return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client,
-                mainRequest.getSearchRequest());
-    }
-
-    /**
-     * Build the response for reindex actions.
-     */
-    protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
-                                                      List<SearchFailure> searchFailures, boolean timedOut) {
-        return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
-    }
-
-    /**
-     * Start the action by firing the initial search request.
-     */
-    public void start() {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
-        try {
-            startTime.set(System.nanoTime());
-            scrollSource.start(response -> onScrollResponse(timeValueNanos(System.nanoTime()), 0, response));
-        } catch (Exception e) {
-            finishHim(e);
-        }
-    }
-
-    /**
-     * Process a scroll response.
-     * @param lastBatchStartTime the time when the last batch started. Used to calculate the throttling delay.
-     * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay.
-     * @param response the scroll response to process
-     */
-    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, ScrollableHitSource.Response response) {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
-        if (    // If any of the shards failed that should abort the request.
-                (response.getFailures().size() > 0)
-                // Timeouts aren't shard failures but we still need to pass them back to the user.
-                || response.isTimedOut()
-                ) {
-            refreshAndFinish(emptyList(), response.getFailures(), response.isTimedOut());
-            return;
-        }
-        long total = response.getTotalHits();
-        if (mainRequest.getSize() > 0) {
-            total = min(total, mainRequest.getSize());
-        }
-        task.setTotal(total);
-        AbstractRunnable prepareBulkRequestRunnable = new AbstractRunnable() {
-            @Override
-            protected void doRun() throws Exception {
-                /*
-                 * It is important that the batch start time be calculated from here, scroll response to scroll response. That way the time
-                 * waiting on the scroll doesn't count against this batch in the throttle.
-                 */
-                prepareBulkRequest(timeValueNanos(System.nanoTime()), response);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                finishHim(e);
-            }
-        };
-        prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable);
-        task.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable);
-    }
-
-    /**
-     * Prepare the bulk request. Called on the generic thread pool after some preflight checks have been done one the SearchResponse and any
-     * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
-     * thread may be blocked by the user script.
-     */
-    void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
-        if (response.getHits().isEmpty()) {
-            refreshAndFinish(emptyList(), emptyList(), false);
-            return;
-        }
-        task.countBatch();
-        List<? extends ScrollableHitSource.Hit> hits = response.getHits();
-        if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
-            // Truncate the hits if we have more than the request size
-            long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed());
-            if (remaining < hits.size()) {
-                hits = hits.subList(0, (int) remaining);
-            }
-        }
-        BulkRequest request = buildBulk(hits);
-        if (request.requests().isEmpty()) {
-            /*
-             * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
-             */
-            startNextScroll(thisBatchStartTime, 0);
-            return;
-        }
-        request.timeout(mainRequest.getTimeout());
-        request.waitForActiveShards(mainRequest.getWaitForActiveShards());
-        if (logger.isDebugEnabled()) {
-            logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(),
-                    new ByteSizeValue(request.estimatedSizeInBytes()));
-        }
-        sendBulkRequest(thisBatchStartTime, request);
-    }
-
-    /**
-     * Send a bulk request, handling retries.
-     */
-    void sendBulkRequest(TimeValue thisBatchStartTime, BulkRequest request) {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
-        bulkRetry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
-            @Override
-            public void onResponse(BulkResponse response) {
-                onBulkResponse(thisBatchStartTime, response);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                finishHim(e);
-            }
-        });
-    }
-
-    /**
-     * Processes bulk responses, accounting for failures.
-     */
-    void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
-        try {
-            List<Failure> failures = new ArrayList<Failure>();
-            Set<String> destinationIndicesThisBatch = new HashSet<>();
-            for (BulkItemResponse item : response) {
-                if (item.isFailed()) {
-                    recordFailure(item.getFailure(), failures);
-                    continue;
-                }
-                switch (item.getOpType()) {
-                    case CREATE:
-                    case INDEX:
-                        if (item.getResponse().getResult() == DocWriteResponse.Result.CREATED) {
-                            task.countCreated();
-                        } else {
-                            task.countUpdated();
-                        }
-                        break;
-                    case UPDATE:
-                        task.countUpdated();
-                        break;
-                    case DELETE:
-                        task.countDeleted();
-                        break;
-                }
-                // Track the indexes we've seen so we can refresh them if requested
-                destinationIndicesThisBatch.add(item.getIndex());
-            }
-
-            if (task.isCancelled()) {
-                finishHim(null);
-                return;
-            }
-
-            addDestinationIndices(destinationIndicesThisBatch);
-
-            if (false == failures.isEmpty()) {
-                refreshAndFinish(unmodifiableList(failures), emptyList(), false);
-                return;
-            }
-
-            if (mainRequest.getSize() != SIZE_ALL_MATCHES && task.getSuccessfullyProcessed() >= mainRequest.getSize()) {
-                // We've processed all the requested docs.
-                refreshAndFinish(emptyList(), emptyList(), false);
-                return;
-            }
-
-            startNextScroll(thisBatchStartTime, response.getItems().length);
-        } catch (Exception t) {
-            finishHim(t);
-        }
-    }
-
-    /**
-     * Start the next scroll request.
-     *
-     * @param lastBatchSize the number of requests sent in the last batch. This is used to calculate the throttling values which are applied
-     *        when the scroll returns
-     */
-    void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
-        TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, lastBatchSize);
-        scrollSource.startNextScroll(extraKeepAlive, response -> {
-            onScrollResponse(lastBatchStartTime, lastBatchSize, response);
-        });
-    }
-
-    private void recordFailure(Failure failure, List<Failure> failures) {
-        if (failure.getStatus() == CONFLICT) {
-            task.countVersionConflict();
-            if (false == mainRequest.isAbortOnVersionConflict()) {
-                return;
-            }
-        }
-        failures.add(failure);
-    }
-
-    /**
-     * Start terminating a request that finished non-catastrophically by refreshing the modified indices and then proceeding to
-     * {@link #finishHim(Exception, List, List, boolean)}.
-     */
-    void refreshAndFinish(List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
-        if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) {
-            finishHim(null, indexingFailures, searchFailures, timedOut);
-            return;
-        }
-        RefreshRequest refresh = new RefreshRequest();
-        refresh.indices(destinationIndices.toArray(new String[destinationIndices.size()]));
-        client.admin().indices().refresh(refresh, new ActionListener<RefreshResponse>() {
-            @Override
-            public void onResponse(RefreshResponse response) {
-                finishHim(null, indexingFailures, searchFailures, timedOut);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                finishHim(e);
-            }
-        });
-    }
-
-    /**
-     * Finish the request.
-     *
-     * @param failure if non null then the request failed catastrophically with this exception
-     */
-    void finishHim(Exception failure) {
-        finishHim(failure, emptyList(), emptyList(), false);
-    }
-
-    /**
-     * Finish the request.
-     * @param failure if non null then the request failed catastrophically with this exception
-     * @param indexingFailures any indexing failures accumulated during the request
-     * @param searchFailures any search failures accumulated during the request
-     * @param timedOut have any of the sub-requests timed out?
-     */
-    void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
-        scrollSource.close();
-        if (failure == null) {
-            listener.onResponse(
-                    buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures, timedOut));
-        } else {
-            listener.onFailure(failure);
-        }
-    }
-
-    /**
-     * Get the backoff policy for use with retries.
-     */
-    BackoffPolicy buildBackoffPolicy() {
-        return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries());
-    }
-
-    /**
-     * Add to the list of indices that were modified by this request. This is the list of indices refreshed at the end of the request if the
-     * request asks for a refresh.
-     */
-    void addDestinationIndices(Collection<String> indices) {
-        destinationIndices.addAll(indices);
-    }
-
-    /**
-     * Set the last returned scrollId. Exists entirely for testing.
-     */
-    void setScroll(String scroll) {
-        scrollSource.setScroll(scroll);
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 451b1199055..d70b3c9c4ce 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -21,16 +21,17 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.tasks.LoggingTaskListener;
 import org.elasticsearch.tasks.Task;
 
@@ -40,18 +41,13 @@ import java.util.Map;
 
 public abstract class AbstractBaseReindexRestHandler<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                A extends GenericAction<Request, BulkIndexByScrollResponse>
+                A extends GenericAction<Request, BulkByScrollResponse>
             > extends BaseRestHandler {
 
-    protected final SearchRequestParsers searchRequestParsers;
-    private final ClusterService clusterService;
     private final A action;
 
-    protected AbstractBaseReindexRestHandler(Settings settings, SearchRequestParsers searchRequestParsers,
-                                             ClusterService clusterService, A action) {
+    protected AbstractBaseReindexRestHandler(Settings settings, A action) {
         super(settings);
-        this.searchRequestParsers = searchRequestParsers;
-        this.clusterService = clusterService;
         this.action = action;
     }
 
@@ -80,7 +76,7 @@ public abstract class AbstractBaseReindexRestHandler<
         if (validationException != null) {
             throw validationException;
         }
-        return sendTask(client.executeLocally(action, internal, LoggingTaskListener.instance()));
+        return sendTask(client.getLocalNodeId(), client.executeLocally(action, internal, LoggingTaskListener.instance()));
     }
 
     /**
@@ -111,11 +107,11 @@ public abstract class AbstractBaseReindexRestHandler<
         return request;
     }
 
-    private RestChannelConsumer sendTask(Task task) throws IOException {
+    private RestChannelConsumer sendTask(String localNodeId, Task task) throws IOException {
         return channel -> {
             try (XContentBuilder builder = channel.newBuilder()) {
                 builder.startObject();
-                builder.field("task", clusterService.localNode().getId() + ":" + task.getId());
+                builder.field("task", localNodeId + ":" + task.getId());
                 builder.endObject();
                 channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
             }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
index 08204dc4a11..480ca80e2ee 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -21,32 +21,31 @@ package org.elasticsearch.index.reindex;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.search.SearchRequestParsers;
 
 import java.io.IOException;
 import java.util.Map;
 import java.util.function.Consumer;
 
-import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
+import static org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
 
 /**
  * Rest handler for reindex actions that accepts a search request like Update-By-Query or Delete-By-Query
  */
 public abstract class AbstractBulkByQueryRestHandler<
         Request extends AbstractBulkByScrollRequest<Request>,
-        A extends GenericAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
+        A extends GenericAction<Request, BulkByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
 
-    protected AbstractBulkByQueryRestHandler(Settings settings, SearchRequestParsers searchRequestParsers,
-                                             ClusterService clusterService, A action) {
-        super(settings, searchRequestParsers, clusterService, action);
+    protected AbstractBulkByQueryRestHandler(Settings settings, A action) {
+        super(settings, action);
     }
 
     protected void parseInternalRequest(Request internal, RestRequest restRequest,
@@ -65,8 +64,7 @@ public abstract class AbstractBulkByQueryRestHandler<
              * the generated parser probably is a noop but we should do the accounting just in case. It doesn't hurt to close twice but it
              * really hurts not to close if by some miracle we have to. */
             try {
-                RestSearchAction.parseSearchRequest(searchRequest, restRequest, searchRequestParsers, parseFieldMatcher,
-                        searchRequestParser);
+                RestSearchAction.parseSearchRequest(searchRequest, restRequest, searchRequestParser);
             } finally {
                 IOUtils.close(searchRequestParser);
             }
@@ -109,7 +107,7 @@ public abstract class AbstractBulkByQueryRestHandler<
             }
 
             try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
-                return parser.contentType().xContent().createParser(builder.map(body).bytes());
+                return parser.contentType().xContent().createParser(parser.getXContentRegistry(), builder.map(body).bytes());
             }
         } finally {
             parser.close();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
index 62c2635b301..6aad6e08a49 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index e1b32baa4af..369510083d1 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.script.Script;
@@ -30,7 +32,7 @@ public abstract class AbstractBulkIndexByScrollRequestBuilder<
         extends AbstractBulkByScrollRequestBuilder<Request, Self> {
 
     protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
+            Action<Request, BulkByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
         super(client, action, search, request);
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index f531d7ce6d2..a8d321a9fab 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -21,9 +21,10 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestResponse;
@@ -35,7 +36,7 @@ import java.util.Map;
 /**
  * RestBuilderListener that returns higher than 200 status if there are any failures and allows to set XContent.Params.
  */
-public class BulkIndexByScrollResponseContentListener extends RestBuilderListener<BulkIndexByScrollResponse> {
+public class BulkIndexByScrollResponseContentListener extends RestBuilderListener<BulkByScrollResponse> {
 
     private final Map<String, String> params;
 
@@ -45,14 +46,14 @@ public class BulkIndexByScrollResponseContentListener extends RestBuilderListene
     }
 
     @Override
-    public RestResponse buildResponse(BulkIndexByScrollResponse response, XContentBuilder builder) throws Exception {
+    public RestResponse buildResponse(BulkByScrollResponse response, XContentBuilder builder) throws Exception {
         builder.startObject();
         response.toXContent(builder, new ToXContent.DelegatingMapParams(params, channel.request()));
         builder.endObject();
         return new BytesRestResponse(getStatus(response), builder);
     }
 
-    private RestStatus getStatus(BulkIndexByScrollResponse response) {
+    private RestStatus getStatus(BulkByScrollResponse response) {
         /*
          * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
          * interesting to the user.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
index c789e9c77b4..b55fe33340e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> {
+public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> {
 
     public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
     public static final String NAME = "indices:data/write/delete/byquery";
@@ -37,7 +39,7 @@ public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkIndexB
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
index eb528793236..3efbc1a562c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -28,12 +31,12 @@ public class DeleteByQueryRequestBuilder extends
         AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, DeleteByQueryRequestBuilder> {
 
     public DeleteByQueryRequestBuilder(ElasticsearchClient client,
-                                       Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action) {
+                                       Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private DeleteByQueryRequestBuilder(ElasticsearchClient client,
-                                        Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action,
+                                        Action<DeleteByQueryRequest, BulkByScrollResponse, DeleteByQueryRequestBuilder> action,
                                         SearchRequestBuilder search) {
         super(client, action, search, new DeleteByQueryRequest(search.request()));
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
index aa16a859c1d..2c84cfc86be 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
@@ -20,9 +20,10 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class ReindexAction extends Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> {
+public class ReindexAction extends Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> {
     public static final ReindexAction INSTANCE = new ReindexAction();
     public static final String NAME = "indices:data/write/reindex";
 
@@ -36,7 +37,7 @@ public class ReindexAction extends Action<ReindexRequest, BulkIndexByScrollRespo
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index 707da4fe5da..fb203ee5c6d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -21,15 +21,24 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.tasks.Task;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -51,9 +60,14 @@ public class ReindexPlugin extends Plugin implements ActionPlugin {
     }
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Arrays.asList(RestReindexAction.class, RestUpdateByQueryAction.class, RestDeleteByQueryAction.class,
-                RestRethrottleAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(
+                new RestReindexAction(settings, restController),
+                new RestUpdateByQueryAction(settings, restController),
+                new RestDeleteByQueryAction(settings, restController),
+                new RestRethrottleAction(settings, restController, nodesInCluster));
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 618db3dfa48..2fa513f9c57 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -71,6 +71,9 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
         if (getSearchRequest().indices() == null || getSearchRequest().indices().length == 0) {
             e = addValidationError("use _all if you really want to copy from all existing indexes", e);
         }
+        if (getSearchRequest().source().fetchSource() != null && getSearchRequest().source().fetchSource().fetchSource() == false) {
+            e = addValidationError("_source:false is not supported in this context", e);
+        }
         /*
          * Note that we don't call index's validator - it won't work because
          * we'll be filling in portions of it as we receive the docs. But we can
@@ -125,7 +128,7 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
     }
 
     @Override
-    ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+    protected ReindexRequest forSlice(TaskId slicingTask, SearchRequest slice) {
         ReindexRequest sliced = doForSlice(new ReindexRequest(slice, destination, false), slicingTask);
         sliced.setRemoteInfo(remoteInfo);
         return sliced;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index 1eadf2c15bc..d7de51c588e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchAction;
@@ -32,13 +33,13 @@ public class ReindexRequestBuilder extends
     private final IndexRequestBuilder destination;
 
     public ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action) {
+            Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE),
                 new IndexRequestBuilder(client, IndexAction.INSTANCE));
     }
 
     private ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action,
+            Action<ReindexRequest, BulkByScrollResponse, ReindexRequestBuilder> action,
             SearchRequestBuilder search, IndexRequestBuilder destination) {
         super(client, action, search, new ReindexRequest(search.request(), destination.request()));
         this.destination = destination;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index ba6e6a942ba..0e11d64a405 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -20,14 +20,12 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.search.SearchRequestParsers;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -37,11 +35,8 @@ import java.util.function.Consumer;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, DeleteByQueryAction> {
-
-    @Inject
-    public RestDeleteByQueryAction(Settings settings, RestController controller,
-                                   SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, DeleteByQueryAction.INSTANCE);
+    public RestDeleteByQueryAction(Settings settings, RestController controller) {
+        super(settings, DeleteByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_delete_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_delete_by_query", this);
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index ad9cd334b88..57d3213da92 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -22,13 +22,9 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -44,7 +40,6 @@ import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.SearchRequestParsers;
 
 import java.io.IOException;
 import java.util.List;
@@ -62,11 +57,11 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
  * Expose reindex over rest.
  */
 public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> {
-    static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
+    static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex");
     private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)");
 
     static {
-        ObjectParser.Parser<ReindexRequest, ReindexParseContext> sourceParser = (parser, request, context) -> {
+        ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> {
             // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote.
             Map<String, Object> source = parser.map();
             String[] indices = extractStringArray(source, "index");
@@ -80,14 +75,12 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
             request.setRemoteInfo(buildRemoteInfo(source));
             XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
             builder.map(source);
-            try (XContentParser innerParser = parser.contentType().xContent().createParser(builder.bytes())) {
-                request.getSearchRequest().source().parseXContent(context.queryParseContext(innerParser),
-                        context.searchRequestParsers.aggParsers, context.searchRequestParsers.suggesters,
-                        context.searchRequestParsers.searchExtParsers);
+            try (XContentParser innerParser = parser.contentType().xContent().createParser(parser.getXContentRegistry(), builder.bytes())) {
+                request.getSearchRequest().source().parseXContent(new QueryParseContext(innerParser));
             }
         };
 
-        ObjectParser<IndexRequest, ParseFieldMatcherSupplier> destParser = new ObjectParser<>("dest");
+        ObjectParser<IndexRequest, Void> destParser = new ObjectParser<>("dest");
         destParser.declareString(IndexRequest::index, new ParseField("index"));
         destParser.declareString(IndexRequest::type, new ParseField("type"));
         destParser.declareString(IndexRequest::routing, new ParseField("routing"));
@@ -95,18 +88,16 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         destParser.declareString(IndexRequest::setPipeline, new ParseField("pipeline"));
         destParser.declareString((s, i) -> s.versionType(VersionType.fromString(i)), new ParseField("version_type"));
 
-        PARSER.declareField((p, v, c) -> sourceParser.parse(p, v, c), new ParseField("source"), ValueType.OBJECT);
+        PARSER.declareField(sourceParser::parse, new ParseField("source"), ValueType.OBJECT);
         PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ValueType.OBJECT);
         PARSER.declareInt(ReindexRequest::setSize, new ParseField("size"));
-        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p, c.getParseFieldMatcher())), new ParseField("script"),
+        PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p)), new ParseField("script"),
                 ValueType.OBJECT);
         PARSER.declareString(ReindexRequest::setConflicts, new ParseField("conflicts"));
     }
 
-    @Inject
-    public RestReindexAction(Settings settings, RestController controller,
-            SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, ReindexAction.INSTANCE);
+    public RestReindexAction(Settings settings, RestController controller) {
+        super(settings, ReindexAction.INSTANCE);
         controller.registerHandler(POST, "/_reindex", this);
     }
 
@@ -123,7 +114,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         }
         ReindexRequest internal = new ReindexRequest(new SearchRequest(), new IndexRequest());
         try (XContentParser parser = request.contentParser()) {
-            PARSER.parse(parser, internal, new ReindexParseContext(searchRequestParsers, parseFieldMatcher));
+            PARSER.parse(parser, internal, null);
         }
         return internal;
     }
@@ -222,23 +213,4 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         Map<String, Object> map = (Map<String, Object>) query;
         return builder.map(map).bytes();
     }
-
-    static class ReindexParseContext implements ParseFieldMatcherSupplier {
-        private final SearchRequestParsers searchRequestParsers;
-        private final ParseFieldMatcher parseFieldMatcher;
-
-        ReindexParseContext(SearchRequestParsers searchRequestParsers, ParseFieldMatcher parseFieldMatcher) {
-            this.searchRequestParsers = searchRequestParsers;
-            this.parseFieldMatcher = parseFieldMatcher;
-        }
-
-        QueryParseContext queryParseContext(XContentParser parser) {
-            return new QueryParseContext(searchRequestParsers.queryParsers, parser, parseFieldMatcher);
-        }
-
-        @Override
-        public ParseFieldMatcher getParseFieldMatcher() {
-            return this.parseFieldMatcher;
-        }
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index 7ada5b9c9cc..4aa46f1a077 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -20,8 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -29,17 +28,17 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.admin.cluster.RestListTasksAction.listTasksResponseListener;
 
 public class RestRethrottleAction extends BaseRestHandler {
-    private final ClusterService clusterService;
+    private final Supplier<DiscoveryNodes> nodesInCluster;
 
-    @Inject
-    public RestRethrottleAction(Settings settings, RestController controller, ClusterService clusterService) {
+    public RestRethrottleAction(Settings settings, RestController controller, Supplier<DiscoveryNodes> nodesInCluster) {
         super(settings);
-        this.clusterService = clusterService;
+        this.nodesInCluster = nodesInCluster;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_reindex/{taskId}/_rethrottle", this);
@@ -56,6 +55,6 @@ public class RestRethrottleAction extends BaseRestHandler {
         internalRequest.setRequestsPerSecond(requestsPerSecond);
         final String groupBy = request.param("group_by", "nodes");
         return channel ->
-            client.execute(RethrottleAction.INSTANCE, internalRequest, listTasksResponseListener(clusterService, groupBy, channel));
+            client.execute(RethrottleAction.INSTANCE, internalRequest, listTasksResponseListener(nodesInCluster, groupBy, channel));
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 8201f05f9ec..30161503c11 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -22,15 +22,11 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.elasticsearch.search.SearchRequestParsers;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -43,11 +39,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.script.Script.DEFAULT_SCRIPT_LANG;
 
 public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, UpdateByQueryAction> {
-
-    @Inject
-    public RestUpdateByQueryAction(Settings settings, RestController controller,
-            SearchRequestParsers searchRequestParsers, ClusterService clusterService) {
-        super(settings, searchRequestParsers, clusterService, UpdateByQueryAction.INSTANCE);
+    public RestUpdateByQueryAction(Settings settings, RestController controller) {
+        super(settings, UpdateByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_update_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_update_by_query", this);
     }
@@ -69,7 +62,7 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<Upda
 
         Map<String, Consumer<Object>> consumers = new HashMap<>();
         consumers.put("conflicts", o -> internal.setConflicts((String) o));
-        consumers.put("script", o -> internal.setScript(parseScript((Map<String, Object>)o, parseFieldMatcher)));
+        consumers.put("script", o -> internal.setScript(parseScript((Map<String, Object>)o)));
 
         parseInternalRequest(internal, request, consumers);
 
@@ -78,7 +71,7 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<Upda
     }
 
     @SuppressWarnings("unchecked")
-    static Script parseScript(Map<String, Object> config, ParseFieldMatcher parseFieldMatcher) {
+    private static Script parseScript(Map<String, Object> config) {
         String script = null;
         ScriptType type = null;
         String lang = DEFAULT_SCRIPT_LANG;
@@ -87,33 +80,33 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<Upda
             Map.Entry<String, Object> entry = itr.next();
             String parameterName = entry.getKey();
             Object parameterValue = entry.getValue();
-            if (parseFieldMatcher.match(parameterName, Script.LANG_PARSE_FIELD)) {
+            if (Script.LANG_PARSE_FIELD.match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     lang = (String) parameterValue;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, Script.PARAMS_PARSE_FIELD)) {
+            } else if (Script.PARAMS_PARSE_FIELD.match(parameterName)) {
                 if (parameterValue instanceof Map || parameterValue == null) {
                     params = (Map<String, Object>) parameterValue;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptType.INLINE.getParseField())) {
+            } else if (ScriptType.INLINE.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
                     type = ScriptType.INLINE;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptType.FILE.getParseField())) {
+            } else if (ScriptType.FILE.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
                     type = ScriptType.FILE;
                 } else {
                     throw new ElasticsearchParseException("Value must be of type String: [" + parameterName + "]");
                 }
-            } else if (parseFieldMatcher.match(parameterName, ScriptType.STORED.getParseField())) {
+            } else if (ScriptType.STORED.getParseField().match(parameterName)) {
                 if (parameterValue instanceof String || parameterValue == null) {
                     script = (String) parameterValue;
                     type = ScriptType.STORED;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
index e387a4b3bcb..a17b2b81f91 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -19,9 +19,13 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.bulk.byscroll.AsyncDeleteByQueryAction;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollParallelizationHelper;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
+import org.elasticsearch.action.bulk.byscroll.ParentBulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
@@ -36,7 +40,7 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkIndexByScrollResponse> {
+public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
@@ -52,70 +56,20 @@ public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteB
     }
 
     @Override
-    protected void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    public void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         if (request.getSlices() > 1) {
-            ReindexParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE, clusterService.localNode().getId(),
-                    (ParentBulkByScrollTask) task, request, listener);
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, DeleteByQueryAction.INSTANCE,
+                    clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener);
         } else {
             ClusterState state = clusterService.state();
             ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-            new AsyncDeleteBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService,
-                    state).start();
+            new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
         }
     }
 
     @Override
-    protected void doExecute(DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(DeleteByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
-
-    /**
-     * Implementation of delete-by-query using scrolling and bulk.
-     */
-    static class AsyncDeleteBySearchAction extends AbstractAsyncBulkIndexByScrollAction<DeleteByQueryRequest> {
-
-        public AsyncDeleteBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                ThreadPool threadPool, DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
-        }
-
-        @Override
-        protected boolean needsSourceDocumentVersions() {
-            /*
-             * We always need the version of the source document so we can report a version conflict if we try to delete it and it has been
-             * changed.
-             */
-            return true;
-        }
-
-        @Override
-        protected boolean accept(ScrollableHitSource.Hit doc) {
-            // Delete-by-query does not require the source to delete a document
-            // and the default implementation checks for it
-            return true;
-        }
-
-        @Override
-        protected RequestWrapper<DeleteRequest> buildRequest(ScrollableHitSource.Hit doc) {
-            DeleteRequest delete = new DeleteRequest();
-            delete.index(doc.getIndex());
-            delete.type(doc.getType());
-            delete.id(doc.getId());
-            delete.version(doc.getVersion());
-            return wrap(delete);
-        }
-
-        /**
-         * Overrides the parent {@link AbstractAsyncBulkIndexByScrollAction#copyMetadata(RequestWrapper, ScrollableHitSource.Hit)}
-         * method that is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
-         * don't care for a deletion.
-         */
-        @Override
-        protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
-            request.setParent(doc.getParent());
-            request.setRouting(doc.getRouting());
-            return request;
-        }
-    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 183b396b6de..deee6ae1160 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -37,6 +37,13 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollParallelizationHelper;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ParentBulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -55,9 +62,12 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.VersionFieldMapper;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
 import org.elasticsearch.script.Script;
@@ -66,6 +76,8 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -79,7 +91,7 @@ import static java.util.Collections.synchronizedList;
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
-public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkIndexByScrollResponse> {
+public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkByScrollResponse> {
     public static final Setting<List<String>> REMOTE_CLUSTER_WHITELIST =
             Setting.listSetting("reindex.remote.whitelist", emptyList(), Function.identity(), Property.NodeScope);
 
@@ -103,9 +115,9 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     }
 
     @Override
-    protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkByScrollResponse> listener) {
         if (request.getSlices() > 1) {
-            ReindexParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, clusterService.localNode().getId(),
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, ReindexAction.INSTANCE, clusterService.localNode().getId(),
                     (ParentBulkByScrollTask) task, request, listener);
         } else {
             checkRemoteWhitelist(remoteWhitelist, request.getRemoteInfo());
@@ -113,13 +125,13 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
             validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(),
                     indexNameExpressionResolver, autoCreateIndex, state);
             ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService,
-                    state).start();
+            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
         }
     }
 
     @Override
-    protected void doExecute(ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(ReindexRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
 
@@ -230,7 +242,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
      * but this makes no attempt to do any of them so it can be as simple
      * possible.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest> {
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction<ReindexRequest> {
         /**
          * List of threads created by this process. Usually actions don't create threads in Elasticsearch. Instead they use the builtin
          * {@link ThreadPool}s. But reindex-from-remote uses Elasticsearch's {@link RestClient} which doesn't use the
@@ -239,10 +251,10 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
          */
         private List<Thread> createdThreads = emptyList();
 
-        public AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                ThreadPool threadPool, ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                ThreadPool threadPool, ReindexRequest request, ScriptService scriptService, ClusterState clusterState,
+                ActionListener<BulkByScrollResponse> listener) {
+            super(task, logger, client, threadPool, request, scriptService, clusterState, listener);
         }
 
         @Override
@@ -267,7 +279,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         }
 
         @Override
-        void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
+        protected void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
             super.finishHim(failure, indexingFailures, searchFailures, timedOut);
             // A little extra paranoia so we log something if we leave any threads running
             for (Thread thread : createdThreads) {
@@ -279,7 +291,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+        public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new ReindexScriptApplier(task, scriptService, script, script.getParams());
@@ -315,7 +327,24 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
 
             // id and source always come from the found doc. Scripts can change them but they operate on the index request.
             index.id(doc.getId());
-            index.source(doc.getSource());
+
+            // the source xcontent type and destination could be different
+            final XContentType sourceXContentType = doc.getXContentType();
+            final XContentType mainRequestXContentType = mainRequest.getDestination().getContentType();
+            if (mainRequestXContentType != null && doc.getXContentType() != mainRequestXContentType) {
+                // we need to convert
+                try (XContentParser parser = sourceXContentType.xContent().createParser(NamedXContentRegistry.EMPTY, doc.getSource());
+                     XContentBuilder builder = XContentBuilder.builder(mainRequestXContentType.xContent())) {
+                    parser.nextToken();
+                    builder.copyCurrentStructure(parser);
+                    index.source(builder.bytes(), builder.contentType());
+                } catch (IOException e) {
+                    throw new UncheckedIOException("failed to convert hit from " + sourceXContentType + " to "
+                        + mainRequestXContentType, e);
+                }
+            } else {
+                index.source(doc.getSource(), doc.getXContentType());
+            }
 
             /*
              * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass
@@ -323,7 +352,6 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
              */
             index.routing(mainRequest.getDestination().routing());
             index.parent(mainRequest.getDestination().parent());
-            index.contentType(mainRequest.getDestination().getContentType());
             index.setPipeline(mainRequest.getDestination().getPipeline());
             // OpType is synthesized from version so it is handled when we copy version above.
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
index 9f27b521e9b..26786bbd74f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.client.Client;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 009ffabb73b..8ee98d1df0a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -21,6 +21,12 @@ package org.elasticsearch.index.reindex;
 
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ParentBulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollParallelizationHelper;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource;
+import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -46,7 +52,7 @@ import org.elasticsearch.transport.TransportService;
 import java.util.Map;
 import java.util.function.BiFunction;
 
-public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
     private final ClusterService clusterService;
@@ -63,32 +69,31 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     }
 
     @Override
-    protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         if (request.getSlices() > 1) {
-            ReindexParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE, clusterService.localNode().getId(),
-                    (ParentBulkByScrollTask) task, request, listener);
+            BulkByScrollParallelizationHelper.startSlices(client, taskManager, UpdateByQueryAction.INSTANCE,
+                    clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener);
         } else {
             ClusterState state = clusterService.state();
             ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService,
-                    state).start();
+            new AsyncIndexBySearchAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state,
+                    listener).start();
         }
     }
 
     @Override
-    protected void doExecute(UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+    protected void doExecute(UpdateByQueryRequest request, ActionListener<BulkByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
 
     /**
      * Simple implementation of update-by-query using scrolling and bulk.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> {
-
-        public AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
-                ThreadPool threadPool, UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
-                ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction<UpdateByQueryRequest> {
+        AsyncIndexBySearchAction(WorkingBulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
+                ThreadPool threadPool, UpdateByQueryRequest request, ScriptService scriptService, ClusterState clusterState,
+                ActionListener<BulkByScrollResponse> listener) {
+            super(task, logger, client, threadPool, request, scriptService, clusterState, listener);
         }
 
         @Override
@@ -101,7 +106,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
+        public BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new UpdateByQueryScriptApplier(task, scriptService, script, script.getParams());
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
index 0ff1b18bde0..cb716e82248 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryAction.java
@@ -20,10 +20,11 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class UpdateByQueryAction extends
-        Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> {
+        Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> {
     public static final UpdateByQueryAction INSTANCE = new UpdateByQueryAction();
     public static final String NAME = "indices:data/write/update/byquery";
 
@@ -37,7 +38,7 @@ public class UpdateByQueryAction extends
     }
 
     @Override
-    public BulkIndexByScrollResponse newResponse() {
-        return new BulkIndexByScrollResponse();
+    public BulkByScrollResponse newResponse() {
+        return new BulkByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 56ba3230133..3e7fac9d454 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -71,7 +71,7 @@ public class UpdateByQueryRequest extends AbstractBulkIndexByScrollRequest<Updat
     }
 
     @Override
-    UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
+    protected UpdateByQueryRequest forSlice(TaskId slicingTask, SearchRequest slice) {
         UpdateByQueryRequest request = doForSlice(new UpdateByQueryRequest(slice, false), slicingTask);
         request.setPipeline(pipeline);
         return request;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
index 311e138fb1f..143d66ae6fb 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -28,12 +29,12 @@ public class UpdateByQueryRequestBuilder extends
         AbstractBulkIndexByScrollRequestBuilder<UpdateByQueryRequest, UpdateByQueryRequestBuilder> {
 
     public UpdateByQueryRequestBuilder(ElasticsearchClient client,
-            Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> action) {
+            Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
     }
 
     private UpdateByQueryRequestBuilder(ElasticsearchClient client,
-            Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> action,
+            Action<UpdateByQueryRequest, BulkByScrollResponse, UpdateByQueryRequestBuilder> action,
             SearchRequestBuilder search) {
         super(client, action, search, new UpdateByQueryRequest(search.request()));
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
index 9ecb4700f58..a6afa6df39d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
@@ -30,6 +30,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -101,25 +102,32 @@ final class RemoteRequestBuilders {
             String storedFieldsParamName = remoteVersion.before(Version.V_5_0_0_alpha4) ? "fields" : "stored_fields";
             params.put(storedFieldsParamName, fields.toString());
         }
-        // We always want the _source document and this will force it to be returned.
-        params.put("_source", "true");
         return params;
     }
 
-    static HttpEntity initialSearchEntity(BytesReference query) {
-        try (XContentBuilder entity = JsonXContent.contentBuilder(); XContentParser queryParser = XContentHelper.createParser(query)) {
+    static HttpEntity initialSearchEntity(SearchRequest searchRequest, BytesReference query) {
+        // EMPTY is safe here because we're not calling namedObject
+        try (XContentBuilder entity = JsonXContent.contentBuilder();
+                XContentParser queryParser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, query)) {
             entity.startObject();
-            entity.field("query");
-            /*
-             * We're intentionally a bit paranoid here - copying the query as xcontent rather than writing a raw field. We don't want poorly
-             * written queries to escape. Ever.
-             */
-            entity.copyCurrentStructure(queryParser);
-            XContentParser.Token shouldBeEof = queryParser.nextToken();
-            if (shouldBeEof != null) {
-                throw new ElasticsearchException(
-                        "query was more than a single object. This first token after the object is [" + shouldBeEof + "]");
+
+            entity.field("query"); {
+                /* We're intentionally a bit paranoid here - copying the query as xcontent rather than writing a raw field. We don't want
+                 * poorly written queries to escape. Ever. */
+                entity.copyCurrentStructure(queryParser);
+                XContentParser.Token shouldBeEof = queryParser.nextToken();
+                if (shouldBeEof != null) {
+                    throw new ElasticsearchException(
+                            "query was more than a single object. This first token after the object is [" + shouldBeEof + "]");
+                }
             }
+
+            if (searchRequest.source().fetchSource() != null) {
+                entity.field("_source", searchRequest.source().fetchSource());
+            } else {
+                entity.field("_source", true);
+            }
+
             entity.endObject();
             BytesRef bytes = entity.bytes().toBytesRef();
             return new ByteArrayEntity(bytes.bytes, bytes.offset, bytes.length, ContentType.APPLICATION_JSON);
@@ -164,6 +172,22 @@ final class RemoteRequestBuilders {
     }
 
     static HttpEntity scrollEntity(String scroll) {
-        return new StringEntity(scroll, ContentType.TEXT_PLAIN);
+        try (XContentBuilder entity = JsonXContent.contentBuilder()) {
+            return new StringEntity(entity.startObject()
+                .field("scroll_id", scroll)
+                .endObject().string(), ContentType.APPLICATION_JSON);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to build scroll entity", e);
+        }
+    }
+
+    static HttpEntity clearScrollEntity(String scroll) {
+        try (XContentBuilder entity = JsonXContent.contentBuilder()) {
+            return new StringEntity(entity.startObject()
+                .array("scroll_id", scroll)
+                .endObject().string(), ContentType.APPLICATION_JSON);
+        } catch (IOException e) {
+            throw new ElasticsearchException("failed to build clear scroll entity", e);
+        }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
index 42e8a598edd..e9807bdfa5b 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
@@ -20,9 +20,13 @@
 package org.elasticsearch.index.reindex.remote;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.BasicHit;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Hit;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Response;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser;
@@ -30,11 +34,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.List;
@@ -55,7 +55,7 @@ final class RemoteResponseParsers {
     /**
      * Parser for an individual {@code hit} element.
      */
-    public static final ConstructingObjectParser<BasicHit, ParseFieldMatcherSupplier> HIT_PARSER =
+    public static final ConstructingObjectParser<BasicHit, XContentType> HIT_PARSER =
             new ConstructingObjectParser<>("hit", true, a -> {
                 int i = 0;
                 String index = (String) a[i++];
@@ -69,15 +69,16 @@ final class RemoteResponseParsers {
         HIT_PARSER.declareString(constructorArg(), new ParseField("_type"));
         HIT_PARSER.declareString(constructorArg(), new ParseField("_id"));
         HIT_PARSER.declareLong(optionalConstructorArg(), new ParseField("_version"));
-        HIT_PARSER.declareObject(BasicHit::setSource, (p, s) -> {
+        HIT_PARSER.declareObject(((basicHit, tuple) -> basicHit.setSource(tuple.v1(), tuple.v2())), (p, s) -> {
             try {
                 /*
                  * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for
                  * now this should do.
                  */
-                try (XContentBuilder b = JsonXContent.contentBuilder()) {
+                try (XContentBuilder b = XContentBuilder.builder(s.xContent())) {
                     b.copyCurrentStructure(p);
-                    return b.bytes();
+                    // a hack but this lets us get the right xcontent type to go with the source
+                    return new Tuple<>(b.bytes(), s);
                 }
             } catch (IOException e) {
                 throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e);
@@ -93,7 +94,7 @@ final class RemoteResponseParsers {
             String routing;
             String parent;
         }
-        ObjectParser<Fields, ParseFieldMatcherSupplier> fieldsParser = new ObjectParser<>("fields", Fields::new);
+        ObjectParser<Fields, XContentType> fieldsParser = new ObjectParser<>("fields", Fields::new);
         HIT_PARSER.declareObject((hit, fields) -> {
             hit.setRouting(fields.routing);
             hit.setParent(fields.parent);
@@ -106,7 +107,7 @@ final class RemoteResponseParsers {
     /**
      * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List<Hit>)]}.
      */
-    public static final ConstructingObjectParser<Object[], ParseFieldMatcherSupplier> HITS_PARSER =
+    public static final ConstructingObjectParser<Object[], XContentType> HITS_PARSER =
             new ConstructingObjectParser<>("hits", true, a -> a);
     static {
         HITS_PARSER.declareLong(constructorArg(), new ParseField("total"));
@@ -116,7 +117,7 @@ final class RemoteResponseParsers {
     /**
      * Parser for {@code failed} shards in the {@code _shards} elements.
      */
-    public static final ConstructingObjectParser<SearchFailure, ParseFieldMatcherSupplier> SEARCH_FAILURE_PARSER =
+    public static final ConstructingObjectParser<SearchFailure, XContentType> SEARCH_FAILURE_PARSER =
             new ConstructingObjectParser<>("failure", true, a -> {
                 int i = 0;
                 String index = (String) a[i++];
@@ -149,7 +150,7 @@ final class RemoteResponseParsers {
      * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it
      * parses to an empty list.
      */
-    public static final ConstructingObjectParser<List<Throwable>, ParseFieldMatcherSupplier> SHARDS_PARSER =
+    public static final ConstructingObjectParser<List<Throwable>, XContentType> SHARDS_PARSER =
             new ConstructingObjectParser<>("_shards", true, a -> {
                 @SuppressWarnings("unchecked")
                 List<Throwable> failures = (List<Throwable>) a[0];
@@ -160,7 +161,7 @@ final class RemoteResponseParsers {
         SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures"));
     }
 
-    public static final ConstructingObjectParser<Response, ParseFieldMatcherSupplier> RESPONSE_PARSER =
+    public static final ConstructingObjectParser<Response, XContentType> RESPONSE_PARSER =
             new ConstructingObjectParser<>("search_response", true, a -> {
                 int i = 0;
                 Throwable catastrophicFailure = (Throwable) a[i++];
@@ -199,9 +200,9 @@ final class RemoteResponseParsers {
      * Collects stuff about Throwables and attempts to rebuild them.
      */
     public static class ThrowableBuilder {
-        public static final BiFunction<XContentParser, ParseFieldMatcherSupplier, Throwable> PARSER;
+        public static final BiFunction<XContentParser, XContentType, Throwable> PARSER;
         static {
-            ObjectParser<ThrowableBuilder, ParseFieldMatcherSupplier> parser = new ObjectParser<>("reason", true, ThrowableBuilder::new);
+            ObjectParser<ThrowableBuilder, XContentType> parser = new ObjectParser<>("reason", true, ThrowableBuilder::new);
             PARSER = parser.andThen(ThrowableBuilder::build);
             parser.declareString(ThrowableBuilder::setType, new ParseField("type"));
             parser.declareString(ThrowableBuilder::setReason, new ParseField("reason"));
@@ -220,7 +221,7 @@ final class RemoteResponseParsers {
 
         public Throwable build() {
             Throwable t = buildWithoutCause();
-            if (causedBy != null) { 
+            if (causedBy != null) {
                 t.initCause(causedBy);
             }
             return t;
@@ -265,10 +266,10 @@ final class RemoteResponseParsers {
     /**
      * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out.
      */
-    public static final ConstructingObjectParser<Version, ParseFieldMatcherSupplier> MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<Version, XContentType> MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
             "/", true, a -> (Version) a[0]);
     static {
-        ConstructingObjectParser<Version, ParseFieldMatcherSupplier> versionParser = new ConstructingObjectParser<>(
+        ConstructingObjectParser<Version, XContentType> versionParser = new ConstructingObjectParser<>(
                 "version", true, a -> Version.fromString((String) a[0]));
         versionParser.declareString(constructorArg(), new ParseField("number"));
         MAIN_ACTION_PARSER.declareObject(constructorArg(), versionParser, new ParseField("version"));
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
index 97276f1b644..796106c269e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.reindex.remote;
 
 import org.apache.http.ContentTooLongException;
 import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -29,22 +30,21 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.ResponseListener;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.reindex.ScrollableHitSource;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -58,6 +58,7 @@ import java.util.function.Consumer;
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.clearScrollEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams;
 import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath;
@@ -81,27 +82,12 @@ public class RemoteScrollableHitSource extends ScrollableHitSource {
         this.client = client;
     }
 
-    @Override
-    public void close() {
-        super.close();
-        /* This might be called on the RestClient's thread pool and attempting to close the client on its own threadpool causes it to fail
-         * to close. So we always shutdown the RestClient asynchronously on a thread in Elasticsearch's generic thread pool. That way we
-         * never close the client in its own thread pool. */
-        threadPool.generic().submit(() -> {
-            try {
-                client.close();
-            } catch (IOException e) {
-                logger.error("Failed to shutdown the remote connection", e);
-            }
-        });
-    }
-
     @Override
     protected void doStart(Consumer<? super Response> onResponse) {
         lookupRemoteVersion(version -> {
             remoteVersion = version;
             execute("POST", initialSearchPath(searchRequest), initialSearchParams(searchRequest, version),
-                    initialSearchEntity(query), RESPONSE_PARSER, r -> onStartResponse(onResponse, r));
+                    initialSearchEntity(searchRequest, query), RESPONSE_PARSER, r -> onStartResponse(onResponse, r));
         });
     }
 
@@ -125,25 +111,53 @@ public class RemoteScrollableHitSource extends ScrollableHitSource {
     }
 
     @Override
-    protected void clearScroll(String scrollId) {
-        // Need to throw out response....
-        client.performRequestAsync("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() {
+    protected void clearScroll(String scrollId, Runnable onCompletion) {
+        client.performRequestAsync("DELETE", scrollPath(), emptyMap(), clearScrollEntity(scrollId), new ResponseListener() {
             @Override
             public void onSuccess(org.elasticsearch.client.Response response) {
                 logger.debug("Successfully cleared [{}]", scrollId);
+                onCompletion.run();
             }
 
             @Override
-            public void onFailure(Exception t) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), t);
+            public void onFailure(Exception e) {
+                logFailure(e);
+                onCompletion.run();
+            }
+
+            private void logFailure(Exception e) {
+                if (e instanceof ResponseException) {
+                    ResponseException re = (ResponseException) e;
+                    if (remoteVersion.before(Version.V_2_0_0) && re.getResponse().getStatusLine().getStatusCode() == 404) {
+                        logger.debug((Supplier<?>) () -> new ParameterizedMessage(
+                                "Failed to clear scroll [{}] from pre-2.0 Elasticsearch. This is normal if the request terminated "
+                                        + "normally as the scroll has already been cleared automatically.", scrollId), e);
+                        return;
+                    }
+                }
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to clear scroll [{}]", scrollId), e);
+            }
+        });
+    }
+
+    @Override
+    protected void cleanup() {
+        /* This is called on the RestClient's thread pool and attempting to close the client on its own threadpool causes it to fail to
+         * close. So we always shutdown the RestClient asynchronously on a thread in Elasticsearch's generic thread pool. */
+        threadPool.generic().submit(() -> {
+            try {
+                client.close();
+                logger.debug("Shut down remote connection");
+            } catch (IOException e) {
+                logger.error("Failed to shutdown the remote connection", e);
             }
         });
     }
 
     private <T> void execute(String method, String uri, Map<String, String> params, HttpEntity entity,
-            BiFunction<XContentParser, ParseFieldMatcherSupplier, T> parser, Consumer<? super T> listener) {
+                             BiFunction<XContentParser, XContentType, T> parser, Consumer<? super T> listener) {
         // Preserve the thread context so headers survive after the call
-        ThreadContext.StoredContext ctx = threadPool.getThreadContext().newStoredContext();
+        java.util.function.Supplier<ThreadContext.StoredContext> contextSupplier = threadPool.getThreadContext().newRestorableContext(true);
         class RetryHelper extends AbstractRunnable {
             private final Iterator<TimeValue> retries = backoffPolicy.iterator();
 
@@ -153,49 +167,69 @@ public class RemoteScrollableHitSource extends ScrollableHitSource {
                     @Override
                     public void onSuccess(org.elasticsearch.client.Response response) {
                         // Restore the thread context to get the precious headers
-                        ctx.restore();
-                        T parsedResponse;
-                        try {
-                            HttpEntity responseEntity = response.getEntity();
-                            InputStream content = responseEntity.getContent();
-                            XContentType xContentType = null;
-                            if (responseEntity.getContentType() != null) {
-                                 xContentType = XContentType.fromMediaTypeOrFormat(responseEntity.getContentType().getValue());
+                        try (ThreadContext.StoredContext ctx = contextSupplier.get()) {
+                            assert ctx != null; // eliminates compiler warning
+                            T parsedResponse;
+                            try {
+                                HttpEntity responseEntity = response.getEntity();
+                                InputStream content = responseEntity.getContent();
+                                XContentType xContentType = null;
+                                if (responseEntity.getContentType() != null) {
+                                    final String mimeType = ContentType.parse(responseEntity.getContentType().getValue()).getMimeType();
+                                    xContentType = XContentType.fromMediaType(mimeType);
+                                }
+                                if (xContentType == null) {
+                                    try {
+                                        throw new ElasticsearchException(
+                                            "Response didn't include Content-Type: " + bodyMessage(response.getEntity()));
+                                    } catch (IOException e) {
+                                        ElasticsearchException ee = new ElasticsearchException("Error extracting body from response");
+                                        ee.addSuppressed(e);
+                                        throw ee;
+                                    }
+                                }
+                                // EMPTY is safe here because we don't call namedObject
+                                try (XContentParser xContentParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY,
+                                    content)) {
+                                    parsedResponse = parser.apply(xContentParser, xContentType);
+                                } catch (ParsingException e) {
+                                /* Because we're streaming the response we can't get a copy of it here. The best we can do is hint that it
+                                 * is totally wrong and we're probably not talking to Elasticsearch. */
+                                    throw new ElasticsearchException(
+                                        "Error parsing the response, remote is likely not an Elasticsearch instance", e);
+                                }
+                            } catch (IOException e) {
+                                throw new ElasticsearchException(
+                                    "Error deserializing response, remote is likely not an Elasticsearch instance", e);
                             }
-                            if (xContentType == null) {
-                                //auto-detect as a fallback
-                                xContentType = XContentFactory.xContentType(content);
-                            }
-                            try(XContentParser xContentParser = xContentType.xContent().createParser(content)) {
-                                parsedResponse = parser.apply(xContentParser, () -> ParseFieldMatcher.STRICT);
-                            }
-                        } catch (IOException e) {
-                            throw new ElasticsearchException("Error deserializing response", e);
+                            listener.accept(parsedResponse);
                         }
-                        listener.accept(parsedResponse);
                     }
 
                     @Override
                     public void onFailure(Exception e) {
-                        if (e instanceof ResponseException) {
-                            ResponseException re = (ResponseException) e;
-                            if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) {
-                                if (retries.hasNext()) {
-                                    TimeValue delay = retries.next();
-                                    logger.trace(
-                                        (Supplier<?>) () -> new ParameterizedMessage("retrying rejected search after [{}]", delay), e);
-                                    countSearchRetry.run();
-                                    threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this);
-                                    return;
+                        try (ThreadContext.StoredContext ctx = contextSupplier.get()) {
+                            assert ctx != null; // eliminates compiler warning
+                            if (e instanceof ResponseException) {
+                                ResponseException re = (ResponseException) e;
+                                if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) {
+                                    if (retries.hasNext()) {
+                                        TimeValue delay = retries.next();
+                                        logger.trace(
+                                            (Supplier<?>) () -> new ParameterizedMessage("retrying rejected search after [{}]", delay), e);
+                                        countSearchRetry.run();
+                                        threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this);
+                                        return;
+                                    }
                                 }
-                            }
-                            e = wrapExceptionToPreserveStatus(re.getResponse().getStatusLine().getStatusCode(),
+                                e = wrapExceptionToPreserveStatus(re.getResponse().getStatusLine().getStatusCode(),
                                     re.getResponse().getEntity(), re);
-                        } else if (e instanceof ContentTooLongException) {
-                            e = new IllegalArgumentException(
+                            } else if (e instanceof ContentTooLongException) {
+                                e = new IllegalArgumentException(
                                     "Remote responded with a chunk that was too large. Use a smaller batch size.", e);
+                            }
+                            fail.accept(e);
                         }
-                        fail.accept(e);
                     }
                 });
             }
@@ -220,18 +254,20 @@ public class RemoteScrollableHitSource extends ScrollableHitSource {
             messagePrefix = "Couldn't extract status [" + statusCode + "]. ";
             status = RestStatus.INTERNAL_SERVER_ERROR;
         }
-        String message;
-        if (entity == null) {
-            message = messagePrefix + "No error body.";
-        } else {
-            try {
-                message = messagePrefix + "body=" + EntityUtils.toString(entity);
-            } catch (IOException ioe) {
-                ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause);
-                e.addSuppressed(ioe);
-                return e;
-            }
+        try {
+            return new ElasticsearchStatusException(messagePrefix + bodyMessage(entity), status, cause);
+        } catch (IOException ioe) {
+            ElasticsearchStatusException e = new ElasticsearchStatusException(messagePrefix + "Failed to extract body.", status, cause);
+            e.addSuppressed(ioe);
+            return e;
+        }
+    }
+
+    static String bodyMessage(@Nullable HttpEntity entity) throws IOException {
+        if (entity == null) {
+            return "No error body.";
+        } else {
+            return "body=" + EntityUtils.toString(entity);
         }
-        return new ElasticsearchStatusException(message, status, cause);
     }
 }
diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 00000000000..f9ce0f894d3
--- /dev/null
+++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+   // reindex opens socket connections using the rest client
+   permission java.net.SocketPermission "*", "connect";
+};
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
similarity index 81%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
rename to modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
index 36c392622bc..b565f0749f1 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollActionScriptTestCase.java
@@ -20,10 +20,14 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction.OpType;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction.RequestWrapper;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollActionTestCase;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.OpType;
-import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.RequestWrapper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -40,10 +44,10 @@ import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
+public abstract class AbstractAsyncBulkByScrollActionScriptTestCase<
                 Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
+                Response extends BulkByScrollResponse>
+        extends AbstractAsyncBulkByScrollActionTestCase<Request, Response> {
 
     private static final Script EMPTY_SCRIPT = new Script("");
 
@@ -62,8 +66,8 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
 
         when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
                 .thenReturn(executableScript);
-        AbstractAsyncBulkIndexByScrollAction<Request> action = action(scriptService, request().setScript(EMPTY_SCRIPT));
-        RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc);
+        AbstractAsyncBulkByScrollAction<Request> action = action(scriptService, request().setScript(EMPTY_SCRIPT));
+        RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkByScrollAction.wrap(index), doc);
         return (result != null) ? (T) result.self() : null;
     }
 
@@ -104,5 +108,5 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
         assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed"));
     }
 
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action(ScriptService scriptService, Request request);
+    protected abstract AbstractAsyncBulkByScrollAction<Request> action(ScriptService scriptService, Request request);
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java
index 4d0795bbd78..d33d8b2c877 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseTests.java
@@ -20,8 +20,10 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -35,7 +37,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 public class BulkIndexByScrollResponseTests extends ESTestCase {
     public void testMergeConstructor() {
         int mergeCount = between(2, 10);
-        List<BulkIndexByScrollResponse> responses = new ArrayList<>(mergeCount);
+        List<BulkByScrollResponse> responses = new ArrayList<>(mergeCount);
         int took = between(1000, 10000);
         int tookIndex = between(0, mergeCount - 1);
         List<BulkItemResponse.Failure> allBulkFailures = new ArrayList<>();
@@ -59,10 +61,10 @@ public class BulkIndexByScrollResponseTests extends ESTestCase {
             allSearchFailures.addAll(searchFailures);
             boolean thisTimedOut = rarely();
             timedOut |= thisTimedOut;
-            responses.add(new BulkIndexByScrollResponse(thisTook, status, bulkFailures, searchFailures, thisTimedOut));
+            responses.add(new BulkByScrollResponse(thisTook, status, bulkFailures, searchFailures, thisTimedOut));
         }
 
-        BulkIndexByScrollResponse merged = new BulkIndexByScrollResponse(responses, reasonCancelled);
+        BulkByScrollResponse merged = new BulkByScrollResponse(responses, reasonCancelled);
 
         assertEquals(timeValueMillis(took), merged.getTook());
         assertEquals(allBulkFailures, merged.getBulkFailures());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 92aa7e393e4..01c5977e822 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -22,13 +22,18 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.Operation.Origin;
 import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskInfo;
@@ -51,7 +56,7 @@ import static org.hamcrest.Matchers.hasSize;
 
 /**
  * Test that you can actually cancel a reindex/update-by-query/delete-by-query request and all the plumbing works. Doesn't test all of the
- * different cancellation places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to
+ * different cancellation places - that is the responsibility of AsyncBulkByScrollActionTests which have more precise control to
  * simulate failures but do not exercise important portion of the stack like transport and task management.
  */
 public class CancelTests extends ReindexTestCase {
@@ -103,7 +108,7 @@ public class CancelTests extends ReindexTestCase {
         ALLOWED_OPERATIONS.release(numModifiedDocs - builder.request().getSlices());
 
         // Now execute the reindex action...
-        ListenableActionFuture<? extends BulkIndexByScrollResponse> future = builder.execute();
+        ListenableActionFuture<? extends BulkByScrollResponse> future = builder.execute();
 
         /* ... and waits for the indexing operation listeners to block. It is important to realize that some of the workers might have
          * exhausted their slice while others might have quite a bit left to work on. We can't control that. */
@@ -155,7 +160,7 @@ public class CancelTests extends ReindexTestCase {
         });
 
         // And check the status of the response
-        BulkIndexByScrollResponse response = future.get();
+        BulkByScrollResponse response = future.get();
         assertThat(response.getReasonCancelled(), equalTo("by user request"));
         assertThat(response.getBulkFailures(), emptyIterable());
         assertThat(response.getSearchFailures(), emptyIterable());
@@ -200,7 +205,7 @@ public class CancelTests extends ReindexTestCase {
                 "      \"test\" : {}\n" +
                 "  } ]\n" +
                 "}");
-        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get());
+        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get());
 
         testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX), (response, total, modified) -> {
             assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")));
@@ -233,7 +238,7 @@ public class CancelTests extends ReindexTestCase {
                 "      \"test\" : {}\n" +
                 "  } ]\n" +
                 "}");
-        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline).get());
+        assertAcked(client().admin().cluster().preparePutPipeline("set-processed", pipeline, XContentType.JSON).get());
 
         testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-processed").source(INDEX).setSlices(5),
                 (response, total, modified) -> {
@@ -255,7 +260,7 @@ public class CancelTests extends ReindexTestCase {
      * Used to check the result of the cancel test.
      */
     private interface CancelAssertion {
-        void assertThat(BulkIndexByScrollResponse response, int total, int modified);
+        void assertThat(BulkByScrollResponse response, int total, int modified);
     }
 
     public static class ReindexCancellationPlugin extends Plugin {
@@ -269,12 +274,12 @@ public class CancelTests extends ReindexTestCase {
     public static class BlockingOperationListener implements IndexingOperationListener {
 
         @Override
-        public Engine.Index preIndex(Engine.Index index) {
+        public Engine.Index preIndex(ShardId shardId, Engine.Index index) {
             return preCheck(index, index.type());
         }
 
         @Override
-        public Engine.Delete preDelete(Engine.Delete delete) {
+        public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
             return preCheck(delete, delete.type());
         }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
index 40a776626a3..b80360f1260 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -96,7 +97,7 @@ public class DeleteByQueryConcurrentTests extends ReindexTestCase {
                 try {
                     start.await();
 
-                    BulkIndexByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
+                    BulkByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
                     // Some deletions might fail due to version conflict, but
                     // what matters here is the total of successful deletions
                     deleted.addAndGet(response.getDeleted());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index 950e98844d1..e7f9994b904 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 
 import java.util.ArrayList;
@@ -111,7 +112,7 @@ public class ReindexBasicTests extends ReindexTestCase {
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
         copy.size(half); // The real "size" of the request.
-        BulkIndexByScrollResponse response = copy.get();
+        BulkByScrollResponse response = copy.get();
         assertThat(response, matcher().created(lessThanOrEqualTo((long) half)).slices(hasSize(workers)));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), response.getCreated());
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
index 54483eae569..0975cad96a4 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class ReindexClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index d089f0427c3..47b79a499b1 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 
 import java.util.ArrayList;
@@ -56,7 +57,7 @@ public class ReindexFailureTests extends ReindexTestCase {
          */
         copy.source().setSize(1);
 
-        BulkIndexByScrollResponse response = copy.get();
+        BulkByScrollResponse response = copy.get();
         assertThat(response, matcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
@@ -76,7 +77,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         // CREATE will cause the conflict to prevent the write.
         copy.destination().setOpType(CREATE);
 
-        BulkIndexByScrollResponse response = copy.get();
+        BulkByScrollResponse response = copy.get();
         assertThat(response, matcher().batches(1).versionConflicts(1).failures(1).created(99));
         for (Failure failure: response.getBulkFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
@@ -98,7 +99,7 @@ public class ReindexFailureTests extends ReindexTestCase {
             indexDocs(100);
             ReindexRequestBuilder copy = reindex().source("source").destination("dest");
             copy.source().setSize(10);
-            Future<BulkIndexByScrollResponse> response = copy.execute();
+            Future<BulkByScrollResponse> response = copy.execute();
             client().admin().indices().prepareDelete("source").get();
 
             try {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
index dab0cab8d8a..172047bfef2 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
@@ -19,58 +19,79 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollActionMetadataTestCase;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Hit;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 
 /**
  * Index-by-search test for ttl, timestamp, and routing.
  */
-public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, BulkIndexByScrollResponse> {
+public class ReindexMetadataTests extends AbstractAsyncBulkByScrollActionMetadataTestCase<ReindexRequest, BulkByScrollResponse> {
     public void testRoutingCopiedByDefault() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     public void testRoutingCopiedIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("keep");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("keep");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     public void testRoutingDiscardedIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("discard");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("discard");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals(null, index.routing());
     }
 
     public void testRoutingSetIfRequested() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("=cat");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("=cat");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("cat", index.routing());
     }
 
     public void testRoutingSetIfWithDegenerateValue() throws Exception {
-        TransportReindexAction.AsyncIndexBySearchAction action = action();
-        action.mainRequest.getDestination().routing("==]");
+        TestAction action = action();
+        action.mainRequest().getDestination().routing("==]");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action.copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("=]", index.routing());
     }
 
     @Override
-    protected TransportReindexAction.AsyncIndexBySearchAction action() {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
+    protected TestAction action() {
+        return new TestAction();
     }
 
     @Override
     protected ReindexRequest request() {
         return new ReindexRequest(new SearchRequest(), new IndexRequest());
     }
+
+    private class TestAction extends TransportReindexAction.AsyncIndexBySearchAction {
+        TestAction() {
+            super(ReindexMetadataTests.this.task, ReindexMetadataTests.this.logger, null, ReindexMetadataTests.this.threadPool, request(),
+                    null, null, listener());
+        }
+
+        public ReindexRequest mainRequest() {
+            return this.mainRequest;
+        }
+
+        @Override
+        public AbstractAsyncBulkByScrollAction.RequestWrapper<?> copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper<?> request,
+                Hit doc) {
+            return super.copyMetadata(request, doc);
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
index e6572548609..e0155ca6cab 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestTestCase;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesArray;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
index 66b681b1494..4a61d733a80 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -31,7 +32,7 @@ import static org.hamcrest.Matchers.containsString;
 /**
  * Tests index-by-search with a script modifying the documents.
  */
-public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<ReindexRequest, BulkIndexByScrollResponse> {
+public class ReindexScriptTests extends AbstractAsyncBulkByScrollActionScriptTestCase<ReindexRequest, BulkByScrollResponse> {
 
     public void testSetIndex() throws Exception {
         Object dest = randomFrom(new Object[] {234, 234L, "pancake"});
@@ -109,8 +110,8 @@ public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScri
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest> action(ScriptService scriptService, ReindexRequest request) {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(), scriptService,
-                null);
+    protected TransportReindexAction.AsyncIndexBySearchAction action(ScriptService scriptService, ReindexRequest request) {
+        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, null,
+                listener());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index fcf80ea283c..e7dc7bf3a77 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkIndexByScrollResponseMatcher;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
index 1ab0613103f..b9b99ce1638 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.get.GetResponse;
 
 import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE;
@@ -88,7 +89,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with EXTERNAL versioning which has "refresh" semantics.
      */
-    private BulkIndexByScrollResponse reindexExternal() {
+    private BulkByScrollResponse reindexExternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(EXTERNAL);
         return reindex.get();
@@ -97,7 +98,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with INTERNAL versioning which has "overwrite" semantics.
      */
-    private BulkIndexByScrollResponse reindexInternal() {
+    private BulkByScrollResponse reindexInternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(INTERNAL);
         return reindex.get();
@@ -106,7 +107,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with CREATE OpType which has "create" semantics.
      */
-    private BulkIndexByScrollResponse reindexCreate() {
+    private BulkByScrollResponse reindexCreate() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setOpType(CREATE);
         return reindex.get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
index a5c68c768b2..b190aee3619 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
@@ -21,17 +21,13 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.reindex.RestReindexAction.ReindexParseContext;
 import org.elasticsearch.index.reindex.remote.RemoteInfo;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.RestController;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
@@ -125,18 +121,16 @@ public class RestReindexActionTests extends ESTestCase {
         }
         try (XContentParser p = createParser(JsonXContent.jsonXContent, request)) {
             ReindexRequest r = new ReindexRequest(new SearchRequest(), new IndexRequest());
-            SearchRequestParsers searchParsers = new SearchRequestParsers(new IndicesQueriesRegistry(), null, null, null);
-            RestReindexAction.PARSER.parse(p, r, new ReindexParseContext(searchParsers, ParseFieldMatcher.STRICT));
+            RestReindexAction.PARSER.parse(p, r, null);
             assertEquals("localhost", r.getRemoteInfo().getHost());
             assertArrayEquals(new String[] {"source"}, r.getSearchRequest().indices());
         }
     }
 
     public void testPipelineQueryParameterIsError() throws IOException {
-        SearchRequestParsers parsers = new SearchRequestParsers(new IndicesQueriesRegistry(), null, null, null);
-        RestReindexAction action = new RestReindexAction(Settings.EMPTY, mock(RestController.class), parsers, null);
+        RestReindexAction action = new RestReindexAction(Settings.EMPTY, mock(RestController.class));
 
-        FakeRestRequest.Builder request = new FakeRestRequest.Builder();
+        FakeRestRequest.Builder request = new FakeRestRequest.Builder(xContentRegistry());
         try (XContentBuilder body = JsonXContent.contentBuilder().prettyPrint()) {
             body.startObject(); {
                 body.startObject("source"); {
@@ -149,7 +143,7 @@ public class RestReindexActionTests extends ESTestCase {
                 body.endObject();
             }
             body.endObject();
-            request.withContent(body.bytes());
+            request.withContent(body.bytes(), body.contentType());
         }
         request.withParams(singletonMap("pipeline", "doesn't matter"));
         Exception e = expectThrows(IllegalArgumentException.class, () -> action.buildRequest(request.build()));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index afc08ed0587..21a0003fe5d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -22,6 +22,9 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.tasks.TaskId;
 
@@ -82,7 +85,7 @@ public class RethrottleTests extends ReindexTestCase {
         // Start a request that will never finish unless we rethrottle it
         request.setRequestsPerSecond(.000001f);  // Throttle "forever"
         request.source().setSize(1);             // Make sure we use multiple batches
-        ListenableActionFuture<? extends BulkIndexByScrollResponse> responseListener = request.execute();
+        ListenableActionFuture<? extends BulkByScrollResponse> responseListener = request.execute();
 
         TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, request.request().getSlices());
         TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId();
@@ -151,7 +154,7 @@ public class RethrottleTests extends ReindexTestCase {
         }
 
         // Now the response should come back quickly because we've rethrottled the request
-        BulkIndexByScrollResponse response = responseListener.get();
+        BulkByScrollResponse response = responseListener.get();
         assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.",
                 response.getBatches(), greaterThanOrEqualTo(request.request().getSlices()));
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 1dcf8811d23..e94dcfeb122 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -26,6 +26,10 @@ import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.BulkIndexByScrollResponseMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -143,7 +147,7 @@ public class RetryTests extends ESSingleNodeTestCase {
         request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
 
         logger.info("Starting request");
-        ListenableActionFuture<BulkIndexByScrollResponse> responseListener = request.execute();
+        ListenableActionFuture<BulkByScrollResponse> responseListener = request.execute();
 
         try {
             logger.info("Waiting for search rejections on the initial search");
@@ -170,13 +174,13 @@ public class RetryTests extends ESSingleNodeTestCase {
             scrollBlock.await();
 
             logger.info("Waiting for the request to finish");
-            BulkIndexByScrollResponse response = responseListener.get();
+            BulkByScrollResponse response = responseListener.get();
             assertThat(response, matcher);
             assertThat(response.getBulkRetries(), greaterThan(0L));
             assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
         } finally {
             // Fetch the response just in case we blew up half way through. This will make sure the failure is thrown up to the top level.
-            BulkIndexByScrollResponse response = responseListener.get();
+            BulkByScrollResponse response = responseListener.get();
             assertThat(response.getSearchFailures(), empty());
             assertThat(response.getBulkFailures(), empty());
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 7c4b4e3274f..abc0d6e1f35 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -31,7 +31,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
@@ -41,18 +40,10 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.stream.IntStream;
 
-import static java.lang.Math.abs;
-import static java.util.Collections.emptyList;
-import static java.util.Collections.singletonList;
-import static java.util.stream.Collectors.toList;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.hamcrest.Matchers.hasSize;
 
 /**
  * Round trip tests for all Streamable things declared in this plugin.
@@ -199,39 +190,6 @@ public class RoundTripTests extends ESTestCase {
         assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0d);
     }
 
-    public void testBulkByTaskStatus() throws IOException {
-        BulkByScrollTask.Status status = randomStatus();
-        BytesStreamOutput out = new BytesStreamOutput();
-        status.writeTo(out);
-        BulkByScrollTask.Status tripped = new BulkByScrollTask.Status(out.bytes().streamInput());
-        assertTaskStatusEquals(out.getVersion(), status, tripped);
-
-        // Also check round tripping pre-5.1 which is the first version to support parallelized scroll
-        out = new BytesStreamOutput();
-        out.setVersion(Version.V_5_0_0_rc1); // This can be V_5_0_0
-        status.writeTo(out);
-        StreamInput in = out.bytes().streamInput();
-        in.setVersion(Version.V_5_0_0_rc1);
-        tripped = new BulkByScrollTask.Status(in);
-        assertTaskStatusEquals(Version.V_5_0_0_rc1, status, tripped);
-    }
-
-    public void testReindexResponse() throws IOException {
-        BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
-                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
-        BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
-        roundTrip(response, tripped);
-        assertResponseEquals(response, tripped);
-    }
-
-    public void testBulkIndexByScrollResponse() throws IOException {
-        BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
-                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
-        BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
-        roundTrip(response, tripped);
-        assertResponseEquals(response, tripped);
-    }
-
     public void testRethrottleRequest() throws IOException {
         RethrottleRequest request = new RethrottleRequest();
         request.setRequestsPerSecond((float) randomDoubleBetween(0, Float.POSITIVE_INFINITY, false));
@@ -247,62 +205,6 @@ public class RoundTripTests extends ESTestCase {
         assertEquals(request.getTaskId(), tripped.getTaskId());
     }
 
-    private BulkByScrollTask.Status randomStatus() {
-        if (randomBoolean()) {
-            return randomWorkingStatus(null);
-        }
-        boolean canHaveNullStatues = randomBoolean();
-        List<BulkByScrollTask.StatusOrException> statuses = IntStream.range(0, between(0, 10))
-                .mapToObj(i -> {
-                    if (canHaveNullStatues && rarely()) {
-                        return null;
-                    }
-                    if (randomBoolean()) {
-                        return new BulkByScrollTask.StatusOrException(new ElasticsearchException(randomAsciiOfLength(5)));
-                    }
-                    return new BulkByScrollTask.StatusOrException(randomWorkingStatus(i));
-                })
-                .collect(toList());
-        return new BulkByScrollTask.Status(statuses, randomBoolean() ? "test" : null);
-    }
-
-    private BulkByScrollTask.Status randomWorkingStatus(Integer sliceId) {
-        // These all should be believably small because we sum them if we have multiple workers
-        int total = between(0, 10000000);
-        int updated = between(0, total);
-        int created = between(0, total - updated);
-        int deleted = between(0, total - updated - created);
-        int noops = total - updated - created - deleted;
-        int batches = between(0, 10000);
-        long versionConflicts = between(0, total);
-        long bulkRetries = between(0, 10000000);
-        long searchRetries = between(0, 100000);
-        return new BulkByScrollTask.Status(sliceId, total, updated, created, deleted, batches, versionConflicts, noops, bulkRetries,
-                searchRetries, parseTimeValue(randomPositiveTimeValue(), "test"), abs(random().nextFloat()),
-                randomBoolean() ? null : randomSimpleString(random()), parseTimeValue(randomPositiveTimeValue(), "test"));
-    }
-
-    private List<Failure> randomIndexingFailures() {
-        return usually() ? emptyList()
-                : singletonList(new Failure(randomSimpleString(random()), randomSimpleString(random()),
-                        randomSimpleString(random()), new IllegalArgumentException("test")));
-    }
-
-    private List<SearchFailure> randomSearchFailures() {
-        if (randomBoolean()) {
-            return emptyList();
-        }
-        String index = null;
-        Integer shardId = null;
-        String nodeId = null;
-        if (randomBoolean()) {
-            index = randomAsciiOfLength(5);
-            shardId = randomInt();
-            nodeId = usually() ? randomAsciiOfLength(5) : null;
-        }
-        return singletonList(new SearchFailure(new ElasticsearchException("foo"), index, shardId, nodeId));
-    }
-
     private void roundTrip(Streamable example, Streamable empty) throws IOException {
         roundTrip(Version.CURRENT, example, empty);
     }
@@ -324,64 +226,4 @@ public class RoundTripTests extends ESTestCase {
 
         return new Script(type, lang, idOrCode, params);
     }
-
-    private void assertResponseEquals(BulkIndexByScrollResponse expected, BulkIndexByScrollResponse actual) {
-        assertEquals(expected.getTook(), actual.getTook());
-        assertTaskStatusEquals(Version.CURRENT, expected.getStatus(), actual.getStatus());
-        assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
-        for (int i = 0; i < expected.getBulkFailures().size(); i++) {
-            Failure expectedFailure = expected.getBulkFailures().get(i);
-            Failure actualFailure = actual.getBulkFailures().get(i);
-            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
-            assertEquals(expectedFailure.getType(), actualFailure.getType());
-            assertEquals(expectedFailure.getId(), actualFailure.getId());
-            assertEquals(expectedFailure.getMessage(), actualFailure.getMessage());
-            assertEquals(expectedFailure.getStatus(), actualFailure.getStatus());
-        }
-        assertEquals(expected.getSearchFailures().size(), actual.getSearchFailures().size());
-        for (int i = 0; i < expected.getSearchFailures().size(); i++) {
-            SearchFailure expectedFailure = expected.getSearchFailures().get(i);
-            SearchFailure actualFailure = actual.getSearchFailures().get(i);
-            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
-            assertEquals(expectedFailure.getShardId(), actualFailure.getShardId());
-            assertEquals(expectedFailure.getNodeId(), actualFailure.getNodeId());
-            assertEquals(expectedFailure.getReason().getClass(), actualFailure.getReason().getClass());
-            assertEquals(expectedFailure.getReason().getMessage(), actualFailure.getReason().getMessage());
-        }
-
-    }
-
-    private void assertTaskStatusEquals(Version version, BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
-        assertEquals(expected.getTotal(), actual.getTotal());
-        assertEquals(expected.getUpdated(), actual.getUpdated());
-        assertEquals(expected.getCreated(), actual.getCreated());
-        assertEquals(expected.getDeleted(), actual.getDeleted());
-        assertEquals(expected.getBatches(), actual.getBatches());
-        assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
-        assertEquals(expected.getNoops(), actual.getNoops());
-        assertEquals(expected.getBulkRetries(), actual.getBulkRetries());
-        assertEquals(expected.getSearchRetries(), actual.getSearchRetries());
-        assertEquals(expected.getThrottled(), actual.getThrottled());
-        assertEquals(expected.getRequestsPerSecond(), actual.getRequestsPerSecond(), 0f);
-        assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
-        assertEquals(expected.getThrottledUntil(), actual.getThrottledUntil());
-        if (version.onOrAfter(Version.V_5_1_1_UNRELEASED)) {
-            assertThat(actual.getSliceStatuses(), hasSize(expected.getSliceStatuses().size()));
-            for (int i = 0; i < expected.getSliceStatuses().size(); i++) {
-                BulkByScrollTask.StatusOrException sliceStatus = expected.getSliceStatuses().get(i);
-                if (sliceStatus == null) {
-                    assertNull(actual.getSliceStatuses().get(i));
-                } else if (sliceStatus.getException() == null) {
-                    assertNull(actual.getSliceStatuses().get(i).getException());
-                    assertTaskStatusEquals(version, sliceStatus.getStatus(), actual.getSliceStatuses().get(i).getStatus());
-                } else {
-                    assertNull(actual.getSliceStatuses().get(i).getStatus());
-                    // Just check the message because we're not testing exception serialization in general here.
-                    assertEquals(sliceStatus.getException().getMessage(), actual.getSliceStatuses().get(i).getException().getMessage());
-                }
-            }
-        } else {
-            assertEquals(emptyList(), actual.getSliceStatuses());
-        }
-    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java
index 276bdb6ea4b..566358f4594 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java
@@ -23,6 +23,9 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollTask;
+import org.elasticsearch.action.bulk.byscroll.ParentBulkByScrollTask;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
@@ -114,7 +117,7 @@ public class TransportRethrottleActionTests extends ESTestCase {
         for (int i = 0; i < succeeded; i++) {
             BulkByScrollTask.Status status = believeableCompletedStatus(i);
             task.onSliceResponse(neverCalled(), i,
-                    new BulkIndexByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
+                    new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
             sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
         }
         List<TaskInfo> tasks = new ArrayList<>();
@@ -132,12 +135,12 @@ public class TransportRethrottleActionTests extends ESTestCase {
         List<BulkByScrollTask.StatusOrException> sliceStatuses = new ArrayList<>(slices);
         for (int i = 0; i < slices; i++) {
             @SuppressWarnings("unchecked")
-            ActionListener<BulkIndexByScrollResponse> listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class);
+            ActionListener<BulkByScrollResponse> listener = i < slices - 1 ? neverCalled() : mock(ActionListener.class);
             BulkByScrollTask.Status status = believeableCompletedStatus(i);
-            task.onSliceResponse(listener, i, new BulkIndexByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
+            task.onSliceResponse(listener, i, new BulkByScrollResponse(timeValueMillis(10), status, emptyList(), emptyList(), false));
             if (i == slices - 1) {
                 // The whole thing succeeded so we should have got the success
-                captureResponse(BulkIndexByScrollResponse.class, listener).getStatus();
+                captureResponse(BulkByScrollResponse.class, listener).getStatus();
             }
             sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
         }
@@ -182,7 +185,7 @@ public class TransportRethrottleActionTests extends ESTestCase {
         return new BulkByScrollTask.Status(sliceId, 10, 10, 0, 0, 0, 0, 0, 0, 0, timeValueMillis(0), 0, null, timeValueMillis(0));
     }
 
-    static <T> ActionListener<T> neverCalled() {
+    private <T> ActionListener<T> neverCalled() {
         return new ActionListener<T>() {
             @Override
             public void onResponse(T response) {
@@ -196,7 +199,7 @@ public class TransportRethrottleActionTests extends ESTestCase {
         };
     }
 
-    static <T> T captureResponse(Class<T> responseClass, ActionListener<T> listener) {
+    private <T> T captureResponse(Class<T> responseClass, ActionListener<T> listener) {
         ArgumentCaptor<Exception> failure = ArgumentCaptor.forClass(Exception.class);
         // Rethrow any failures just so we get a nice exception if there were any. We don't expect any though.
         verify(listener, atMost(1)).onFailure(failure.capture());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
index 6ebb0749792..c2b1dc50871 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
@@ -19,24 +19,41 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollAction;
+import org.elasticsearch.action.bulk.byscroll.AbstractAsyncBulkByScrollActionMetadataTestCase;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Hit;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 
 public class UpdateByQueryMetadataTests
-        extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+        extends AbstractAsyncBulkByScrollActionMetadataTestCase<UpdateByQueryRequest, BulkByScrollResponse> {
     public void testRoutingIsCopied() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
+        action().copyMetadata(AbstractAsyncBulkByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
     @Override
-    protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action() {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
+    protected TestAction action() {
+        return new TestAction();
     }
 
     @Override
     protected UpdateByQueryRequest request() {
         return new UpdateByQueryRequest(new SearchRequest());
     }
+
+    private class TestAction extends TransportUpdateByQueryAction.AsyncIndexBySearchAction {
+        TestAction() {
+            super(UpdateByQueryMetadataTests.this.task, UpdateByQueryMetadataTests.this.logger, null,
+                    UpdateByQueryMetadataTests.this.threadPool, request(), null, null, listener());
+        }
+
+        @Override
+        public AbstractAsyncBulkByScrollAction.RequestWrapper<?> copyMetadata(AbstractAsyncBulkByScrollAction.RequestWrapper<?> request,
+                Hit doc) {
+            return super.copyMetadata(request, doc);
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
index 2d881aff91a..9ecb0089187 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestTestCase;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.script.Script;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index 6bbcbd6e643..cb027d156e6 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
@@ -38,7 +39,7 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
     private static final int MAX_MUTATIONS = 50;
-    private static final int MAX_ATTEMPTS = 10;
+    private static final int MAX_ATTEMPTS = 50;
 
     public void testUpdateWhileReindexing() throws Exception {
         AtomicReference<String> value = new AtomicReference<>(randomSimpleString(random()));
@@ -49,7 +50,7 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
         Thread updater = new Thread(() -> {
             while (keepUpdating.get()) {
                 try {
-                    BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
+                    BulkByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
                     assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L)))
                             .versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
                 } catch (Exception e) {
@@ -82,9 +83,8 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
                             throw new RuntimeException(
                                     "Failed to index after [" + MAX_ATTEMPTS + "] attempts. Too many version conflicts!");
                         }
-                        logger.info(
-                                "Caught expected version conflict trying to perform mutation number {} with version {}. Retrying.",
-                                i, get.getVersion());
+                        logger.info("Caught expected version conflict trying to perform mutation number [{}] with version [{}] "
+                                + "on attempt [{}]. Retrying.", i, get.getVersion(), attempts);
                         get = client().prepareGet("test", "test", "test").get();
                     }
                 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
index 5ff54e4e06d..394669f00ec 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.script.ScriptService;
 
@@ -28,7 +29,7 @@ import java.util.Map;
 import static org.hamcrest.Matchers.containsString;
 
 public class UpdateByQueryWithScriptTests
-        extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+        extends AbstractAsyncBulkByScrollActionScriptTestCase<UpdateByQueryRequest, BulkByScrollResponse> {
 
     public void testModifyingCtxNotAllowed() {
         /*
@@ -53,8 +54,8 @@ public class UpdateByQueryWithScriptTests
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> action(ScriptService scriptService, UpdateByQueryRequest request) {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(),
-                scriptService, null);
+    protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptService scriptService, UpdateByQueryRequest request) {
+        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, scriptService, null,
+                listener());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
index dec5263352a..f9ab72506a8 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
@@ -152,20 +152,29 @@ public class RemoteRequestBuildersTests extends ESTestCase {
         assertThat(params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
         assertThat(params, hasEntry("size", Integer.toString(size)));
         assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null)));
-        assertThat(params, hasEntry("_source", "true"));
     }
 
     public void testInitialSearchEntity() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.source(new SearchSourceBuilder());
         String query = "{\"match_all\":{}}";
-        HttpEntity entity = initialSearchEntity(new BytesArray(query));
+        HttpEntity entity = initialSearchEntity(searchRequest, new BytesArray(query));
         assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
-        assertEquals("{\"query\":" + query + "}",
+        assertEquals("{\"query\":" + query + ",\"_source\":true}",
+                Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+
+        // Source filtering is included if set up
+        searchRequest.source().fetchSource(new String[] {"in1", "in2"}, new String[] {"out"});
+        entity = initialSearchEntity(searchRequest, new BytesArray(query));
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        assertEquals("{\"query\":" + query + ",\"_source\":{\"includes\":[\"in1\",\"in2\"],\"excludes\":[\"out\"]}}",
                 Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
 
         // Invalid XContent fails
-        RuntimeException e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{}, \"trailing\": {}")));
+        RuntimeException e = expectThrows(RuntimeException.class,
+                () -> initialSearchEntity(searchRequest, new BytesArray("{}, \"trailing\": {}")));
         assertThat(e.getCause().getMessage(), containsString("Unexpected character (',' (code 44))"));
-        e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{")));
+        e = expectThrows(RuntimeException.class, () -> initialSearchEntity(searchRequest, new BytesArray("{")));
         assertThat(e.getCause().getMessage(), containsString("Unexpected end-of-input"));
     }
 
@@ -177,7 +186,8 @@ public class RemoteRequestBuildersTests extends ESTestCase {
     public void testScrollEntity() throws IOException {
         String scroll = randomAsciiOfLength(30);
         HttpEntity entity = scrollEntity(scroll);
-        assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue());
-        assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        assertThat(Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)),
+            containsString("\"" + scroll + "\""));
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
index f898483a124..3d4a0aa15d7 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
@@ -26,6 +26,7 @@ import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.StatusLine;
+import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.concurrent.FutureCallback;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.InputStreamEntity;
@@ -39,6 +40,7 @@ import org.apache.http.nio.protocol.HttpAsyncResponseConsumer;
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.byscroll.ScrollableHitSource.Response;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.HeapBufferedAsyncResponseConsumer;
 import org.elasticsearch.client.RestClient;
@@ -50,7 +52,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESTestCase;
@@ -73,6 +74,7 @@ import java.util.function.Consumer;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
@@ -120,37 +122,37 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
 
     public void testLookupRemoteVersion() throws Exception {
         AtomicBoolean called = new AtomicBoolean();
-        sourceWithMockedRemoteCall(false, "main/0_20_5.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_20_5.json").lookupRemoteVersion(v -> {
             assertEquals(Version.fromString("0.20.5"), v);
             called.set(true);
         });
         assertTrue(called.get());
         called.set(false);
-        sourceWithMockedRemoteCall(false, "main/0_90_13.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/0_90_13.json").lookupRemoteVersion(v -> {
             assertEquals(Version.fromString("0.90.13"), v);
             called.set(true);
         });
         assertTrue(called.get());
         called.set(false);
-        sourceWithMockedRemoteCall(false, "main/1_7_5.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/1_7_5.json").lookupRemoteVersion(v -> {
             assertEquals(Version.fromString("1.7.5"), v);
             called.set(true);
         });
         assertTrue(called.get());
         called.set(false);
-        sourceWithMockedRemoteCall(false, "main/2_3_3.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/2_3_3.json").lookupRemoteVersion(v -> {
             assertEquals(Version.V_2_3_3, v);
             called.set(true);
         });
         assertTrue(called.get());
         called.set(false);
-        sourceWithMockedRemoteCall(false, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> {
             assertEquals(Version.V_5_0_0_alpha3, v);
             called.set(true);
         });
         assertTrue(called.get());
         called.set(false);
-        sourceWithMockedRemoteCall(false, "main/with_unknown_fields.json").lookupRemoteVersion(v -> {
+        sourceWithMockedRemoteCall(false, ContentType.APPLICATION_JSON, "main/with_unknown_fields.json").lookupRemoteVersion(v -> {
             assertEquals(Version.V_5_0_0_alpha3, v);
             called.set(true);
         });
@@ -223,7 +225,6 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
         assertTrue(called.get());
     }
 
-
     /**
      * Versions of Elasticsearch before 2.1.0 don't support sort:_doc and instead need to use search_type=scan. Scan doesn't return
      * documents the first iteration but reindex doesn't like that. So we jump start strait to the next iteration.
@@ -430,11 +431,11 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
         ContentTooLongException tooLong = new ContentTooLongException("too long!");
         CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).then(new Answer<Future<HttpResponse>>() {
+                any(HttpClientContext.class), any(FutureCallback.class))).then(new Answer<Future<HttpResponse>>() {
             @Override
             public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable {
                 HeapBufferedAsyncResponseConsumer consumer = (HeapBufferedAsyncResponseConsumer) invocationOnMock.getArguments()[1];
-                FutureCallback callback = (FutureCallback) invocationOnMock.getArguments()[2];
+                FutureCallback callback = (FutureCallback) invocationOnMock.getArguments()[3];
                 assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(), consumer.getBufferLimit());
                 callback.failed(tooLong);
                 return null;
@@ -457,8 +458,25 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
         assertFalse(called.get());
     }
 
+    public void testNoContentTypeIsError() throws Exception {
+        Exception e = expectThrows(RuntimeException.class, () ->
+                sourceWithMockedRemoteCall(false, null, "main/0_20_5.json").lookupRemoteVersion(null));
+        assertThat(e.getCause().getCause().getMessage(), containsString("Response didn't include Content-Type: body={"));
+    }
+
+    public void testInvalidJsonThinksRemoveIsNotES() throws IOException {
+        Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("some_text.txt").doStart(null));
+        assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", e.getCause().getCause().getMessage());
+    }
+
+    public void testUnexpectedJsonThinksRemoveIsNotES() throws IOException {
+        // Use the response from a main action instead of a proper start response to generate a parse error
+        Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("main/2_3_3.json").doStart(null));
+        assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", e.getCause().getCause().getMessage());
+    }
+
     private RemoteScrollableHitSource sourceWithMockedRemoteCall(String... paths) throws Exception {
-        return sourceWithMockedRemoteCall(true, paths);
+        return sourceWithMockedRemoteCall(true, ContentType.APPLICATION_JSON, paths);
     }
 
     /**
@@ -466,7 +484,8 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
      * synchronously rather than asynchronously.
      */
     @SuppressWarnings("unchecked")
-    private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, String... paths) throws Exception {
+    private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, ContentType contentType, String... paths)
+            throws Exception {
         URL[] resources = new URL[paths.length];
         for (int i = 0; i < paths.length; i++) {
             resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", ""));
@@ -477,7 +496,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
 
         CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class);
         when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class),
-                any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
+                any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() {
 
             int responseCount = 0;
 
@@ -486,7 +505,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
                 // Throw away the current thread context to simulate running async httpclient's thread pool
                 threadPool.getThreadContext().stashContext();
                 HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0];
-                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[2];
+                FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3];
                 HttpEntityEnclosingRequest request = (HttpEntityEnclosingRequest)requestProducer.generateRequest();
                 URL resource = resources[responseCount];
                 String path = paths[responseCount++];
@@ -503,8 +522,7 @@ public class RemoteScrollableHitSourceTests extends ESTestCase {
                 } else {
                     StatusLine statusLine = new BasicStatusLine(protocolVersion, 200, "");
                     HttpResponse httpResponse = new BasicHttpResponse(statusLine);
-                    httpResponse.setEntity(new InputStreamEntity(resource.openStream(),
-                            randomBoolean() ? ContentType.APPLICATION_JSON : null));
+                    httpResponse.setEntity(new InputStreamEntity(resource.openStream(), contentType));
                     futureCallback.completed(httpResponse);
                 }
                 return null;
diff --git a/modules/reindex/src/test/resources/responses/some_text.txt b/modules/reindex/src/test/resources/responses/some_text.txt
new file mode 100644
index 00000000000..39fe6c971a8
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/some_text.txt
@@ -0,0 +1 @@
+I'm just text!
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index a567ca67bfa..3557cf9bad7 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -26,6 +26,13 @@
   - is_false: task
   - is_false: deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for updated":
   - do:
@@ -60,6 +67,13 @@
   - is_false: task
   - is_false: deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "wait_for_completion=false":
   - do:
@@ -110,6 +124,13 @@
   - is_false: response.task
   - is_false: response.deleted
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for version conflict":
   - do:
@@ -151,6 +172,13 @@
   - match: {failures.0.cause.index:  dest}
   - gte: { took: 0 }
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Response format for version conflict with conflicts=proceed":
   - do:
@@ -185,210 +213,12 @@
   - match: {throttled_millis: 0}
   - gte: { took: 0 }
 
----
-"Simplest example in docs":
+  # Make sure reindex closed all the scroll contexts
   - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
-
----
-"Limit by type example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:   twitter
-        type:    junk
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-            type:  tweet
-          dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
-
----
-"Limit by query example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      2
-        body:    { "user": "junk" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-            query:
-              match:
-                user: kimchy
-          dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-  - match: { hits.total: 1 }
-
----
-"Override type example in docs":
-  - do:
-      index:
-        index:   twitter
-        type:    tweet
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      index:
-        index:   twitter
-        type:    junk
-        id:      1
-        body:    { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: twitter
-            type:  tweet
-          dest:
-            index: new_twitter
-            type:  chirp
-
-  - do:
-      search:
-        index: new_twitter
-        type:  chirp
-  - match: { hits.total: 1 }
-
----
-"Multi index, multi type example from docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  blog
-        type:   post
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          source:
-            index: [twitter, blog]
-            type:  [tweet, post]
-          dest:
-            index: all_together
-
-  - do:
-      search:
-        index: all_together
-        type: tweet
-        body:
-          query:
-            match:
-              user: kimchy
-  - match: { hits.total: 1 }
-
-  - do:
-      search:
-        index: all_together
-        type: post
-        body:
-          query:
-            match:
-              user: kimchy
-  - match: { hits.total: 1 }
-
----
-"Limit by size example from docs":
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     1
-        body:   { "user": "kimchy" }
-  - do:
-      index:
-        index:  twitter
-        type:   tweet
-        id:     2
-        body:   { "user": "kimchy" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        refresh: true
-        body:
-          size: 1
-          source:
-            index: twitter
-          dest:
-            index: new_twitter
-
-  - do:
-      search:
-        index: new_twitter
-        type: tweet
-  - match: { hits.total: 1 }
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
 
 ---
 "Source document without any fields works":
@@ -416,3 +246,56 @@
         type:    foo
         id:      1
   - match: { _source: {} }
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
+---
+"Reindex with source filtering":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test", "filtered": "removed" }
+        refresh: true
+
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: source
+            _source:
+              excludes:
+                - filtered
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      get:
+        index: dest
+        type: foo
+        id: 1
+  - match: { _source.text: "test" }
+  - is_false: _source.filtered
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index 50f12192960..c9f441c9cd3 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -295,3 +295,21 @@
             index: test
           dest:
             index: dest
+
+---
+"_source:false is rejected":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /_source:false is not supported in this context/
+      reindex:
+        body:
+          source:
+            index: source
+            _source: false
+          dest:
+            index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml
new file mode 100644
index 00000000000..874174fda4c
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yaml
@@ -0,0 +1,72 @@
+teardown:
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: null
+
+---
+"auto_create_index is false":
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: false
+  - do:
+      catch: /no such index and \[action.auto_create_index\] is \[false\]/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"auto_create_index doesn't include new index":
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: test
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /no such index and \[action.auto_create_index\] \(\[test\]\) doesn't match/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"auto_create_index forbids new index":
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            action.auto_create_index: test,-dest
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /no such index and \[action.auto_create_index\] contains \[-dest\] which forbids automatic creation of the index/
+      reindex:
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
index 86b4bcc1132..576447b4e54 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
@@ -46,6 +46,13 @@
               text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with query":
   - do:
@@ -95,6 +102,13 @@
             match_all: {}
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with routing":
   - do:
@@ -137,6 +151,13 @@
               text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with parent/child":
   - do:
@@ -206,6 +227,13 @@
                   text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with timeouts":
   # Validates that you can configure the socket_timeout and connect_timeout,
@@ -258,6 +286,76 @@
               text: test
   - match: {hits.total: 1}
 
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
+---
+"Reindex from remote with size":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+        refresh: true
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          size: 1
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            match:
+              text: test
+  - match: {hits.total: 1}
+
+  # Make sure reindex closed all the scroll contexts
+  - do:
+      indices.stats:
+        index: source
+        metric: search
+  - match: {indices.source.total.search.open_contexts: 0}
+
 ---
 "Reindex from remote with broken query":
   - do:
@@ -301,7 +399,8 @@
         refresh: true
 
   - do:
-      catch: /Connection refused/
+      # sometimes IIS is listening on port 0. In that case we fail in other ways and this test isn't useful.
+      catch: /connect_exception|IIS Windows Server/
       reindex:
         body:
           source:
@@ -310,3 +409,53 @@
             index: source
           dest:
             index: dest
+
+---
+"Reindex from remote with source filtering":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test", "filtered": "removed" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+            _source:
+              excludes:
+                - filtered
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      get:
+        index: dest
+        type: foo
+        id: 1
+  - match: { _source.text: "test" }
+  - is_false: _source.filtered
diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle
new file mode 100644
index 00000000000..dc769e3c1f3
--- /dev/null
+++ b/modules/repository-url/build.gradle
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+    description 'Module for URL repository'
+    classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
+}
+
+integTest {
+    cluster {
+        setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
index ede57d461a1..bc7381f91c6 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
@@ -29,6 +29,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.NoSuchFileException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
 /**
@@ -102,7 +105,7 @@ public class URLBlobContainer extends AbstractBlobContainer {
     @Override
     public InputStream readBlob(String name) throws IOException {
         try {
-            return new BufferedInputStream(new URL(path, name).openStream(), blobStore.bufferSizeInBytes());
+            return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes());
         } catch (FileNotFoundException fnfe) {
             throw new NoSuchFileException("[" + name + "] blob not found");
         }
@@ -113,4 +116,12 @@ public class URLBlobContainer extends AbstractBlobContainer {
         throw new UnsupportedOperationException("URL repository doesn't support this operation");
     }
 
+    private static InputStream getInputStream(URL url) throws IOException {
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<InputStream>) url::openStream);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
rename to modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
index 2386f79c076..158ecff9b2b 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java
@@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
     public URLBlobStore(Settings settings, URL path) {
         super(settings);
         this.path = path;
-        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
+        this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size",
+            new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
     }
 
     /**
diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java
new file mode 100644
index 00000000000..a28413d213a
--- /dev/null
+++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.repository.url;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.repositories.url.URLRepository;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin {
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+            URLRepository.ALLOWED_URLS_SETTING,
+            URLRepository.REPOSITORIES_URL_SETTING,
+            URLRepository.SUPPORTED_PROTOCOLS_SETTING
+        );
+    }
+
+    @Override
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+        return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
rename to modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
index 5ca335c5953..c1128fd683a 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.repositories.uri;
+package org.elasticsearch.repositories.url;
 
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -26,6 +26,7 @@ import org.elasticsearch.common.blobstore.url.URLBlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.URIPattern;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -77,8 +78,9 @@ public class URLRepository extends BlobStoreRepository {
     /**
      * Constructs a read-only URL-based repository
      */
-    public URLRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public URLRepository(RepositoryMetaData metadata, Environment environment,
+                         NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
 
         if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
             throw new RepositoryException(metadata.name(), "missing url");
@@ -125,8 +127,12 @@ public class URLRepository extends BlobStoreRepository {
                 // We didn't match white list - try to resolve against path.repo
                 URL normalizedUrl = environment.resolveRepoURL(url);
                 if (normalizedUrl == null) {
-                    logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
-                    throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey());
+                    String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " +
+                        "path.repo setting or by {} setting: [{}] ";
+                    logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
+                    String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or "
+                        + ALLOWED_URLS_SETTING.getKey();
+                    throw new RepositoryException(getMetadata().name(), exceptionMessage);
                 }
                 return normalizedUrl;
             }
diff --git a/modules/repository-url/src/main/plugin-metadata/plugin-security.policy b/modules/repository-url/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 00000000000..b878c485755
--- /dev/null
+++ b/modules/repository-url/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  permission java.net.SocketPermission "*", "connect";
+};
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java
new file mode 100644
index 00000000000..4df83c58bd3
--- /dev/null
+++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.url;
+
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.NoSuchFileException;
+
+@SuppressForbidden(reason = "use http server")
+public class URLBlobStoreTests extends ESTestCase {
+
+    private static HttpServer httpServer;
+    private static String blobName;
+    private static byte[] message = new byte[512];
+    private URLBlobStore urlBlobStore;
+
+    @BeforeClass
+    public static void startHttp() throws Exception {
+        for (int i = 0; i < message.length; ++i) {
+            message[i] = randomByte();
+        }
+        blobName = randomAsciiOfLength(8);
+
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0);
+
+        httpServer.createContext("/indices/" + blobName, (s) -> {
+            s.sendResponseHeaders(200, message.length);
+            OutputStream responseBody = s.getResponseBody();
+            responseBody.write(message);
+            responseBody.close();
+        });
+
+        httpServer.start();
+    }
+
+    @AfterClass
+    public static void stopHttp() throws IOException {
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
+    @Before
+    public void storeSetup() throws MalformedURLException {
+        Settings settings = Settings.EMPTY;
+        String spec = "http://localhost:6001/";
+        urlBlobStore = new URLBlobStore(settings, new URL(spec));
+    }
+
+    public void testURLBlobStoreCanReadBlob() throws IOException {
+        BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
+        try (InputStream stream = container.readBlob(blobName)) {
+            byte[] bytes = new byte[message.length];
+            int read = stream.read(bytes);
+            assertEquals(message.length, read);
+            assertArrayEquals(message, bytes);
+        }
+    }
+
+    public void testNoBlobFound() throws IOException {
+        BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices"));
+        String incorrectBlobName = "incorrect_" + blobName;
+        try (InputStream ignored = container.readBlob(incorrectBlobName)) {
+            fail("Should have thrown NoSuchFileException exception");
+            ignored.read();
+        } catch (NoSuchFileException e) {
+            assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage());
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java
similarity index 54%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
rename to modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java
index 3d9593ce290..66fba622e60 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java
+++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java
@@ -16,19 +16,25 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
 
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
+package org.elasticsearch.repositories.url;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
 import java.io.IOException;
 
-/**
- * Parser for is_false assert sections
- */
-public class IsFalseParser implements ClientYamlTestFragmentParser<IsFalseAssertion> {
+public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
-    @Override
-    public IsFalseAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        return new IsFalseAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField());
+    public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException {
+        return ESClientYamlSuiteTestCase.createParameters();
     }
 }
+
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java
new file mode 100644
index 00000000000..ea274eeae60
--- /dev/null
+++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.url;
+
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Collections;
+
+public class URLRepositoryTests extends ESTestCase {
+
+    public void testWhiteListingRepoURL() throws IOException {
+        String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoPath)
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+    }
+
+    public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
+        String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        try {
+            new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+            fail("RepositoryException should have been thrown.");
+        } catch (RepositoryException e) {
+            String msg = "[url] file url [" + repoPath
+                + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls";
+            assertEquals(msg, e.getMessage());
+        }
+    }
+
+    public void testMustBeSupportedProtocol() throws IOException {
+        Path directory = createTempDir();
+        String repoPath = directory.resolve("repository").toUri().toURL().toString();
+        Settings baseSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(Environment.PATH_REPO_SETTING.getKey(), directory.toString())
+            .put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
+            .put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
+            .build();
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
+        try {
+            new URLRepository(repositoryMetaData, new Environment(baseSettings), new NamedXContentRegistry(Collections.emptyList()));
+            fail("RepositoryException should have been thrown.");
+        } catch (RepositoryException e) {
+            assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
+        }
+    }
+
+}
diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java
new file mode 100644
index 00000000000..f7fb7af055c
--- /dev/null
+++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.url;
+
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.plugin.repository.url.URLRepositoryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.SnapshotState;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class URLSnapshotRestoreTests extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singletonList(URLRepositoryPlugin.class);
+    }
+
+    public void testUrlRepository() throws Exception {
+        Client client = client();
+
+        logger.info("-->  creating repository");
+        Path repositoryLocation = randomRepoPath();
+        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+            .setType(FsRepository.TYPE).setSettings(Settings.builder()
+                .put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
+                .put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
+                .put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+
+        createIndex("test-idx");
+        ensureGreen();
+
+        logger.info("--> indexing some data");
+        for (int i = 0; i < 100; i++) {
+            index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
+
+        logger.info("--> snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client
+            .admin()
+            .cluster()
+            .prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        int actualTotalShards = createSnapshotResponse.getSnapshotInfo().totalShards();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(actualTotalShards));
+
+        SnapshotState state = client
+            .admin()
+            .cluster()
+            .prepareGetSnapshots("test-repo")
+            .setSnapshots("test-snap")
+            .get()
+            .getSnapshots()
+            .get(0)
+            .state();
+        assertThat(state, equalTo(SnapshotState.SUCCESS));
+
+        logger.info("--> delete index");
+        cluster().wipeIndices("test-idx");
+
+        logger.info("--> create read-only URL repository");
+        assertAcked(client.admin().cluster().preparePutRepository("url-repo")
+            .setType(URLRepository.TYPE).setSettings(Settings.builder()
+                .put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL())
+                .put("list_directories", randomBoolean())));
+        logger.info("--> restore index after deletion");
+        RestoreSnapshotResponse restoreSnapshotResponse = client
+            .admin()
+            .cluster()
+            .prepareRestoreSnapshot("url-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .setIndices("test-idx")
+            .execute()
+            .actionGet();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
+
+        logger.info("--> list available shapshots");
+        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
+        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
+
+        logger.info("--> delete snapshot");
+        DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
+        assertAcked(deleteSnapshotResponse);
+
+        logger.info("--> list available shapshot again, no snapshots should be returned");
+        getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
+        assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
+    }
+}
diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml
new file mode 100644
index 00000000000..75e78732998
--- /dev/null
+++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yaml
@@ -0,0 +1,32 @@
+# Integration tests for URL Repository component
+#
+"URL Repository plugin loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.modules.0.name: repository-url }
+
+---
+setup:
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo1
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test1"
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo2
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test2"
diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml
new file mode 100644
index 00000000000..39cfeee192c
--- /dev/null
+++ b/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yaml
@@ -0,0 +1,16 @@
+"Repository can be registered":
+
+  - do:
+      snapshot.create_repository:
+        repository: test_repo1
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test1"
+  - is_true: acknowledged
+
+  - do:
+      snapshot.get_repository:
+        repository: test_repo1
+
+  - is_true : test_repo1
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
index 472dce3c4eb..3dd233edf6a 100644
--- a/modules/transport-netty4/build.gradle
+++ b/modules/transport-netty4/build.gradle
@@ -33,13 +33,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.6.Final"
-    compile "io.netty:netty-codec:4.1.6.Final"
-    compile "io.netty:netty-codec-http:4.1.6.Final"
-    compile "io.netty:netty-common:4.1.6.Final"
-    compile "io.netty:netty-handler:4.1.6.Final"
-    compile "io.netty:netty-resolver:4.1.6.Final"
-    compile "io.netty:netty-transport:4.1.6.Final"
+    compile "io.netty:netty-buffer:4.1.7.Final"
+    compile "io.netty:netty-codec:4.1.7.Final"
+    compile "io.netty:netty-codec-http:4.1.7.Final"
+    compile "io.netty:netty-common:4.1.7.Final"
+    compile "io.netty:netty-handler:4.1.7.Final"
+    compile "io.netty:netty-resolver:4.1.7.Final"
+    compile "io.netty:netty-transport:4.1.7.Final"
 }
 
 thirdPartyAudit.excludes = [
@@ -131,9 +131,6 @@ thirdPartyAudit.excludes = [
         'io.netty.util.internal.PlatformDependent0',
         'io.netty.util.internal.PlatformDependent0$2',
         'io.netty.util.internal.PlatformDependent0$3',
-        'io.netty.util.internal.UnsafeAtomicIntegerFieldUpdater',
-        'io.netty.util.internal.UnsafeAtomicLongFieldUpdater',
-        'io.netty.util.internal.UnsafeAtomicReferenceFieldUpdater',
         'io.netty.util.internal.chmv8.ConcurrentHashMapV8',
         'io.netty.util.internal.chmv8.ConcurrentHashMapV8$1',
         'io.netty.util.internal.chmv8.ConcurrentHashMapV8$TreeBin',
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.6.Final.jar.sha1
deleted file mode 100644
index a1b16cc76e9..00000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dc1a304ee7c448bfa241f60c41948db6f12b653b
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..c3b37d04751
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+65e89b4d0737cf1ed3eac2916ebf365f5312294c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.6.Final.jar.sha1
deleted file mode 100644
index 45cf352fc1b..00000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-086ce3c559af6759a0b991b9ad2faec20981e18e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..acfce49a8cc
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+b3ecd69e7ab0eb2054b26ad3efb13c34204ffe19
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.6.Final.jar.sha1
deleted file mode 100644
index 31ba47f6a30..00000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ee6c65cb3f39f212241e9131dc05a837e776fd70
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..1f17e2a90c2
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+9f957998c651e7b73d6dc878f704d81b4c085387
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.6.Final.jar.sha1
deleted file mode 100644
index 1e2bbb9656e..00000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-654e4d8e87bc442be5c1d82d0369a3885f7168ea
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..f9a3b210333
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+06766071efa0426fb33101d1758a8d0e09d16ac7
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.6.Final.jar.sha1
deleted file mode 100644
index 9cfb2d9d0d0..00000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a24e92d67e467a3b97ff63642ab56637990685fd
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..f8968ddda7e
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+6f7dfa01c1af4280c154213b6d548405df352df9
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.6.Final.jar.sha1
deleted file mode 100644
index 67c8474f268..00000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a1c97445b6aa544cd029faa72abfb3f85e9957e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..79a0b112d6b
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+077734f94029b47887f6f9392d1fc62729e155aa
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.6.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.6.Final.jar.sha1
deleted file mode 100644
index 7107502b453..00000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.6.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4cba91ad2711c13a063d480f01edf70aa3f9387d
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.7.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.7.Final.jar.sha1
new file mode 100644
index 00000000000..54b3501de87
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.7.Final.jar.sha1
@@ -0,0 +1 @@
+469e86d4dda1dca8b88d2b1faa8e0f078243ba12
\ No newline at end of file
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
index 2e511d15622..f3099db08e9 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequest.java
@@ -19,20 +19,25 @@
 
 package org.elasticsearch.http.netty4;
 
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.HttpRequest;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.transport.netty4.Netty4Utils;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestUtils;
-
 import io.netty.channel.Channel;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpMethod;
 
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.transport.netty4.Netty4Utils;
+
 import java.net.SocketAddress;
-import java.util.HashMap;
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 public class Netty4HttpRequest extends RestRequest {
 
@@ -40,8 +45,8 @@ public class Netty4HttpRequest extends RestRequest {
     private final Channel channel;
     private final BytesReference content;
 
-    Netty4HttpRequest(FullHttpRequest request, Channel channel) {
-        super(request.uri());
+    Netty4HttpRequest(NamedXContentRegistry xContentRegistry, FullHttpRequest request, Channel channel) {
+        super(xContentRegistry, request.uri(), new HttpHeadersMap(request.headers()));
         this.request = request;
         this.channel = channel;
         if (request.content().isReadable()) {
@@ -122,14 +127,81 @@ public class Netty4HttpRequest extends RestRequest {
         return channel;
     }
 
-    @Override
-    public String header(String name) {
-        return request.headers().get(name);
-    }
+    /**
+     * A wrapper of {@link HttpHeaders} that implements a map to prevent copying unnecessarily. This class does not support modifications
+     * and due to the underlying implementation, it performs case insensitive lookups of key to values.
+     *
+     * It is important to note that this implementation does have some downsides in that each invocation of the
+     * {@link #values()} and {@link #entrySet()} methods will perform a copy of the values in the HttpHeaders rather than returning a
+     * view of the underlying values.
+     */
+    private static class HttpHeadersMap implements Map<String, List<String>> {
 
-    @Override
-    public Iterable<Map.Entry<String, String>> headers() {
-        return request.headers().entries();
-    }
+        private final HttpHeaders httpHeaders;
 
+        private HttpHeadersMap(HttpHeaders httpHeaders) {
+            this.httpHeaders = httpHeaders;
+        }
+
+        @Override
+        public int size() {
+            return httpHeaders.size();
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return httpHeaders.isEmpty();
+        }
+
+        @Override
+        public boolean containsKey(Object key) {
+            return key instanceof String && httpHeaders.contains((String) key);
+        }
+
+        @Override
+        public boolean containsValue(Object value) {
+            return value instanceof List && httpHeaders.names().stream().map(httpHeaders::getAll).anyMatch(value::equals);
+        }
+
+        @Override
+        public List<String> get(Object key) {
+            return key instanceof String ? httpHeaders.getAll((String) key) : null;
+        }
+
+        @Override
+        public List<String> put(String key, List<String> value) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public List<String> remove(Object key) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void putAll(Map<? extends String, ? extends List<String>> m) {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public void clear() {
+            throw new UnsupportedOperationException("modifications are not supported");
+        }
+
+        @Override
+        public Set<String> keySet() {
+            return httpHeaders.names();
+        }
+
+        @Override
+        public Collection<List<String>> values() {
+            return httpHeaders.names().stream().map(k -> Collections.unmodifiableList(httpHeaders.getAll(k))).collect(Collectors.toList());
+        }
+
+        @Override
+        public Set<Entry<String, List<String>>> entrySet() {
+            return httpHeaders.names().stream().map(k -> new AbstractMap.SimpleImmutableEntry<>(k, httpHeaders.getAll(k)))
+                    .collect(Collectors.toSet());
+        }
+    }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
index 6e12970c6df..a83c62e5221 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpRequestHandler.java
@@ -25,6 +25,7 @@ import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
+
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.netty4.pipelining.HttpPipelinedRequest;
 import org.elasticsearch.transport.netty4.Netty4Utils;
@@ -65,7 +66,7 @@ class Netty4HttpRequestHandler extends SimpleChannelInboundHandler<Object> {
                         request.headers(),
                         request.trailingHeaders());
 
-        final Netty4HttpRequest httpRequest = new Netty4HttpRequest(copy, ctx.channel());
+        final Netty4HttpRequest httpRequest = new Netty4HttpRequest(serverTransport.xContentRegistry, copy, ctx.channel());
         serverTransport.dispatchRequest(
             httpRequest,
             new Netty4HttpChannel(serverTransport, httpRequest, pipelinedRequest, detailedErrorsEnabled, threadContext));
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
index 90ca1b9a6a5..b9a81010a72 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java
@@ -32,9 +32,6 @@ import io.netty.channel.ChannelOption;
 import io.netty.channel.FixedRecvByteBufAllocator;
 import io.netty.channel.RecvByteBufAllocator;
 import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.oio.OioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.channel.socket.oio.OioServerSocketChannel;
 import io.netty.handler.codec.ByteToMessageDecoder;
 import io.netty.handler.codec.http.HttpContentCompressor;
 import io.netty.handler.codec.http.HttpContentDecompressor;
@@ -61,9 +58,10 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.http.BindHttpException;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.http.HttpServerAdapter;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.http.netty4.cors.Netty4CorsConfig;
@@ -78,6 +76,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BindTransportException;
 import org.elasticsearch.transport.netty4.Netty4OpenChannelsHandler;
 import org.elasticsearch.transport.netty4.Netty4Utils;
+import org.elasticsearch.transport.netty4.channel.PrivilegedNioServerSocketChannel;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -134,8 +133,6 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         boolSetting("http.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Property.NodeScope, Property.Shared);
     public static final Setting<Boolean> SETTING_HTTP_TCP_KEEP_ALIVE =
         boolSetting("http.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope, Property.Shared);
-    public static final Setting<Boolean> SETTING_HTTP_TCP_BLOCKING_SERVER =
-        boolSetting("http.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope, Property.Shared);
     public static final Setting<Boolean> SETTING_HTTP_TCP_REUSE_ADDRESS =
         boolSetting("http.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope, Property.Shared);
 
@@ -173,8 +170,6 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final int workerCount;
 
-    protected final boolean blockingServer;
-
     protected final boolean pipelining;
 
     protected final int pipeliningMaxEvents;
@@ -193,6 +188,10 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final boolean detailedErrorsEnabled;
     protected final ThreadPool threadPool;
+    /**
+     * The registry used to construct parsers so they support {@link XContentParser#namedObject(Class, String, Object)}.
+     */
+    protected final NamedXContentRegistry xContentRegistry;
 
     protected final boolean tcpNoDelay;
     protected final boolean tcpKeepAlive;
@@ -204,6 +203,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
 
     protected final ByteSizeValue maxCumulationBufferCapacity;
     protected final int maxCompositeBufferComponents;
+    private final Dispatcher dispatcher;
 
     protected volatile ServerBootstrap serverBootstrap;
 
@@ -214,15 +214,17 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     // package private for testing
     Netty4OpenChannelsHandler serverOpenChannels;
 
-    protected volatile HttpServerAdapter httpServerAdapter;
 
     private final Netty4CorsConfig corsConfig;
 
-    public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool) {
+    public Netty4HttpServerTransport(Settings settings, NetworkService networkService, BigArrays bigArrays, ThreadPool threadPool,
+                                     NamedXContentRegistry xContentRegistry, Dispatcher dispatcher) {
         super(settings);
         this.networkService = networkService;
         this.bigArrays = bigArrays;
         this.threadPool = threadPool;
+        this.xContentRegistry = xContentRegistry;
+        this.dispatcher = dispatcher;
 
         ByteSizeValue maxContentLength = SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings);
         this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
@@ -232,7 +234,6 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         this.maxCumulationBufferCapacity = SETTING_HTTP_NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
         this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
         this.workerCount = SETTING_HTTP_WORKER_COUNT.get(settings);
-        this.blockingServer = SETTING_HTTP_TCP_BLOCKING_SERVER.get(settings);
         this.port = SETTING_HTTP_PORT.get(settings);
         this.bindHosts = SETTING_HTTP_BIND_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
         this.publishHosts = SETTING_HTTP_PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
@@ -278,11 +279,6 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         return this.settings;
     }
 
-    @Override
-    public void httpServerAdapter(HttpServerAdapter httpServerAdapter) {
-        this.httpServerAdapter = httpServerAdapter;
-    }
-
     @Override
     protected void doStart() {
         boolean success = false;
@@ -290,15 +286,10 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
             this.serverOpenChannels = new Netty4OpenChannelsHandler(logger);
 
             serverBootstrap = new ServerBootstrap();
-            if (blockingServer) {
-                serverBootstrap.group(new OioEventLoopGroup(workerCount, daemonThreadFactory(settings,
-                    HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
-                serverBootstrap.channel(OioServerSocketChannel.class);
-            } else {
-                serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings,
-                    HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
-                serverBootstrap.channel(NioServerSocketChannel.class);
-            }
+
+            serverBootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings,
+                HTTP_SERVER_WORKER_THREAD_NAME_PREFIX)));
+            serverBootstrap.channel(PrivilegedNioServerSocketChannel.class);
 
             serverBootstrap.childHandler(configureServerChannelHandler());
 
@@ -323,6 +314,9 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
             serverBootstrap.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
 
             this.boundAddress = createBoundHttpAddress();
+            if (logger.isInfoEnabled()) {
+                logger.info("{}", boundAddress);
+            }
             success = true;
         } finally {
             if (success == false) {
@@ -412,14 +406,14 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         if (SETTING_CORS_ALLOW_CREDENTIALS.get(settings)) {
             builder.allowCredentials();
         }
-        String[] strMethods = Strings.splitStringByCommaToArray(SETTING_CORS_ALLOW_METHODS.get(settings));
+        String[] strMethods = Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_METHODS.get(settings), ",");
         HttpMethod[] methods = Arrays.asList(strMethods)
             .stream()
             .map(HttpMethod::valueOf)
             .toArray(size -> new HttpMethod[size]);
         return builder.allowedRequestMethods(methods)
             .maxAge(SETTING_CORS_MAX_AGE.get(settings))
-            .allowedRequestHeaders(Strings.splitStringByCommaToArray(SETTING_CORS_ALLOW_HEADERS.get(settings)))
+            .allowedRequestHeaders(Strings.tokenizeToStringArray(SETTING_CORS_ALLOW_HEADERS.get(settings), ","))
             .shortCircuit()
             .build();
     }
@@ -503,7 +497,7 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
     }
 
     protected void dispatchRequest(RestRequest request, RestChannel channel) {
-        httpServerAdapter.dispatchRequest(request, channel, threadPool.getThreadContext());
+        dispatcher.dispatch(request, channel, threadPool.getThreadContext());
     }
 
     protected void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
@@ -543,9 +537,9 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
         private final Netty4HttpRequestHandler requestHandler;
 
         protected HttpChannelHandler(
-            final Netty4HttpServerTransport transport,
-            final boolean detailedErrorsEnabled,
-            final ThreadContext threadContext) {
+                final Netty4HttpServerTransport transport,
+                final boolean detailedErrorsEnabled,
+                final ThreadContext threadContext) {
             this.transport = transport;
             this.requestHandler = new Netty4HttpRequestHandler(transport, detailedErrorsEnabled, threadContext);
         }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
index 32170a7d499..59f9447d61c 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.netty4.Netty4HttpServerTransport;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -57,7 +58,6 @@ public class Netty4Plugin extends Plugin implements NetworkPlugin {
             Netty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_NO_DELAY,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_KEEP_ALIVE,
-            Netty4HttpServerTransport.SETTING_HTTP_TCP_BLOCKING_SERVER,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_REUSE_ADDRESS,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_SEND_BUFFER_SIZE,
             Netty4HttpServerTransport.SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE,
@@ -94,8 +94,10 @@ public class Netty4Plugin extends Plugin implements NetworkPlugin {
     public Map<String, Supplier<HttpServerTransport>> getHttpTransports(Settings settings, ThreadPool threadPool, BigArrays bigArrays,
                                                                         CircuitBreakerService circuitBreakerService,
                                                                         NamedWriteableRegistry namedWriteableRegistry,
-                                                                        NetworkService networkService) {
+                                                                        NamedXContentRegistry xContentRegistry,
+                                                                        NetworkService networkService,
+                                                                        HttpServerTransport.Dispatcher dispatcher) {
         return Collections.singletonMap(NETTY_HTTP_TRANSPORT_NAME,
-            () -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool));
+            () -> new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher));
     }
 }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
index 45aa029b46f..f29ca9c10ff 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/ByteBufStreamInput.java
@@ -36,7 +36,7 @@ class ByteBufStreamInput extends StreamInput {
     private final int startIndex;
     private final int endIndex;
 
-    public ByteBufStreamInput(ByteBuf buffer, int length) {
+    ByteBufStreamInput(ByteBuf buffer, int length) {
         if (length > buffer.readableBytes()) {
             throw new IndexOutOfBoundsException();
         }
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
index d92313fb0bb..bb0fda63642 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java
@@ -32,11 +32,6 @@ import io.netty.channel.ChannelOption;
 import io.netty.channel.FixedRecvByteBufAllocator;
 import io.netty.channel.RecvByteBufAllocator;
 import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.oio.OioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.channel.socket.oio.OioServerSocketChannel;
-import io.netty.channel.socket.oio.OioSocketChannel;
 import io.netty.util.concurrent.Future;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
@@ -67,6 +62,8 @@ import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.TransportSettings;
+import org.elasticsearch.transport.netty4.channel.PrivilegedNioServerSocketChannel;
+import org.elasticsearch.transport.netty4.channel.PrivilegedNioSocketChannel;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -193,13 +190,8 @@ public class Netty4Transport extends TcpTransport<Channel> {
 
     private Bootstrap createBootstrap() {
         final Bootstrap bootstrap = new Bootstrap();
-        if (TCP_BLOCKING_CLIENT.get(settings)) {
-            bootstrap.group(new OioEventLoopGroup(1, daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)));
-            bootstrap.channel(OioSocketChannel.class);
-        } else {
-            bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)));
-            bootstrap.channel(NioSocketChannel.class);
-        }
+        bootstrap.group(new NioEventLoopGroup(workerCount, daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)));
+        bootstrap.channel(PrivilegedNioSocketChannel.class);
 
         bootstrap.handler(getClientChannelInitializer());
 
@@ -282,13 +274,8 @@ public class Netty4Transport extends TcpTransport<Channel> {
 
         final ServerBootstrap serverBootstrap = new ServerBootstrap();
 
-        if (TCP_BLOCKING_SERVER.get(settings)) {
-            serverBootstrap.group(new OioEventLoopGroup(workerCount, workerFactory));
-            serverBootstrap.channel(OioServerSocketChannel.class);
-        } else {
-            serverBootstrap.group(new NioEventLoopGroup(workerCount, workerFactory));
-            serverBootstrap.channel(NioServerSocketChannel.class);
-        }
+        serverBootstrap.group(new NioEventLoopGroup(workerCount, workerFactory));
+        serverBootstrap.channel(PrivilegedNioServerSocketChannel.class);
 
         serverBootstrap.childHandler(getServerChannelInitializer(name, settings));
 
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
index 08d825a354d..4d28bf9a257 100644
--- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java
@@ -133,8 +133,7 @@ public class Netty4Utils {
              * Here be dragons. We want to rethrow this so that it bubbles up to the uncaught exception handler. Yet, Netty wraps too many
              * invocations of user-code in try/catch blocks that swallow all throwables. This means that a rethrow here will not bubble up
              * to where we want it to. So, we fork a thread and throw the exception from there where Netty can not get to it. We do not wrap
-             * the exception so as to not lose the original cause during exit, so we give the thread a name based on the previous stack
-             * frame so that at least we know where it came from (in case logging the current stack trace fails).
+             * the exception so as to not lose the original cause during exit.
              */
             try {
                 // try to log the current stack trace
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioServerSocketChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioServerSocketChannel.java
new file mode 100644
index 00000000000..f49d0fed04c
--- /dev/null
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioServerSocketChannel.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty4.channel;
+
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import org.elasticsearch.SpecialPermission;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+/**
+ * Wraps netty calls to {@link java.nio.channels.ServerSocketChannel#accept()} in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks. This is necessary to limit
+ * {@link java.net.SocketPermission} to the transport module.
+ */
+public class PrivilegedNioServerSocketChannel extends NioServerSocketChannel {
+
+    @Override
+    protected int doReadMessages(List<Object> buf) throws Exception {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<Integer>) () -> super.doReadMessages(buf));
+        } catch (PrivilegedActionException e) {
+            throw (Exception) e.getCause();
+        }
+    }
+}
diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioSocketChannel.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioSocketChannel.java
new file mode 100644
index 00000000000..1eed118b8af
--- /dev/null
+++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/channel/PrivilegedNioSocketChannel.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty4.channel;
+
+import io.netty.channel.socket.nio.NioSocketChannel;
+import org.elasticsearch.SpecialPermission;
+
+import java.net.SocketAddress;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * Wraps netty calls to {@link java.nio.channels.SocketChannel#connect(SocketAddress)} in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks. This is necessary to limit
+ * {@link java.net.SocketPermission} to the transport module.
+ */
+public class PrivilegedNioSocketChannel extends NioSocketChannel {
+
+    @Override
+    protected boolean doConnect(SocketAddress remoteAddress, SocketAddress localAddress) throws Exception {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<Boolean>) () -> super.doConnect(remoteAddress, localAddress));
+        } catch (PrivilegedActionException e) {
+            throw (Exception) e.getCause();
+        }
+    }
+}
diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
index ce39869d2fb..11c564cc383 100644
--- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy
@@ -17,12 +17,17 @@
  * under the License.
  */
 
-grant codeBase "${codebase.netty-common-4.1.6.Final.jar}" {
+grant {
+   // netty makes and accepts socket connections
+   permission java.net.SocketPermission "*", "accept,connect";
+};
+
+grant codeBase "${codebase.netty-common-4.1.7.Final.jar}" {
    // for reading the system-wide configuration for the backlog of established sockets
    permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read";
 };
 
-grant codeBase "${codebase.netty-transport-4.1.6.Final.jar}" {
+grant codeBase "${codebase.netty-transport-4.1.7.Final.jar}" {
    // Netty NioEventLoop wants to change this, because of https://bugs.openjdk.java.net/browse/JDK-6427854
    // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely!
    permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write";
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
index 237227cd4df..2341f3905ef 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java
@@ -26,7 +26,6 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -39,7 +38,7 @@ public class Netty4ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
index a5e0381b3fd..c7427e717b3 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java
@@ -41,6 +41,7 @@ import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.util.Attribute;
 import io.netty.util.AttributeKey;
+
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -187,12 +188,13 @@ public class Netty4HttpChannelTests extends ESTestCase {
     public void testHeadersSet() {
         Settings settings = Settings.builder().build();
         try (Netty4HttpServerTransport httpServerTransport =
-                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) {
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+                         (request, channel, context) -> {})) {
             httpServerTransport.start();
             final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
             httpRequest.headers().add(HttpHeaderNames.ORIGIN, "remote");
             final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-            Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel);
+            Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
             // send a response
             Netty4HttpChannel channel =
@@ -217,7 +219,8 @@ public class Netty4HttpChannelTests extends ESTestCase {
     public void testConnectionClose() throws Exception {
         final Settings settings = Settings.builder().build();
         try (Netty4HttpServerTransport httpServerTransport =
-                 new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) {
+                 new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+                     (request, channel, context) -> {})) {
             httpServerTransport.start();
             final FullHttpRequest httpRequest;
             final boolean close = randomBoolean();
@@ -233,7 +236,7 @@ public class Netty4HttpChannelTests extends ESTestCase {
                 }
             }
             final EmbeddedChannel embeddedChannel = new EmbeddedChannel();
-            final Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, embeddedChannel);
+            final Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, embeddedChannel);
 
             // send a response, the channel close status should match
             assertTrue(embeddedChannel.isOpen());
@@ -252,7 +255,8 @@ public class Netty4HttpChannelTests extends ESTestCase {
     private FullHttpResponse executeRequest(final Settings settings, final String originValue, final String host) {
         // construct request and send it over the transport layer
         try (Netty4HttpServerTransport httpServerTransport =
-                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool)) {
+                     new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool, xContentRegistry(),
+                         (request, channel, context) -> {})) {
             httpServerTransport.start();
             final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
             if (originValue != null) {
@@ -260,7 +264,7 @@ public class Netty4HttpChannelTests extends ESTestCase {
             }
             httpRequest.headers().add(HttpHeaderNames.HOST, host);
             final WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
-            final Netty4HttpRequest request = new Netty4HttpRequest(httpRequest, writeCapturingChannel);
+            final Netty4HttpRequest request = new Netty4HttpRequest(xContentRegistry(), httpRequest, writeCapturingChannel);
 
             Netty4HttpChannel channel =
                     new Netty4HttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
index ad1fedc49b0..10504045634 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpClient.java
@@ -28,7 +28,6 @@ import io.netty.channel.ChannelInitializer;
 import io.netty.channel.SimpleChannelInboundHandler;
 import io.netty.channel.nio.NioEventLoopGroup;
 import io.netty.channel.socket.SocketChannel;
-import io.netty.channel.socket.nio.NioSocketChannel;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
@@ -44,6 +43,7 @@ import io.netty.handler.codec.http.HttpVersion;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.transport.netty4.channel.PrivilegedNioSocketChannel;
 
 import java.io.Closeable;
 import java.net.SocketAddress;
@@ -82,7 +82,7 @@ class Netty4HttpClient implements Closeable {
     private final Bootstrap clientBootstrap;
 
     Netty4HttpClient() {
-        clientBootstrap = new Bootstrap().channel(NioSocketChannel.class).group(new NioEventLoopGroup());
+        clientBootstrap = new Bootstrap().channel(PrivilegedNioSocketChannel.class).group(new NioEventLoopGroup());
     }
 
     public Collection<FullHttpResponse> get(SocketAddress remoteAddress, String... uris) throws InterruptedException {
@@ -122,6 +122,7 @@ class Netty4HttpClient implements Closeable {
             HttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, method, uriAndBody.v1(), content);
             request.headers().add(HttpHeaderNames.HOST, "localhost");
             request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
+            request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json");
             requests.add(request);
         }
         return sendRequests(remoteAddress, requests);
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
index 701baf80aed..d7e2d7782d5 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java
@@ -87,7 +87,7 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
             .put("http.pipelining", true)
             .put("http.port", "0")
             .build();
-        try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
+        try (HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
             httpServerTransport.start();
             final TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
 
@@ -114,7 +114,7 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
             .put("http.pipelining", false)
             .put("http.port", "0")
             .build();
-        try (final HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
+        try (HttpServerTransport httpServerTransport = new CustomNettyHttpServerTransport(settings)) {
             httpServerTransport.start();
             final TransportAddress transportAddress = randomFrom(httpServerTransport.boundAddress().boundAddresses());
 
@@ -159,7 +159,8 @@ public class Netty4HttpServerPipeliningTests extends ESTestCase {
             super(settings,
                 Netty4HttpServerPipeliningTests.this.networkService,
                 Netty4HttpServerPipeliningTests.this.bigArrays,
-                Netty4HttpServerPipeliningTests.this.threadPool);
+                Netty4HttpServerPipeliningTests.this.threadPool,
+                xContentRegistry(), (request, channel, context) -> {});
         }
 
         @Override
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
index 498daf63226..69dcaad9dd3 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java
@@ -51,6 +51,7 @@ import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.common.Strings.collectionToDelimitedString;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
@@ -89,11 +90,12 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
     public void testCorsConfig() {
         final Set<String> methods = new HashSet<>(Arrays.asList("get", "options", "post"));
         final Set<String> headers = new HashSet<>(Arrays.asList("Content-Type", "Content-Length"));
+        final String suffix = randomBoolean() ? " " : ""; // sometimes have a leading whitespace between comma delimited elements
         final Settings settings = Settings.builder()
                                       .put(SETTING_CORS_ENABLED.getKey(), true)
                                       .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "*")
-                                      .put(SETTING_CORS_ALLOW_METHODS.getKey(), Strings.collectionToCommaDelimitedString(methods))
-                                      .put(SETTING_CORS_ALLOW_HEADERS.getKey(), Strings.collectionToCommaDelimitedString(headers))
+                                      .put(SETTING_CORS_ALLOW_METHODS.getKey(), collectionToDelimitedString(methods, ",", suffix, ""))
+                                      .put(SETTING_CORS_ALLOW_HEADERS.getKey(), collectionToDelimitedString(headers, ",", suffix, ""))
                                       .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
                                       .build();
         final Netty4CorsConfig corsConfig = Netty4HttpServerTransport.buildCorsConfig(settings);
@@ -120,9 +122,9 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
      * Test that {@link Netty4HttpServerTransport} supports the "Expect: 100-continue" HTTP header
      */
     public void testExpectContinueHeader() throws Exception {
-        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool)) {
-            transport.httpServerAdapter((request, channel, context) ->
-                    channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done"))));
+        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
+                xContentRegistry(), (request, channel, context) ->
+            channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done"))))) {
             transport.start();
             TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
 
@@ -143,12 +145,13 @@ public class Netty4HttpServerTransportTests extends ESTestCase {
     }
 
     public void testBindUnavailableAddress() {
-        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool)) {
+        try (Netty4HttpServerTransport transport = new Netty4HttpServerTransport(Settings.EMPTY, networkService, bigArrays, threadPool,
+                xContentRegistry(), (request, channel, context) -> {})) {
             transport.start();
             TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
             Settings settings = Settings.builder().put("http.port", remoteAddress.getPort()).build();
-            try (Netty4HttpServerTransport otherTransport = new Netty4HttpServerTransport(settings, networkService, bigArrays,
-                threadPool)) {
+            try (Netty4HttpServerTransport otherTransport = new Netty4HttpServerTransport(settings, networkService, bigArrays, threadPool,
+                    xContentRegistry(), (request, channel, context) -> {})) {
                 BindHttpException bindHttpException = expectThrows(BindHttpException.class, () -> otherTransport.start());
                 assertEquals("Failed to bind to [" + remoteAddress.getPort() + "]", bindHttpException.getMessage());
             }
diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
index 0e9ebe5f7f4..15dea8fe871 100644
--- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
+++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportSettings;
@@ -84,7 +85,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase {
         String randomMethod = randomFrom("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH");
         String data = randomMethod + " / HTTP/1.1";
 
-        try (Socket socket = new Socket(host, port)) {
+        try (Socket socket = new MockSocket(host, port)) {
             socket.getOutputStream().write(data.getBytes(StandardCharsets.UTF_8));
             socket.getOutputStream().flush();
 
@@ -95,7 +96,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase {
     }
 
     public void testThatNothingIsReturnedForOtherInvalidPackets() throws Exception {
-        try (Socket socket = new Socket(host, port)) {
+        try (Socket socket = new MockSocket(host, port)) {
             socket.getOutputStream().write("FOOBAR".getBytes(StandardCharsets.UTF_8));
             socket.getOutputStream().flush();
 
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 7ee6c4f0787..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0850319baf063c5ee54aecabeaddb95efde8711b
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0.jar.sha1
new file mode 100644
index 00000000000..c5796663317
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.4.0.jar.sha1
@@ -0,0 +1 @@
+0242cfe1a026ec3afd5a567237f634da1d95fa24
\ No newline at end of file
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
index 091189f361a..7710c2e6715 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
@@ -131,7 +131,7 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
             }
         }
 
-        Boolean caseLevel = settings.getAsBoolean("caseLevel", null);
+        Boolean caseLevel = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "caseLevel", null, deprecationLogger);
         if (caseLevel != null) {
             rbc.setCaseLevel(caseLevel);
         }
@@ -147,7 +147,7 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
             }
         }
 
-        Boolean numeric = settings.getAsBoolean("numeric", null);
+        Boolean numeric = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "numeric", null, deprecationLogger);
         if (numeric != null) {
             rbc.setNumericCollation(numeric);
         }
@@ -157,7 +157,8 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
             rbc.setVariableTop(variableTop);
         }
 
-        Boolean hiraganaQuaternaryMode = settings.getAsBoolean("hiraganaQuaternaryMode", null);
+        Boolean hiraganaQuaternaryMode = settings
+            .getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "hiraganaQuaternaryMode", null, deprecationLogger);
         if (hiraganaQuaternaryMode != null) {
             rbc.setHiraganaQuaternary(hiraganaQuaternaryMode);
         }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
index ce2e660ecfc..04fc222063c 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IcuClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index c66710ea344..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3a2af1d2e80b9901b3e950f5ac1b6cd1eb408fd3
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0.jar.sha1
new file mode 100644
index 00000000000..b1974f267b4
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.4.0.jar.sha1
@@ -0,0 +1 @@
+fdefb46dde0019b8173d19fe4b164dca9efcba14
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java
index d10fe4089f2..712ad0add15 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/JapaneseStopTokenFilterFactory.java
@@ -45,8 +45,9 @@ public class JapaneseStopTokenFilterFactory extends AbstractTokenFilterFactory{
 
     public JapaneseStopTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        this.ignoreCase = settings.getAsBoolean("ignore_case", false);
-        this.removeTrailing = settings.getAsBoolean("remove_trailing", true);
+        this.ignoreCase = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "ignore_case", false, deprecationLogger);
+        this.removeTrailing = settings
+            .getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "remove_trailing", true, deprecationLogger);
         this.stopWords = Analysis.parseWords(env, settings, "stopwords", JapaneseAnalyzer.getDefaultStopSet(), NAMED_STOP_WORDS, ignoreCase);
     }
 
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
index 1776977c8e2..3749f957a3c 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiAnalyzerProvider.java
@@ -35,7 +35,8 @@ public class KuromojiAnalyzerProvider extends AbstractIndexAnalyzerProvider<Japa
 
     public KuromojiAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        final Set<?> stopWords = Analysis.parseStopWords(env, settings, JapaneseAnalyzer.getDefaultStopSet());
+        final Set<?> stopWords = Analysis.parseStopWords(
+            env, indexSettings.getIndexVersionCreated(), settings, JapaneseAnalyzer.getDefaultStopSet());
         final JapaneseTokenizer.Mode mode = KuromojiTokenizerFactory.getMode(settings);
         final UserDictionary userDictionary = KuromojiTokenizerFactory.getUserDictionary(env, settings);
         analyzer = new JapaneseAnalyzer(userDictionary, mode, CharArraySet.copy(stopWords), JapaneseAnalyzer.getDefaultStopTags());
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
index 836dbbdfae2..5bdafbe39e4 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
@@ -33,8 +33,10 @@ public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFa
 
     public KuromojiIterationMarkCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name);
-        normalizeKanji = settings.getAsBoolean("normalize_kanji", JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT);
-        normalizeKana = settings.getAsBoolean("normalize_kana", JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT);
+        normalizeKanji = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "normalize_kanji",
+            JapaneseIterationMarkCharFilter.NORMALIZE_KANJI_DEFAULT, deprecationLogger);
+        normalizeKana = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "normalize_kana",
+            JapaneseIterationMarkCharFilter.NORMALIZE_KANA_DEFAULT, deprecationLogger);
     }
 
     @Override
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
index d0eb0cecdb9..4b85368da93 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiReadingFormFilterFactory.java
@@ -31,7 +31,8 @@ public class KuromojiReadingFormFilterFactory extends AbstractTokenFilterFactory
 
     public KuromojiReadingFormFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        useRomaji = settings.getAsBoolean("use_romaji", false);
+        useRomaji =
+            settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "use_romaji", false, deprecationLogger);
     }
 
     @Override
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
index 2f00e68a75e..69da3bd2bc4 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiTokenizerFactory.java
@@ -48,7 +48,8 @@ public class KuromojiTokenizerFactory extends AbstractTokenizerFactory {
         super(indexSettings, name, settings);
         mode = getMode(settings);
         userDictionary = getUserDictionary(env, settings);
-        discartPunctuation = settings.getAsBoolean("discard_punctuation", true);
+        discartPunctuation = settings
+            .getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "discard_punctuation", true, deprecationLogger);
         nBestCost = settings.getAsInt(NBEST_COST, -1);
         nBestExamples = settings.get(NBEST_EXAMPLES);
     }
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
index 0797b10d774..bc5360f94d1 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class KuromojiClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 06702f8d87e..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7e9243da1482f88a91bd5239316b571259d24341
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0.jar.sha1
new file mode 100644
index 00000000000..8d80c614fe1
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.4.0.jar.sha1
@@ -0,0 +1 @@
+e75472bdbd3480ceddc570e747c70640429598d6
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
index ff4ab4943e3..5c9a6a463d3 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/PhoneticTokenFilterFactory.java
@@ -60,7 +60,7 @@ public class PhoneticTokenFilterFactory extends AbstractTokenFilterFactory {
         this.nametype = null;
         this.ruletype = null;
         this.maxcodelength = 0;
-        this.replace = settings.getAsBoolean("replace", true);
+        this.replace = settings.getAsBooleanLenientForPreEs6Indices(indexSettings.getIndexVersionCreated(), "replace", true, deprecationLogger);
         // weird, encoder is null at last step in SimplePhoneticAnalysisTests, so we set it to metaphone as default
         String encodername = settings.get("encoder", "metaphone");
         if ("metaphone".equalsIgnoreCase(encodername)) {
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
index 447eb1d6cd7..3d892bddee4 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class PhoneticClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 561a46f2a2c..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2ead714733bb3cc90e9792d76021497946d5af09
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0.jar.sha1
new file mode 100644
index 00000000000..b9a56da53da
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.4.0.jar.sha1
@@ -0,0 +1 @@
+31ebae985b936c2dcbe16126492deccbb5192ce8
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
index 534af79a199..24a581d9e26 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmartCNClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index c3b55dc76a1..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9a8f3b58e6c672276331f54b5c3be49c8014ec5c
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0.jar.sha1
new file mode 100644
index 00000000000..373102540ea
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.4.0.jar.sha1
@@ -0,0 +1 @@
+684bc70e964bfb93224962b24912b1bab2a5b673
\ No newline at end of file
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
index afc7d527a6c..aa3194c5831 100644
--- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/index/analysis/pl/PolishStemTokenFilterFactory.java
@@ -35,20 +35,11 @@ import java.io.IOException;
 
 public class PolishStemTokenFilterFactory extends AbstractTokenFilterFactory {
 
-    private final StempelStemmer stemmer;
-
     public PolishStemTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
-        Trie tire;
-        try {
-            tire = StempelStemmer.load(PolishAnalyzer.class.getResourceAsStream(PolishAnalyzer.DEFAULT_STEMMER_FILE));
-        } catch (IOException ex) {
-            throw new RuntimeException("Unable to load default stemming tables", ex);
-        }
-        stemmer = new StempelStemmer(tire);
     }
 
     @Override public TokenStream create(TokenStream tokenStream) {
-        return new StempelFilter(tokenStream, stemmer);
+        return new StempelFilter(tokenStream, new StempelStemmer(PolishAnalyzer.getDefaultTable()));
     }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
index abf739d010a..e68cb260b0b 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
@@ -19,12 +19,24 @@
 
 package org.elasticsearch.index.analysis;
 
-import org.elasticsearch.AnalysisFactoryTestCase;
-import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
-
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.AnalysisFactoryTestCase;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
+
 public class AnalysisPolishFactoryTests extends AnalysisFactoryTestCase {
 
     @Override
@@ -34,4 +46,31 @@ public class AnalysisPolishFactoryTests extends AnalysisFactoryTestCase {
         return filters;
     }
 
+    public void testThreadSafety() throws IOException {
+        // TODO: is this the right boilerplate?  I forked this out of TransportAnalyzeAction.java:
+        Settings settings = Settings.builder()
+            // for _na_
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        Environment environment = new Environment(settings);
+        IndexMetaData metaData = IndexMetaData.builder(IndexMetaData.INDEX_UUID_NA_VALUE).settings(settings).build();
+        IndexSettings indexSettings = new IndexSettings(metaData, Settings.EMPTY);
+        testThreadSafety(new PolishStemTokenFilterFactory(indexSettings, environment, "stempelpolishstem", settings));
+    }
+
+    // TODO: move to AnalysisFactoryTestCase so we can more easily test thread safety for all factories
+    private void testThreadSafety(TokenFilterFactory factory) throws IOException {
+        final Analyzer analyzer = new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                Tokenizer tokenizer = new MockTokenizer();
+                return new TokenStreamComponents(tokenizer, factory.create(tokenizer));
+            }
+        };
+        BaseTokenStreamTestCase.checkRandomData(random(), analyzer, 100);
+    }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
index 56edcdb692c..371431e1c25 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class StempelClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-ec38570.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-ec38570.jar.sha1
deleted file mode 100644
index 3fbc82d91c1..00000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0-snapshot-ec38570.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-763b3144b9bc53328e923242a3c6614903ee2d7e
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0.jar.sha1
new file mode 100644
index 00000000000..7573eb01c88
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.4.0.jar.sha1
@@ -0,0 +1 @@
+07a2471f5da4a69be367851505338b53ceca0ef1
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java
index 45bf27b954b..0a00b752b99 100644
--- a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java
+++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java
@@ -24,8 +24,6 @@ import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider;
-import org.elasticsearch.index.analysis.Analysis;
 
 public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider<UkrainianMorfologikAnalyzer> {
 
@@ -33,8 +31,10 @@ public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Ukr
 
     public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
-        analyzer = new UkrainianMorfologikAnalyzer(Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()),
-                Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
+        analyzer = new UkrainianMorfologikAnalyzer(
+            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()),
+            Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
+        );
         analyzer.setVersion(version);
     }
 
diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
index dd77fdf74a5..50d935e5228 100644
--- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
+++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
index 09519b14499..4a922e36084 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.cloud.azure.classic.management;
 
 import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.ServiceLoader;
 
 import com.microsoft.windowsazure.Configuration;
@@ -31,6 +34,7 @@ import com.microsoft.windowsazure.management.compute.ComputeManagementService;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.configuration.ManagementConfiguration;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cloud.azure.classic.AzureServiceRemoteException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -89,10 +93,12 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent
 
     @Override
     public HostedServiceGetDetailedResponse getServiceDetails() {
+        SpecialPermission.check();
         try {
-            return client.getHostedServicesOperations().getDetailed(serviceName);
-        } catch (Exception e) {
-            throw new AzureServiceRemoteException("can not get list of azure nodes", e);
+            return AccessController.doPrivileged((PrivilegedExceptionAction<HostedServiceGetDetailedResponse>)
+                () -> client.getHostedServicesOperations().getDetailed(serviceName));
+        } catch (PrivilegedActionException e) {
+            throw new AzureServiceRemoteException("can not get list of azure nodes", e.getCause());
         }
     }
 
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
index 9735b83d9e6..1661c69c725 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceImpl;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
@@ -74,10 +75,11 @@ public class AzureDiscoveryPlugin extends Plugin implements DiscoveryPlugin {
 
     @Override
     public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
                                                               ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
         return Collections.singletonMap(AZURE, () ->
-            new ZenDiscovery(settings, threadPool, transportService, clusterService, hostsProvider));
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
     }
 
     @Override
diff --git a/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 00000000000..4407898ece4
--- /dev/null
+++ b/plugins/discovery-azure-classic/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // azure client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
+};
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index 7f977592e8d..9787dc136e3 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -131,7 +132,7 @@ public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
     public static void startHttpd() throws Exception {
         logDir = createTempDir();
         SSLContext sslContext = getSSLContext();
-        httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+        httpsServer = MockHttpServer.createHttps(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
         httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
         httpsServer.createContext("/subscription/services/hostedservices/myservice", (s) -> {
             Headers headers = s.getResponseHeaders();
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
index 33c5d41f70c..4f6a44ef235 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class DiscoveryAzureClassicClientYamlTestSuiteIT extends ESClientYamlSuit
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
index d15e96f36ab..125dfa4a69a 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
@@ -164,20 +164,22 @@ public interface AwsEc2Service {
      * Defines discovery settings for ec2. Starting with discovery.ec2.
      */
     interface DISCOVERY_EC2 {
-        enum HostType {
-            PRIVATE_IP,
-            PUBLIC_IP,
-            PRIVATE_DNS,
-            PUBLIC_DNS
+        class HostType {
+            public static final String PRIVATE_IP = "private_ip";
+            public static final String PUBLIC_IP = "public_ip";
+            public static final String PRIVATE_DNS = "private_dns";
+            public static final String PUBLIC_DNS = "public_dns";
+            public static final String TAG_PREFIX = "tag:";
         }
 
         /**
          * discovery.ec2.host_type: The type of host type to use to communicate with other instances.
-         * Can be one of private_ip, public_ip, private_dns, public_dns. Defaults to private_ip.
+         * Can be one of private_ip, public_ip, private_dns, public_dns or tag:XXXX where
+         * XXXX refers to a name of a tag configured for all EC2 instances. Instances which don't
+         * have this tag set will be ignored by the discovery process. Defaults to private_ip.
          */
-        Setting<HostType> HOST_TYPE_SETTING =
-            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP.name(), s -> HostType.valueOf(s.toUpperCase(Locale.ROOT)),
-                Property.NodeScope);
+        Setting<String> HOST_TYPE_SETTING =
+            new Setting<>("discovery.ec2.host_type", HostType.PRIVATE_IP, Function.identity(), Property.NodeScope);
         /**
          * discovery.ec2.any_group: If set to false, will require all security groups to be present for the instance to be used for the
          * discovery. Defaults to true.
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
index 7eac1184fbf..f47a2f398da 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
@@ -178,6 +178,9 @@ public class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Servic
                 case "eu-west-1":
                     endpoint = "ec2.eu-west-1.amazonaws.com";
                     break;
+                case "eu-west-2":
+                    endpoint = "ec2.eu-west-2.amazonaws.com";
+                    break;
                 case "eu-central":
                 case "eu-central-1":
                     endpoint = "ec2.eu-central-1.amazonaws.com";
@@ -190,6 +193,10 @@ public class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Servic
                 case "cn-north-1":
                     endpoint = "ec2.cn-north-1.amazonaws.com.cn";
                     break;
+                case "ca-central":
+                case "ca-central-1":
+                    endpoint = "ec2.ca-central-1.amazonaws.com";
+                    break;
                 default:
                     throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
             }
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java
index 3788f8237dc..4350b4d5277 100755
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cloud.aws.network;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
+import org.elasticsearch.cloud.aws.util.SocketAccess;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
 import org.elasticsearch.common.settings.Settings;
@@ -58,7 +59,7 @@ public class Ec2NameResolver extends AbstractComponent implements CustomNameReso
      *
      * @author Paul_Loy
      */
-    private static enum Ec2HostnameType {
+    private enum Ec2HostnameType {
 
         PRIVATE_IPv4("ec2:privateIpv4", "local-ipv4"),
         PRIVATE_DNS("ec2:privateDns", "local-hostname"),
@@ -73,7 +74,7 @@ public class Ec2NameResolver extends AbstractComponent implements CustomNameReso
         final String configName;
         final String ec2Name;
 
-        private Ec2HostnameType(String configName, String ec2Name) {
+        Ec2HostnameType(String configName, String ec2Name) {
             this.configName = configName;
             this.ec2Name = ec2Name;
         }
@@ -97,9 +98,9 @@ public class Ec2NameResolver extends AbstractComponent implements CustomNameReso
         try {
             URL url = new URL(metadataUrl);
             logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url);
-            URLConnection urlConnection = url.openConnection();
+            URLConnection urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
             urlConnection.setConnectTimeout(2000);
-            in = urlConnection.getInputStream();
+            in = SocketAccess.doPrivilegedIOException(urlConnection::getInputStream);
             BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
 
             String metadataResult = urlReader.readLine();
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java
new file mode 100644
index 00000000000..2c68369bb99
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.aws.util;
+
+import org.elasticsearch.SpecialPermission;
+
+import java.io.IOException;
+import java.net.SocketPermission;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * This plugin uses aws libraries to connect to aws services. For these remote calls the plugin needs
+ * {@link SocketPermission} 'connect' to establish connections. This class wraps the operations requiring access in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks.
+ */
+public final class SocketAccess {
+
+    private SocketAccess() {}
+
+    public static <T> T doPrivileged(PrivilegedAction<T> operation) {
+        SpecialPermission.check();
+        return AccessController.doPrivileged(operation);
+    }
+
+    public static <T> T doPrivilegedIOException(PrivilegedExceptionAction<T> operation) throws IOException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
+}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
index c3b7731692b..d2a8f6e5f3f 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
@@ -19,12 +19,6 @@
 
 package org.elasticsearch.discovery.ec2;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
@@ -33,11 +27,13 @@ import com.amazonaws.services.ec2.model.Filter;
 import com.amazonaws.services.ec2.model.GroupIdentifier;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.Reservation;
+import com.amazonaws.services.ec2.model.Tag;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.Version;
 import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2;
+import org.elasticsearch.cloud.aws.util.SocketAccess;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
@@ -56,6 +52,11 @@ import java.util.Set;
 import static java.util.Collections.disjoint;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.TAG_PREFIX;
+import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_DNS;
+import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PRIVATE_IP;
+import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_DNS;
+import static org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2.HostType.PUBLIC_IP;
 
 public class AwsEc2UnicastHostsProvider extends AbstractComponent implements UnicastHostsProvider {
 
@@ -71,7 +72,7 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
 
     private final Set<String> availabilityZones;
 
-    private final DISCOVERY_EC2.HostType hostType;
+    private final String hostType;
 
     private final DiscoNodesCache discoNodes;
 
@@ -114,7 +115,7 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
             // NOTE: we don't filter by security group during the describe instances request for two reasons:
             // 1. differences in VPCs require different parameters during query (ID vs Name)
             // 2. We want to use two different strategies: (all security groups vs. any security groups)
-            descInstances = client.describeInstances(buildDescribeInstancesRequest());
+            descInstances = SocketAccess.doPrivileged(() -> client.describeInstances(buildDescribeInstancesRequest()));
         } catch (AmazonClientException e) {
             logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
             logger.debug("Full exception:", e);
@@ -154,19 +155,27 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
                 }
 
                 String address = null;
-                switch (hostType) {
-                    case PRIVATE_DNS:
-                        address = instance.getPrivateDnsName();
-                        break;
-                    case PRIVATE_IP:
-                        address = instance.getPrivateIpAddress();
-                        break;
-                    case PUBLIC_DNS:
-                        address = instance.getPublicDnsName();
-                        break;
-                    case PUBLIC_IP:
-                        address = instance.getPublicIpAddress();
-                        break;
+                if (hostType.equals(PRIVATE_DNS)) {
+                    address = instance.getPrivateDnsName();
+                } else if (hostType.equals(PRIVATE_IP)) {
+                    address = instance.getPrivateIpAddress();
+                } else if (hostType.equals(PUBLIC_DNS)) {
+                    address = instance.getPublicDnsName();
+                } else if (hostType.equals(PUBLIC_IP)) {
+                    address = instance.getPublicIpAddress();
+                } else if (hostType.startsWith(TAG_PREFIX)) {
+                    // Reading the node host from its metadata
+                    String tagName = hostType.substring(TAG_PREFIX.length());
+                    logger.debug("reading hostname from [{}] instance tag", tagName);
+                    List<Tag> tags = instance.getTags();
+                    for (Tag tag : tags) {
+                        if (tag.getKey().equals(tagName)) {
+                            address = tag.getValue();
+                            logger.debug("using [{}] as the instance address", address);
+                        }
+                    }
+                } else {
+                    throw new IllegalArgumentException(hostType + " is unknown for discovery.ec2.host_type");
                 }
                 if (address != null) {
                     try {
@@ -174,8 +183,8 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
                         TransportAddress[] addresses = transportService.addressesFromString(address, 1);
                         for (int i = 0; i < addresses.length; i++) {
                             logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
-                            discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i],
-                                    emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
+                            discoNodes.add(new DiscoveryNode(instance.getInstanceId(), "#cloud-" + instance.getInstanceId() + "-" + i,
+                                addresses[i], emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                         }
                     } catch (Exception e) {
                         final String finalAddress = address;
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index b7181270f62..fd30623fb80 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -44,7 +44,9 @@ import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
 import org.elasticsearch.cloud.aws.network.Ec2NameResolver;
+import org.elasticsearch.cloud.aws.util.SocketAccess;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
@@ -55,7 +57,6 @@ import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.ZenPing;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -70,24 +71,20 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
     public static final String EC2 = "ec2";
 
     static {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                try {
-                    // kick jackson to do some static caching of declared members info
-                    Jackson.jsonNodeOf("{}");
-                    // ClientConfiguration clinit has some classloader problems
-                    // TODO: fix that
-                    Class.forName("com.amazonaws.ClientConfiguration");
-                } catch (ClassNotFoundException e) {
-                    throw new RuntimeException(e);
-                }
-                return null;
+        SpecialPermission.check();
+        // Initializing Jackson requires RuntimePermission accessDeclaredMembers
+        // The ClientConfiguration class requires RuntimePermission getClassLoader
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            try {
+                // kick jackson to do some static caching of declared members info
+                Jackson.jsonNodeOf("{}");
+                // ClientConfiguration clinit has some classloader problems
+                // TODO: fix that
+                Class.forName("com.amazonaws.ClientConfiguration");
+            } catch (ClassNotFoundException e) {
+                throw new RuntimeException(e);
             }
+            return null;
         });
     }
 
@@ -101,10 +98,11 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
 
     @Override
     public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
                                                               ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
         return Collections.singletonMap(EC2, () ->
-            new ZenDiscovery(settings, threadPool, transportService, clusterService, hostsProvider));
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
     }
 
     @Override
@@ -192,14 +190,14 @@ public class Ec2DiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
         try {
             url = new URL(azMetadataUrl);
             logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
-            urlConnection = url.openConnection();
+            urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
             urlConnection.setConnectTimeout(2000);
         } catch (IOException e) {
             // should not happen, we know the url is not malformed, and openConnection does not actually hit network
             throw new UncheckedIOException(e);
         }
 
-        try (InputStream in = urlConnection.getInputStream();
+        try (InputStream in = SocketAccess.doPrivilegedIOException(urlConnection::getInputStream);
              BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
 
             String metadataResult = urlReader.readLine();
diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
index d5c92a9d67b..8f0278de392 100644
--- a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
@@ -24,4 +24,7 @@ grant {
   // NOTE: no tests fail without this, but we know the problem
   // exists in AWS sdk, and tests here are not thorough
   permission java.lang.RuntimePermission "getClassLoader";
+
+  // ec2 client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
 };
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
index 3cd30c187d4..3488b5ea49c 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class CloudAwsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
index b4a1f55a3c6..4def3737563 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -96,7 +97,7 @@ public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase {
     @BeforeClass
     public static void startHttpd() throws Exception {
         logDir = createTempDir();
-        httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
 
         httpServer.createContext("/", (s) -> {
             Headers headers = s.getResponseHeaders();
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index fe6e19f966f..d05ca712f18 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -41,6 +41,7 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -195,12 +196,11 @@ public class Ec2DiscoveryTests extends ESTestCase {
         Settings nodeSettings = Settings.builder()
                 .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "does_not_exist")
                 .build();
-        try {
+
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
             buildDynamicNodes(nodeSettings, 1);
-            fail("Expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("No enum constant"));
-        }
+        });
+        assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type"));
     }
 
     public void testFilterByTags() throws InterruptedException {
@@ -259,9 +259,42 @@ public class Ec2DiscoveryTests extends ESTestCase {
         assertThat(discoveryNodes, hasSize(prodInstances));
     }
 
+    public void testReadHostFromTag() throws InterruptedException, UnknownHostException {
+        int nodes = randomIntBetween(5, 10);
+
+        String[] addresses = new String[nodes];
+
+        for (int node = 0; node < nodes; node++) {
+            addresses[node] = "192.168.0." + (node + 1);
+            poorMansDNS.put("node" + (node + 1), new TransportAddress(InetAddress.getByName(addresses[node]), 9300));
+        }
+
+        Settings nodeSettings = Settings.builder()
+            .put(DISCOVERY_EC2.HOST_TYPE_SETTING.getKey(), "tag:foo")
+            .build();
+
+        List<List<Tag>> tagsList = new ArrayList<>();
+
+        for (int node = 0; node < nodes; node++) {
+            List<Tag> tags = new ArrayList<>();
+            tags.add(new Tag("foo", "node" + (node + 1)));
+            tagsList.add(tags);
+        }
+
+        logger.info("started [{}] instances", nodes);
+        List<DiscoveryNode> discoveryNodes = buildDynamicNodes(nodeSettings, nodes, tagsList);
+        assertThat(discoveryNodes, hasSize(nodes));
+        for (DiscoveryNode discoveryNode : discoveryNodes) {
+            TransportAddress address = discoveryNode.getAddress();
+            TransportAddress expected = poorMansDNS.get(discoveryNode.getName());
+            assertEquals(address, expected);
+        }
+    }
+
+
     abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider {
         public int fetchCount = 0;
-        public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {
+        DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {
             super(settings, transportService, service);
         }
     }
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
index 7499232346d..d809fd3fa88 100644
--- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPlugin.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.zen.UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.UnicastZenPing;
@@ -34,7 +35,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchRequestParsers;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -67,12 +67,12 @@ public class FileBasedDiscoveryPlugin extends Plugin implements DiscoveryPlugin
 
     @Override
     public Collection<Object> createComponents(
-        Client client,
-        ClusterService clusterService,
-        ThreadPool threadPool,
-        ResourceWatcherService resourceWatcherService,
-        ScriptService scriptService,
-        SearchRequestParsers searchRequestParsers) {
+            Client client,
+            ClusterService clusterService,
+            ThreadPool threadPool,
+            ResourceWatcherService resourceWatcherService,
+            ScriptService scriptService,
+            NamedXContentRegistry xContentRegistry) {
         final int concurrentConnects = UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING.get(settings);
         final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(settings, "[file_based_discovery_resolve]");
         fileBasedDiscoveryExecutorService = EsExecutors.newScaling(
diff --git a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
index 55e2029c8b7..196e98d6582 100644
--- a/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
+++ b/plugins/discovery-file/src/main/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProvider.java
@@ -43,7 +43,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.discovery.zen.UnicastZenPing.DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT;
-import static org.elasticsearch.discovery.zen.UnicastZenPing.resolveDiscoveryNodes;
+import static org.elasticsearch.discovery.zen.UnicastZenPing.resolveHostsLists;
 
 /**
  * An implementation of {@link UnicastHostsProvider} that reads hosts/ports
@@ -97,13 +97,13 @@ class FileBasedUnicastHostsProvider extends AbstractComponent implements Unicast
 
         final List<DiscoveryNode> discoNodes = new ArrayList<>();
         try {
-            discoNodes.addAll(resolveDiscoveryNodes(
+            discoNodes.addAll(resolveHostsLists(
                 executorService,
                 logger,
                 hostsList,
                 1,
                 transportService,
-                () -> UNICAST_HOST_PREFIX + nodeIdGenerator.incrementAndGet() + "#",
+                UNICAST_HOST_PREFIX,
                 resolveTimeout));
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
index 8a0bd808dba..dbc6212af9f 100644
--- a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java
@@ -21,9 +21,9 @@ package org.elasticsearch.discovery.file;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -37,7 +37,7 @@ public class FileBasedDiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java
index 3dc378f6cb0..7a7ee9dbd03 100644
--- a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryPluginTests.java
@@ -23,17 +23,23 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
+
 public class FileBasedDiscoveryPluginTests extends ESTestCase {
 
     public void testHostsProviderBwc() {
         FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(Settings.EMPTY);
         Settings additionalSettings = plugin.additionalSettings();
         assertEquals("file", additionalSettings.get(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey()));
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
     }
 
     public void testHostsProviderExplicit() {
         Settings settings = Settings.builder().put(DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "foo").build();
         FileBasedDiscoveryPlugin plugin = new FileBasedDiscoveryPlugin(settings);
         assertEquals(Settings.EMPTY, plugin.additionalSettings());
+        assertWarnings("Using discovery.type setting to set hosts provider is deprecated. " +
+                "Set \"discovery.zen.hosts_provider: file\" instead");
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java
index 920792b6c7a..314e195754a 100644
--- a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java
+++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedUnicastHostsProviderTests.java
@@ -33,9 +33,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.MockTcpTransport;
 import org.elasticsearch.transport.TransportService;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -44,7 +42,6 @@ import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -99,13 +96,13 @@ public class FileBasedUnicastHostsProviderTests extends ESTestCase {
         assertEquals(hostEntries.size() - 1, nodes.size()); // minus 1 because we are ignoring the first line that's a comment
         assertEquals("192.168.0.1", nodes.get(0).getAddress().getAddress());
         assertEquals(9300, nodes.get(0).getAddress().getPort());
-        assertEquals(UNICAST_HOST_PREFIX + "1#", nodes.get(0).getId());
+        assertEquals(UNICAST_HOST_PREFIX + "192.168.0.1_0#", nodes.get(0).getId());
         assertEquals("192.168.0.2", nodes.get(1).getAddress().getAddress());
         assertEquals(9305, nodes.get(1).getAddress().getPort());
-        assertEquals(UNICAST_HOST_PREFIX + "2#", nodes.get(1).getId());
+        assertEquals(UNICAST_HOST_PREFIX + "192.168.0.2:9305_0#", nodes.get(1).getId());
         assertEquals("255.255.23.15", nodes.get(2).getAddress().getAddress());
         assertEquals(9300, nodes.get(2).getAddress().getPort());
-        assertEquals(UNICAST_HOST_PREFIX + "3#", nodes.get(2).getId());
+        assertEquals(UNICAST_HOST_PREFIX + "255.255.23.15_0#", nodes.get(2).getId());
     }
 
     public void testEmptyUnicastHostsFile() throws Exception {
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
index bda68780c79..ea5b44d994e 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceInstancesServiceImpl.java
@@ -21,10 +21,7 @@ package org.elasticsearch.cloud.gce;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.security.AccessController;
 import java.security.GeneralSecurityException;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -43,6 +40,7 @@ import com.google.api.services.compute.model.InstanceList;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -68,21 +66,14 @@ public class GceInstancesServiceImpl extends AbstractComponent implements GceIns
             try {
                 // hack around code messiness in GCE code
                 // TODO: get this fixed
-                SecurityManager sm = System.getSecurityManager();
-                if (sm != null) {
-                    sm.checkPermission(new SpecialPermission());
-                }
-                InstanceList instanceList = AccessController.doPrivileged(new PrivilegedExceptionAction<InstanceList>() {
-                    @Override
-                    public InstanceList run() throws Exception {
-                        Compute.Instances.List list = client().instances().list(project, zoneId);
-                        return list.execute();
-                    }
+                InstanceList instanceList = Access.doPrivilegedIOException(() -> {
+                    Compute.Instances.List list = client().instances().list(project, zoneId);
+                    return list.execute();
                 });
                 // assist type inference
-                return instanceList.isEmpty()  || instanceList.getItems() == null ?
+                return instanceList.isEmpty() || instanceList.getItems() == null ?
                     Collections.<Instance>emptyList() : instanceList.getItems();
-            } catch (PrivilegedActionException e) {
+            } catch (IOException e) {
                 logger.warn((Supplier<?>) () -> new ParameterizedMessage("Problem fetching instance list for zone {}", zoneId), e);
                 logger.debug("Full exception:", e);
                 // assist type inference
@@ -134,7 +125,7 @@ public class GceInstancesServiceImpl extends AbstractComponent implements GceIns
     public synchronized Compute client() {
         if (refreshInterval != null && refreshInterval.millis() != 0) {
             if (client != null &&
-                    (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) {
+                (refreshInterval.millis() < 0 || (System.currentTimeMillis() - lastRefresh) < refreshInterval.millis())) {
                 if (logger.isTraceEnabled()) logger.trace("using cache to retrieve client");
                 return client;
             }
@@ -151,22 +142,12 @@ public class GceInstancesServiceImpl extends AbstractComponent implements GceIns
             String tokenServerEncodedUrl = GceMetadataService.GCE_HOST.get(settings) +
                 "/computeMetadata/v1/instance/service-accounts/default/token";
             ComputeCredential credential = new ComputeCredential.Builder(getGceHttpTransport(), gceJsonFactory)
-                    .setTokenServerEncodedUrl(tokenServerEncodedUrl)
-                    .build();
+                .setTokenServerEncodedUrl(tokenServerEncodedUrl)
+                .build();
 
             // hack around code messiness in GCE code
             // TODO: get this fixed
-            SecurityManager sm = System.getSecurityManager();
-            if (sm != null) {
-                sm.checkPermission(new SpecialPermission());
-            }
-            AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
-                @Override
-                public Void run() throws IOException {
-                    credential.refreshToken();
-                    return null;
-                }
-            });
+            Access.doPrivilegedIOException(credential::refreshToken);
 
             logger.debug("token [{}] will expire in [{}] s", credential.getAccessToken(), credential.getExpiresInSeconds());
             if (credential.getExpiresInSeconds() != null) {
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
index 5145dc2a51c..f17126328db 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceMetadataService.java
@@ -34,6 +34,7 @@ import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.http.HttpResponse;
 import com.google.api.client.http.HttpTransport;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -56,8 +57,8 @@ public class GceMetadataService extends AbstractLifecycleComponent {
 
     protected synchronized HttpTransport getGceHttpTransport() throws GeneralSecurityException, IOException {
         if (gceHttpTransport == null) {
-                gceHttpTransport = GoogleNetHttpTransport.newTrustedTransport();
-            }
+            gceHttpTransport = GoogleNetHttpTransport.newTrustedTransport();
+        }
         return gceHttpTransport;
     }
 
@@ -71,30 +72,16 @@ public class GceMetadataService extends AbstractLifecycleComponent {
         try {
             // hack around code messiness in GCE code
             // TODO: get this fixed
-            SecurityManager sm = System.getSecurityManager();
-            if (sm != null) {
-                sm.checkPermission(new SpecialPermission());
-            }
-            headers = AccessController.doPrivileged(new PrivilegedExceptionAction<HttpHeaders>() {
-                @Override
-                public HttpHeaders run() throws IOException {
-                    return new HttpHeaders();
-                }
-            });
-            GenericUrl genericUrl = AccessController.doPrivileged(new PrivilegedAction<GenericUrl>() {
-                @Override
-                public GenericUrl run() {
-                    return new GenericUrl(urlMetadataNetwork);
-                }
-            });
+            headers = Access.doPrivileged(HttpHeaders::new);
+            GenericUrl genericUrl = Access.doPrivileged(() -> new GenericUrl(urlMetadataNetwork));
 
             // This is needed to query meta data: https://cloud.google.com/compute/docs/metadata
             headers.put("Metadata-Flavor", "Google");
-            HttpResponse response;
-            response = getGceHttpTransport().createRequestFactory()
-                .buildGetRequest(genericUrl)
-                .setHeaders(headers)
-                .execute();
+            HttpResponse response = Access.doPrivilegedIOException(() ->
+                getGceHttpTransport().createRequestFactory()
+                    .buildGetRequest(genericUrl)
+                    .setHeaders(headers)
+                    .execute());
             String metadata = response.parseAsString();
             logger.debug("metadata found [{}]", metadata);
             return metadata;
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
index 54f96a971ca..10d3fd0e772 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cloud.gce.network;
 
 import org.elasticsearch.cloud.gce.GceMetadataService;
+import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
@@ -28,6 +29,9 @@ import org.elasticsearch.common.settings.Settings;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 
 /**
  * <p>Resolves certain GCE related 'meta' hostnames into an actual hostname
@@ -106,13 +110,13 @@ public class GceNameResolver extends AbstractComponent implements CustomNameReso
         }
 
         try {
-            String metadataResult = gceMetadataService.metadata(gceMetadataPath);
+            String metadataResult = Access.doPrivilegedIOException(() -> gceMetadataService.metadata(gceMetadataPath));
             if (metadataResult == null || metadataResult.length() == 0) {
                 throw new IOException("no gce metadata returned from [" + gceMetadataPath + "] for [" + value + "]");
             }
             // only one address: because we explicitly ask for only one via the GceHostnameType
             return new InetAddress[] { InetAddress.getByName(metadataResult) };
-        } catch (IOException | URISyntaxException e) {
+        } catch (IOException e) {
             throw new IOException("IOException caught when fetching InetAddress from [" + gceMetadataPath + "]", e);
         }
     }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java
new file mode 100644
index 00000000000..ae96dee64c4
--- /dev/null
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/util/Access.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.gce.util;
+
+import org.elasticsearch.SpecialPermission;
+
+import java.io.IOException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * GCE's http client changes access levels. Specifically it needs {@link RuntimePermission} accessDeclaredMembers and
+ * setFactory and {@link java.lang.reflect.ReflectPermission} suppressAccessChecks. For remote calls the plugin needs
+ * SocketPermissions for 'connect'. This class wraps the operations requiring access in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks.
+ */
+public final class Access {
+
+    private Access() {}
+
+    public static <T> T doPrivileged(PrivilegedAction<T> operation) {
+        SpecialPermission.check();
+        return AccessController.doPrivileged(operation);
+    }
+
+    public static void doPrivilegedVoid(Runnable action) {
+        SpecialPermission.check();
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            action.run();
+            return null;
+        });
+    }
+
+    public static <T> T doPrivilegedIOException(PrivilegedExceptionAction<T> operation) throws IOException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+}
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java
index c005aa05a78..68cceab76d3 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapper.java
@@ -31,6 +31,7 @@ import com.google.api.client.util.ExponentialBackOff;
 import com.google.api.client.util.Sleeper;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -73,11 +74,7 @@ public class RetryHttpInitializerWrapper implements HttpRequestInitializer {
     // Use only for testing
     static MockGoogleCredential.Builder newMockCredentialBuilder() {
         // TODO: figure out why GCE is so bad like this
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        return AccessController.doPrivileged((PrivilegedAction<MockGoogleCredential.Builder>) () -> new MockGoogleCredential.Builder());
+        return Access.doPrivileged(MockGoogleCredential.Builder::new);
     }
 
     @Override
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index f53abc4241c..acad7e099f5 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -30,9 +30,11 @@ import org.elasticsearch.cloud.gce.GceInstancesServiceImpl;
 import org.elasticsearch.cloud.gce.GceMetadataService;
 import org.elasticsearch.cloud.gce.GceModule;
 import org.elasticsearch.cloud.gce.network.GceNameResolver;
+import org.elasticsearch.cloud.gce.util.Access;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
@@ -79,17 +81,7 @@ public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
          * our plugin permissions don't allow core to "reach through" plugins to
          * change the permission. Because that'd be silly.
          */
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                ClassInfo.of(HttpHeaders.class, true);
-                return null;
-            }
-        });
+        Access.doPrivilegedVoid( () -> ClassInfo.of(HttpHeaders.class, true));
     }
 
     public GceDiscoveryPlugin(Settings settings) {
@@ -99,10 +91,11 @@ public class GceDiscoveryPlugin extends Plugin implements DiscoveryPlugin, Close
 
     @Override
     public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                              NamedWriteableRegistry namedWriteableRegistry,
                                                               ClusterService clusterService, UnicastHostsProvider hostsProvider) {
         // this is for backcompat with pre 5.1, where users would set discovery.type to use ec2 hosts provider
         return Collections.singletonMap(GCE, () ->
-            new ZenDiscovery(settings, threadPool, transportService, clusterService, hostsProvider));
+            new ZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
     }
 
     @Override
diff --git a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy
index 1463f5d1119..f47803cebb0 100644
--- a/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/discovery-gce/src/main/plugin-metadata/plugin-security.policy
@@ -22,4 +22,7 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "setFactory";
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+
+  // gce client opens socket connections for discovery
+  permission java.net.SocketPermission "*", "connect";
 };
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
index 3af39b6da5d..67ef297609f 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class DiscoveryGceClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index 76d7c6408d5..fc0234671f0 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.mocksocket.MockHttpServer;
 import org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -108,8 +109,8 @@ public class GceDiscoverTests extends ESIntegTestCase {
     public static void startHttpd() throws Exception {
         logDir = createTempDir();
         SSLContext sslContext = getSSLContext();
-        httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
-        httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+        httpsServer = MockHttpServer.createHttps(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
         httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
         httpServer.createContext("/computeMetadata/v1/instance/service-accounts/default/token", (s) -> {
             String response = GceMockUtils.readGoogleInternalJsonResponse(
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index c80a8a1db59..97b5a23f116 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -26,7 +26,8 @@ versions << [
   'tika': '1.14',
   'pdfbox': '2.0.3',
   'bouncycastle': '1.55',
-  'poi': '3.15'
+  'poi': '3.15',
+  'mime4j': '0.7.2'
 ]
 
 dependencies {
@@ -59,11 +60,19 @@ dependencies {
   compile "org.apache.poi:poi-scratchpad:${versions.poi}"
   // Apple iWork
   compile 'org.apache.commons:commons-compress:1.10'
+  // Outlook documents
+  compile "org.apache.james:apache-mime4j-core:${versions.mime4j}"
+  compile "org.apache.james:apache-mime4j-dom:${versions.mime4j}"
 }
 
 // TODO: stop using LanguageIdentifier...
 compileJava.options.compilerArgs << "-Xlint:-deprecation"
 
+
+dependencyLicenses {
+  mapping from: /apache-mime4j-.*/, to: 'apache-mime4j'
+}
+
 forbiddenPatterns {
   exclude '**/*.docx'
   exclude '**/*.pdf'
@@ -530,25 +539,6 @@ thirdPartyAudit.excludes = [
   'org.apache.http.client.utils.URIBuilder',
   'org.apache.http.entity.ByteArrayEntity',
   'org.apache.http.impl.client.DefaultHttpClient',
-  'org.apache.james.mime4j.MimeException',
-  'org.apache.james.mime4j.codec.DecodeMonitor',
-  'org.apache.james.mime4j.codec.DecoderUtil',
-  'org.apache.james.mime4j.dom.FieldParser',
-  'org.apache.james.mime4j.dom.address.Address',
-  'org.apache.james.mime4j.dom.address.AddressList',
-  'org.apache.james.mime4j.dom.address.Mailbox',
-  'org.apache.james.mime4j.dom.address.MailboxList',
-  'org.apache.james.mime4j.dom.field.AddressListField',
-  'org.apache.james.mime4j.dom.field.DateTimeField',
-  'org.apache.james.mime4j.dom.field.MailboxListField',
-  'org.apache.james.mime4j.dom.field.ParsedField',
-  'org.apache.james.mime4j.dom.field.UnstructuredField',
-  'org.apache.james.mime4j.field.LenientFieldParser',
-  'org.apache.james.mime4j.parser.ContentHandler',
-  'org.apache.james.mime4j.parser.MimeStreamParser',
-  'org.apache.james.mime4j.stream.BodyDescriptor',
-  'org.apache.james.mime4j.stream.Field',
-  'org.apache.james.mime4j.stream.MimeConfig',
   'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod',
   'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo',
   'org.apache.jcp.xml.dsig.internal.dom.DOMReference',
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt b/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt
new file mode 100644
index 00000000000..63151ee1458
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-LICENSE.txt
@@ -0,0 +1,361 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+
+
+
+   THIS PRODUCT ALSO INCLUDES THIRD PARTY SOFTWARE REDISTRIBUTED UNDER THE
+   FOLLOWING LICENSES:
+
+	Apache Commons Logging,
+	   The Apache Software License, Version 1.1 (commons-logging-1.1.1.jar)
+
+		 The Apache Software License, Version 1.1
+
+		 Redistribution and use in source and binary forms, with or without
+		 modification, are permitted provided that the following conditions
+		 are met:
+
+		 1. Redistributions of source code must retain the above copyright
+		    notice, this list of conditions and the following disclaimer.
+
+		 2. Redistributions in binary form must reproduce the above copyright
+		    notice, this list of conditions and the following disclaimer in
+		    the documentation and/or other materials provided with the
+		    distribution.
+
+		 3. The end-user documentation included with the redistribution,
+		    if any, must include the following acknowledgment:
+		       "This product includes software developed by the
+		        Apache Software Foundation (http://www.apache.org/)."
+		    Alternately, this acknowledgment may appear in the software itself,
+		    if and wherever such third-party acknowledgments normally appear.
+
+		 4. The names "Apache" and "Apache Software Foundation" must
+		    not be used to endorse or promote products derived from this
+		    software without prior written permission. For written
+		    permission, please contact apache@apache.org.
+
+		 5. Products derived from this software may not be called "Apache",
+		    nor may "Apache" appear in their name, without prior written
+		    permission of the Apache Software Foundation.
+
+		 THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+		 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+		 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+		 DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+		 ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+		 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+		 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+		 USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+		 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+		 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+		 OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+		 SUCH DAMAGE.
+
+
+	Test messages from the Perl-MIME-Tools project,
+
+					 The "Artistic License"
+
+						Preamble
+
+		The intent of this document is to state the conditions under which a
+		Package may be copied, such that the Copyright Holder maintains some
+		semblance of artistic control over the development of the package,
+		while giving the users of the package the right to use and distribute
+		the Package in a more-or-less customary fashion, plus the right to make
+		reasonable modifications.
+
+		Definitions:
+
+			"Package" refers to the collection of files distributed by the
+			Copyright Holder, and derivatives of that collection of files
+			created through textual modification.
+
+			"Standard Version" refers to such a Package if it has not been
+			modified, or has been modified in accordance with the wishes
+			of the Copyright Holder as specified below.
+
+			"Copyright Holder" is whoever is named in the copyright or
+			copyrights for the package.
+
+			"You" is you, if you're thinking about copying or distributing
+			this Package.
+
+			"Reasonable copying fee" is whatever you can justify on the
+			basis of media cost, duplication charges, time of people involved,
+			and so on.  (You will not be required to justify it to the
+			Copyright Holder, but only to the computing community at large
+			as a market that must bear the fee.)
+
+			"Freely Available" means that no fee is charged for the item
+			itself, though there may be fees involved in handling the item.
+			It also means that recipients of the item may redistribute it
+			under the same conditions they received it.
+
+		1. You may make and give away verbatim copies of the source form of the
+		Standard Version of this Package without restriction, provided that you
+		duplicate all of the original copyright notices and associated disclaimers.
+
+		2. You may apply bug fixes, portability fixes and other modifications
+		derived from the Public Domain or from the Copyright Holder.  A Package
+		modified in such a way shall still be considered the Standard Version.
+
+		3. You may otherwise modify your copy of this Package in any way, provided
+		that you insert a prominent notice in each changed file stating how and
+		when you changed that file, and provided that you do at least ONE of the
+		following:
+
+		    a) place your modifications in the Public Domain or otherwise make them
+		    Freely Available, such as by posting said modifications to Usenet or
+		    an equivalent medium, or placing the modifications on a major archive
+		    site such as uunet.uu.net, or by allowing the Copyright Holder to include
+		    your modifications in the Standard Version of the Package.
+
+		    b) use the modified Package only within your corporation or organization.
+
+		    c) rename any non-standard executables so the names do not conflict
+		    with standard executables, which must also be provided, and provide
+		    a separate manual page for each non-standard executable that clearly
+		    documents how it differs from the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		4. You may distribute the programs of this Package in object code or
+		executable form, provided that you do at least ONE of the following:
+
+		    a) distribute a Standard Version of the executables and library files,
+		    together with instructions (in the manual page or equivalent) on where
+		    to get the Standard Version.
+
+		    b) accompany the distribution with the machine-readable source of
+		    the Package with your modifications.
+
+		    c) give non-standard executables non-standard names, and clearly
+		    document the differences in manual pages (or equivalent), together
+		    with instructions on where to get the Standard Version.
+
+		    d) make other distribution arrangements with the Copyright Holder.
+
+		5. You may charge a reasonable copying fee for any distribution of this
+		Package.  You may charge any fee you choose for support of this
+		Package.  You may not charge a fee for this Package itself.  However,
+		you may distribute this Package in aggregate with other (possibly
+		commercial) programs as part of a larger (possibly commercial) software
+		distribution provided that you do not advertise this Package as a
+		product of your own.  You may embed this Package's interpreter within
+		an executable of yours (by linking); this shall be construed as a mere
+		form of aggregation, provided that the complete Standard Version of the
+		interpreter is so embedded.
+
+		6. The scripts and library files supplied as input to or produced as
+		output from the programs of this Package do not automatically fall
+		under the copyright of this Package, but belong to whoever generated
+		them, and may be sold commercially, and may be aggregated with this
+		Package.  If such scripts or library files are aggregated with this
+		Package via the so-called "undump" or "unexec" methods of producing a
+		binary executable image, then distribution of such an image shall
+		neither be construed as a distribution of this Package nor shall it
+		fall under the restrictions of Paragraphs 3 and 4, provided that you do
+		not represent such an executable image as a Standard Version of this
+		Package.
+
+		7. C subroutines (or comparably compiled subroutines in other
+		languages) supplied by you and linked into this Package in order to
+		emulate subroutines and variables of the language defined by this
+		Package shall not be considered part of this Package, but are the
+		equivalent of input as in Paragraph 6, provided these subroutines do
+		not change the language in any way that would cause it to fail the
+		regression tests for the language.
+
+		8. Aggregation of this Package with a commercial distribution is always
+		permitted provided that the use of this Package is embedded; that is,
+		when no overt attempt is made to make this Package's interfaces visible
+		to the end user of the commercial distribution.  Such use shall not be
+		construed as a distribution of this Package.
+
+		9. The name of the Copyright Holder may not be used to endorse or promote
+		products derived from this software without specific prior written permission.
+
+		10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+		IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+		WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+
+						The End
+
+
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt b/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt
new file mode 100644
index 00000000000..61523975eb2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-NOTICE.txt
@@ -0,0 +1,13 @@
+   =========================================================================
+   ==      NOTICE file for use with the Apache License, Version 2.0,      ==
+   =========================================================================
+   
+   Apache JAMES Mime4j
+   Copyright 2004-2010 The Apache Software Foundation
+   
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+
+   This product test suite includes data (mimetools-testmsgs folder) developed 
+   by Eryq and ZeeGee Software Inc as part of the "MIME-tools" Perl5 toolkit
+   and licensed under the Artistic License
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
new file mode 100644
index 00000000000..8210fc7fc16
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.7.2.jar.sha1
@@ -0,0 +1 @@
+a81264fe0265ebe8fd1d8128aad06dc320de6eef
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
new file mode 100644
index 00000000000..3ede85a0191
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.7.2.jar.sha1
@@ -0,0 +1 @@
+1c289aa264548a0a1f1b43685a9cb2ab23f67287
\ No newline at end of file
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 0ba79ecc9fb..f7f474711be 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -38,6 +38,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readIntProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
@@ -52,23 +53,36 @@ public final class AttachmentProcessor extends AbstractProcessor {
     private final String targetField;
     private final Set<Property> properties;
     private final int indexedChars;
+    private final boolean ignoreMissing;
 
     AttachmentProcessor(String tag, String field, String targetField, Set<Property> properties,
-                        int indexedChars) throws IOException {
+                        int indexedChars, boolean ignoreMissing) throws IOException {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.properties = properties;
         this.indexedChars = indexedChars;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) {
         Map<String, Object> additionalFields = new HashMap<>();
 
+        byte[] input = ingestDocument.getFieldValueAsBytes(field, ignoreMissing);
+
+        if (input == null && ignoreMissing) {
+            return;
+        } else if (input == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot parse.");
+        }
+
         try {
             Metadata metadata = new Metadata();
-            byte[] input = ingestDocument.getFieldValueAsBytes(field);
             String parsedContent = TikaImpl.parse(input, metadata, indexedChars);
 
             if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
@@ -166,6 +180,7 @@ public final class AttachmentProcessor extends AbstractProcessor {
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
             List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
             int indexedChars = readIntProperty(TYPE, processorTag, config, "indexed_chars", NUMBER_OF_CHARS_INDEXED);
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             final Set<Property> properties;
             if (properyNames != null) {
@@ -182,7 +197,7 @@ public final class AttachmentProcessor extends AbstractProcessor {
                 properties = DEFAULT_PROPERTIES;
             }
 
-            return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars);
+            return new AttachmentProcessor(processorTag, field, targetField, properties, indexedChars, ignoreMissing);
         }
     }
 
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
index 27a4cfebbcd..f99a2a630ab 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java
@@ -82,18 +82,11 @@ final class TikaImpl {
     // only package private for testing!
     static String parse(final byte content[], final Metadata metadata, final int limit) throws TikaException, IOException {
         // check that its not unprivileged code like a script
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
 
         try {
-            return AccessController.doPrivileged(new PrivilegedExceptionAction<String>() {
-                @Override
-                public String run() throws TikaException, IOException {
-                    return TIKA_INSTANCE.parseToString(new ByteArrayInputStream(content), metadata, limit);
-                }
-            }, RESTRICTED_CONTEXT);
+            return AccessController.doPrivileged((PrivilegedExceptionAction<String>)
+                () -> TIKA_INSTANCE.parseToString(new ByteArrayInputStream(content), metadata, limit), RESTRICTED_CONTEXT);
         } catch (PrivilegedActionException e) {
             // checked exception from tika: unbox it
             Throwable cause = e.getCause();
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 2848d6c3c64..376214eb173 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -52,6 +52,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
         assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testConfigureIndexedChars() throws Exception {
@@ -64,6 +65,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         AttachmentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getIndexedChars(), is(indexedChars));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
@@ -73,6 +75,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         AttachmentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildFields() throws Exception {
@@ -90,6 +93,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         AttachmentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildIllegalFieldOption() throws Exception {
@@ -117,4 +121,19 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
         }
     }
+
+    public void testIgnoreMissing() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+
+        String processorTag = randomAsciiOfLength(10);
+
+        AttachmentProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("attachment"));
+        assertThat(processor.getProperties(), sameInstance(AttachmentProcessor.Factory.DEFAULT_PROPERTIES));
+        assertTrue(processor.isIgnoreMissing());
+    }
 }
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
index 3708a290dec..b59457b5b01 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.ingest.attachment;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -30,14 +31,17 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -52,7 +56,7 @@ public class AttachmentProcessorTests extends ESTestCase {
     @Before
     public void createStandardProcessor() throws IOException {
         processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
-            "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000);
+            "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 10000, false);
     }
 
     public void testEnglishTextDocument() throws Exception {
@@ -85,7 +89,7 @@ public class AttachmentProcessorTests extends ESTestCase {
             selectedProperties.add(AttachmentProcessor.Property.DATE);
         }
         processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field",
-            "target_field", selectedProperties, 10000);
+            "target_field", selectedProperties, 10000, false);
 
         Map<String, Object> attachmentData = parseDocument("htmlWithEmptyDateMeta.html", processor);
         assertThat(attachmentData.keySet(), hasSize(selectedFieldNames.length));
@@ -199,6 +203,40 @@ public class AttachmentProcessorTests extends ESTestCase {
         assertThat(attachmentData.get("content_length"), is(notNullValue()));
     }
 
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", "randomTarget", null, 10, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", "randomTarget", null, 10, true);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", "randomTarget", null, 10, false);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Processor processor = new AttachmentProcessor(randomAsciiOfLength(10), "source_field", "randomTarget", null, 10, false);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
     private Map<String, Object> parseDocument(String file, AttachmentProcessor processor) throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", getAsBase64(file));
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
index 40e95451e49..43e9cd394f0 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestAttachmentClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
index 4b9a40dd8a9..b4445d16b77 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.ingest.attachment;
 import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.DirectoryStream;
@@ -37,11 +38,17 @@ import java.nio.file.Path;
 public class TikaDocTests extends ESTestCase {
 
     /** some test files from tika test suite, zipped up */
-    static final String TIKA_FILES = "/org/elasticsearch/ingest/attachment/test/tika-files.zip";
+    static final String TIKA_FILES = "/org/elasticsearch/ingest/attachment/test/tika-files/";
 
     public void testFiles() throws Exception {
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES), tmp);
+        logger.debug("unzipping all tika sample files");
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(PathUtils.get(getClass().getResource(TIKA_FILES).toURI()))) {
+            for (Path doc : stream) {
+                String filename = doc.getFileName().toString();
+                TestUtil.unzip(getClass().getResourceAsStream(TIKA_FILES + filename), tmp);
+            }
+        }
 
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmp)) {
             for (Path doc : stream) {
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip
deleted file mode 100644
index 10f5d507677..00000000000
Binary files a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files.zip and /dev/null differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip
new file mode 100644
index 00000000000..f9bd7c46b2b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/Doc1_ole.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip
new file mode 100644
index 00000000000..bb4e8e1ef3e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedDocument.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip
new file mode 100644
index 00000000000..67de787f017
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedOutlook.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip
new file mode 100644
index 00000000000..7e02f55c03d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/EmbeddedPDF.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip
new file mode 100644
index 00000000000..0516c35b071
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/NullHeader.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip
new file mode 100644
index 00000000000..fc851923abb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/big-preamble.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip
new file mode 100644
index 00000000000..2d1f8e601ce
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate-whitespace.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip
new file mode 100644
index 00000000000..c728a6303b4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/boilerplate.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip
new file mode 100644
index 00000000000..21376b69179
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/footnotes.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip
new file mode 100644
index 00000000000..6bb9b3f463d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headerPic.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip
new file mode 100644
index 00000000000..f36e6b8de47
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/headers.mbox.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip
new file mode 100644
index 00000000000..55dde5eada7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/pictures.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip
new file mode 100644
index 00000000000..9823a8debf8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protect.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip
new file mode 100644
index 00000000000..b4de09b0fdc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedFile.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip
new file mode 100644
index 00000000000..8dde8a493a2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/protectedSheets.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip
new file mode 100644
index 00000000000..0c53eae3b3a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/resume.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip
new file mode 100644
index 00000000000..e9a0bfbed00
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook.msg.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip
new file mode 100644
index 00000000000..bce3da6e05b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test-outlook2003.msg.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip
new file mode 100644
index 00000000000..4bcf68dcebc
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip
new file mode 100644
index 00000000000..3268c952a33
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testAnnotations.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip
new file mode 100644
index 00000000000..4a7efa19f0a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBinControlWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip
new file mode 100644
index 00000000000..41b896525ca
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testBulletPoints.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip
new file mode 100644
index 00000000000..15a056cb3a3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip
new file mode 100644
index 00000000000..30b4f6877a8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip
new file mode 100644
index 00000000000..bb21a9d7103
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip
new file mode 100644
index 00000000000..e0122d4004b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip
new file mode 100644
index 00000000000..77967db80c6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip
new file mode 100644
index 00000000000..b7b6bdf5c13
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip
new file mode 100644
index 00000000000..7250235b89c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip
new file mode 100644
index 00000000000..7fe71de731d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testComment.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip
new file mode 100644
index 00000000000..e0fd82103fe
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testDOCX_Thumbnail.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip
new file mode 100644
index 00000000000..fd5d8472d49
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-charts.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip
new file mode 100644
index 00000000000..f1015248b63
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip
new file mode 100644
index 00000000000..b62c8813e96
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL-formats.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip
new file mode 100644
index 00000000000..f0dc3b00f96
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip
new file mode 100644
index 00000000000..45b3dae6f44
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip
new file mode 100644
index 00000000000..09cb2161697
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip
new file mode 100644
index 00000000000..81228661820
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_1img.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip
new file mode 100644
index 00000000000..46da553af95
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_4.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip
new file mode 100644
index 00000000000..ca3e08c4701
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_5.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip
new file mode 100644
index 00000000000..00e3d373d11
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_95.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip
new file mode 100644
index 00000000000..2fa4d6b8b91
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip
new file mode 100644
index 00000000000..9cfd16cec25
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_custom_props.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip
new file mode 100644
index 00000000000..b7291cafc98
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip
new file mode 100644
index 00000000000..755b7395e83
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_embeded.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip
new file mode 100644
index 00000000000..6cb18625ca7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xls.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip
new file mode 100644
index 00000000000..924ad5eece2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_headers_footers.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip
new file mode 100644
index 00000000000..43046dff526
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testEXCEL_textbox.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip
new file mode 100644
index 00000000000..8f6ed7a19f2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testExtraSpaces.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip
new file mode 100644
index 00000000000..77a5ac60dc4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFontAfterBufferedText.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip
new file mode 100644
index 00000000000..8659af1e691
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.ods.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip
new file mode 100644
index 00000000000..85ba5501448
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testFooter.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip
new file mode 100644
index 00000000000..58cfd638254
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip
new file mode 100644
index 00000000000..a12735a5205
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_1.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip
new file mode 100644
index 00000000000..89a3f9d7aaa
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_2.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip
new file mode 100644
index 00000000000..da23b13ee44
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_3.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip
new file mode 100644
index 00000000000..15850474d98
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTMLNoisyMetaEncoding_4.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip
new file mode 100644
index 00000000000..bdb5e58f373
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testHTML_utf8.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip
new file mode 100644
index 00000000000..68693369d10
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testKeynote.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip
new file mode 100644
index 00000000000..42467aa951b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterFooter.odp.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip
new file mode 100644
index 00000000000..8d7803d396c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testMasterSlideTable.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip
new file mode 100644
index 00000000000..4c41bc7a466
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testNPEOpenDocument.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip
new file mode 100644
index 00000000000..9202fe44e44
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testODFwithOOo3.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip
new file mode 100644
index 00000000000..1993365b5d1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOpenOffice2.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip
new file mode 100644
index 00000000000..b9f515941e3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip
new file mode 100644
index 00000000000..033871556d1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip
new file mode 100644
index 00000000000..37fab939058
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip
new file mode 100644
index 00000000000..e50261f4324
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip
new file mode 100644
index 00000000000..19d6e78e6df
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip
new file mode 100644
index 00000000000..d86048798c2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOptionalHyphen.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip
new file mode 100644
index 00000000000..c7f8b787048
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testOverlappingText.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip
new file mode 100644
index 00000000000..44865eca76c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF-custommetadata.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip
new file mode 100644
index 00000000000..fd5195b0538
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip
new file mode 100644
index 00000000000..a7ceed20665
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFEmbeddingAndEmbedded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip
new file mode 100644
index 00000000000..4b6e3338950
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFFileEmbInAnnotation.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip
new file mode 100644
index 00000000000..8bbb9166bb3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFPackage.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip
new file mode 100644
index 00000000000..2d1ebdbdb2f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTripleLangTitle.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip
new file mode 100644
index 00000000000..d05907a3da1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFTwoTextBoxes.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip
new file mode 100644
index 00000000000..cf86cbae79e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDFVarious.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip
new file mode 100644
index 00000000000..e8377846e79
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_PDFEncodedStringInXMP.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip
new file mode 100644
index 00000000000..4fffed77ad8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.10.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip
new file mode 100644
index 00000000000..16098515ce2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.11.x.PDFA-1b.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip
new file mode 100644
index 00000000000..cdf93347572
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.4.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip
new file mode 100644
index 00000000000..e5ef413b8be
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.5.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip
new file mode 100644
index 00000000000..b57b4dc4efb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.6.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip
new file mode 100644
index 00000000000..d69bbb50fee
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.7.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip
new file mode 100644
index 00000000000..e077af20743
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.8.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip
new file mode 100644
index 00000000000..c9b6857c152
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_Version.9.x.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip
new file mode 100644
index 00000000000..267d41df314
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_acroform3.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip
new file mode 100644
index 00000000000..c4ea31f96f4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bom.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip
new file mode 100644
index 00000000000..88a5db5e63e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_bookmarks.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip
new file mode 100644
index 00000000000..e61f2bf1112
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_multiFormatEmbFiles.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip
new file mode 100644
index 00000000000..16920b49e3a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_no_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip
new file mode 100644
index 00000000000..59887f7e5d2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_no_extract_yes_accessibility_owner_empty.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip
new file mode 100644
index 00000000000..d240c51d822
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPDF_twoAuthors.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.potm.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.potm.zip
new file mode 100644
index 00000000000..0f87b774a2a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.potm.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip
new file mode 100644
index 00000000000..47f652d14af
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsm.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip
new file mode 100644
index 00000000000..0f21f13be0f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip
new file mode 100644
index 00000000000..00c5c88c6ee
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip
new file mode 100644
index 00000000000..edd3e444fd7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptm.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip
new file mode 100644
index 00000000000..68a9d662756
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip
new file mode 100644
index 00000000000..23dba29f324
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPTX_Thumbnail.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip
new file mode 100644
index 00000000000..317e1bbe352
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip
new file mode 100644
index 00000000000..62096338a49
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_2imgs.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip
new file mode 100644
index 00000000000..5bd71fbd2e4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip
new file mode 100644
index 00000000000..811ab85cc26
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_autodate.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip
new file mode 100644
index 00000000000..3e94ba4cce1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip
new file mode 100644
index 00000000000..68d2135d412
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_comment.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip
new file mode 100644
index 00000000000..80673e625d7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip
new file mode 100644
index 00000000000..d27c98d2c2d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_custom_props.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip
new file mode 100644
index 00000000000..017f9f500c7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded2.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip
new file mode 100644
index 00000000000..dfa459cccbd
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embedded_two_slides.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip
new file mode 100644
index 00000000000..fb8a3caf755
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip
new file mode 100644
index 00000000000..f33f463c335
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_embeded.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip
new file mode 100644
index 00000000000..6a727889394
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip
new file mode 100644
index 00000000000..992dad41c53
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterFooter.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip
new file mode 100644
index 00000000000..e31c4719f47
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip
new file mode 100644
index 00000000000..1fa4f487487
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip
new file mode 100644
index 00000000000..f58055ecfcb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip
new file mode 100644
index 00000000000..f8ff31052d1
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_masterText2.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip
new file mode 100644
index 00000000000..5d13efb8d4a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.ppt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip
new file mode 100644
index 00000000000..5d146a1de0b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPPT_various.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip
new file mode 100644
index 00000000000..4f9db82c299
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPageNumber.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip
new file mode 100644
index 00000000000..3db77c08733
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPages.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip
new file mode 100644
index 00000000000..55004e812d2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesComments.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip
new file mode 100644
index 00000000000..bfea0c7825a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaLower.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip
new file mode 100644
index 00000000000..3298d7eaa8f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersAlphaUpper.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip
new file mode 100644
index 00000000000..34a4fb83230
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersFootnotes.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip
new file mode 100644
index 00000000000..b48ebe4458d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanLower.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip
new file mode 100644
index 00000000000..967e139595f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesHeadersFootersRomanUpper.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip
new file mode 100644
index 00000000000..c6f0e04a536
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPagesLayout.pages.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip
new file mode 100644
index 00000000000..4eb78653562
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testPopupAnnotation.pdf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip
new file mode 100644
index 00000000000..bd80f2b061a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF-ms932.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip
new file mode 100644
index 00000000000..72ef8ebfc56
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTF.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip
new file mode 100644
index 00000000000..935c249176b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFBoldItalic.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip
new file mode 100644
index 00000000000..0c55479cc89
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFControls.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip
new file mode 100644
index 00000000000..e6ea82d4760
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFCorruptListOverride.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip
new file mode 100644
index 00000000000..857c589d9eb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFEmbeddedLink.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip
new file mode 100644
index 00000000000..f10b768d5a2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHexEscapeInsideWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip
new file mode 100644
index 00000000000..658eb87784f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFHyperlink.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip
new file mode 100644
index 00000000000..15a74d726b6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFIgnoredControlWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip
new file mode 100644
index 00000000000..f24a5ab9c08
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFInvalidUnicode.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip
new file mode 100644
index 00000000000..8bf4f6e7165
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFJapanese.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip
new file mode 100644
index 00000000000..e2a406d99a6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListLibreOffice.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip
new file mode 100644
index 00000000000..155718ac8b3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListMicrosoftWord.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip
new file mode 100644
index 00000000000..c9b7f3fdbee
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFListOverride.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip
new file mode 100644
index 00000000000..8b0dffe1351
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFNewlines.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip
new file mode 100644
index 00000000000..2bf8d9b5301
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFRegularImages.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip
new file mode 100644
index 00000000000..5e110de5169
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip
new file mode 100644
index 00000000000..599e7f22edb
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFTableCellSeparation2.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip
new file mode 100644
index 00000000000..f3cbec9e51d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip
new file mode 100644
index 00000000000..2c905405ca4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUmlautSpaces2.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip
new file mode 100644
index 00000000000..40263351a04
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeGothic.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip
new file mode 100644
index 00000000000..c8f40b549c3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFUnicodeUCNControlWordCharacterDoubling.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip
new file mode 100644
index 00000000000..60281e7f9ae
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFVarious.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip
new file mode 100644
index 00000000000..4a7852c97d3
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWindowsCodepage1250.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip
new file mode 100644
index 00000000000..48268a457c0
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWithCurlyBraces.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip
new file mode 100644
index 00000000000..be845e4f925
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWord2010CzechCharacters.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip
new file mode 100644
index 00000000000..508801675de
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testRTFWordPadCzechCharacters.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip
new file mode 100644
index 00000000000..22f6c76b9a8
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testStyles.odt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip
new file mode 100644
index 00000000000..7326a28ad76
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTXTNonASCIIUTF8.txt.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip
new file mode 100644
index 00000000000..3c963dcc077
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTables.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip
new file mode 100644
index 00000000000..aea9b1cf66c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testTextBoxes.key.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip
new file mode 100644
index 00000000000..38b2954a4ff
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip
new file mode 100644
index 00000000000..2ed0aae33f4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip
new file mode 100644
index 00000000000..b8dbbd8322e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD6.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip
new file mode 100644
index 00000000000..1baa330e5ce
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip
new file mode 100644
index 00000000000..8920521bd2d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_1img.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip
new file mode 100644
index 00000000000..b60716b93d2
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip
new file mode 100644
index 00000000000..062713d9318
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_3imgs.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip
new file mode 100644
index 00000000000..b0a6af23d34
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip
new file mode 100644
index 00000000000..546f98bf093
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip
new file mode 100644
index 00000000000..5dbedec5e86
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip
new file mode 100644
index 00000000000..8edde24f993
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_bold_character_runs2.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip
new file mode 100644
index 00000000000..4484a6b0052
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_closingSmartQInHyperLink.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip
new file mode 100644
index 00000000000..72a94832d3e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip
new file mode 100644
index 00000000000..30aaef7d817
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_custom_props.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip
new file mode 100644
index 00000000000..2b74f67dc9f
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_pdf.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip
new file mode 100644
index 00000000000..e456721065c
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embedded_rtf.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip
new file mode 100644
index 00000000000..699a13b652a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip
new file mode 100644
index 00000000000..b53088ddb71
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_embeded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip
new file mode 100644
index 00000000000..867dc1808f6
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_header_hyperlink.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip
new file mode 100644
index 00000000000..04172a65382
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_ooxml_bean1.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip
new file mode 100644
index 00000000000..325ecce9279
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_missing_text.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip
new file mode 100644
index 00000000000..38eecb941d4
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip
new file mode 100644
index 00000000000..1c19cf3e982
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_no_format.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip
new file mode 100644
index 00000000000..605eeb25023
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_null_style.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip
new file mode 100644
index 00000000000..7f8d3b9ad20
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip
new file mode 100644
index 00000000000..42b4ed17c4d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_numbered_list.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip
new file mode 100644
index 00000000000..39c954b2d19
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip
new file mode 100644
index 00000000000..77bb9dbda68
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_override_list_numbering.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip
new file mode 100644
index 00000000000..0c63f982567
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_tabular_symbol.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip
new file mode 100644
index 00000000000..504c56728c9
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_text_box.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip
new file mode 100644
index 00000000000..9d7c24e0520
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip
new file mode 100644
index 00000000000..393aca11ab7
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWORD_various.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip
new file mode 100644
index 00000000000..75234fde98d
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testWordArt.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip
new file mode 100644
index 00000000000..45c4dd0479a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXHTML.html.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip
new file mode 100644
index 00000000000..919cf31e75b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXLSX_Thumbnail.xlsx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip
new file mode 100644
index 00000000000..a6520a8a811
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip
new file mode 100644
index 00000000000..5b89863685a
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML2.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip
new file mode 100644
index 00000000000..08a066d9c1e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/testXML3.xml.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip
new file mode 100644
index 00000000000..4d307129021
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_TIKA-1251.doc.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip
new file mode 100644
index 00000000000..58d7f15b23b
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_package.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip
new file mode 100644
index 00000000000..e9d802cfd4e
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_embedded_zip.pptx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip
new file mode 100644
index 00000000000..57d55538294
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_list_override.rtf.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip
new file mode 100644
index 00000000000..edd97967e95
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip
new file mode 100644
index 00000000000..84da48f0250
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/test_recursive_embedded_npe.docx.zip differ
diff --git a/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip
new file mode 100644
index 00000000000..41972bc8447
Binary files /dev/null and b/plugins/ingest-attachment/src/test/resources/org/elasticsearch/ingest/attachment/test/tika-files/tika434.html.zip differ
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 95a0b85dba3..3d1418dc940 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -50,6 +50,7 @@ import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
@@ -63,18 +64,32 @@ public final class GeoIpProcessor extends AbstractProcessor {
     private final String targetField;
     private final DatabaseReader dbReader;
     private final Set<Property> properties;
+    private final boolean ignoreMissing;
 
-    GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties) throws IOException {
+    GeoIpProcessor(String tag, String field, DatabaseReader dbReader, String targetField, Set<Property> properties,
+                   boolean ignoreMissing) throws IOException {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.dbReader = dbReader;
         this.properties = properties;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) {
-        String ip = ingestDocument.getFieldValue(field, String.class);
+        String ip = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
+
+        if (ip == null && ignoreMissing) {
+            return;
+        } else if (ip == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information.");
+        }
+
         final InetAddress ipAddress = InetAddresses.forString(ip);
 
         Map<String, Object> geoData;
@@ -124,10 +139,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
     }
 
     private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
         CityResponse response = AccessController.doPrivileged((PrivilegedAction<CityResponse>) () -> {
             try {
                 return dbReader.city(ipAddress);
@@ -202,10 +214,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
     }
 
     private Map<String, Object> retrieveCountryGeoData(InetAddress ipAddress) {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
         CountryResponse response = AccessController.doPrivileged((PrivilegedAction<CountryResponse>) () -> {
             try {
                 return dbReader.country(ipAddress);
@@ -268,6 +277,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             DatabaseReader databaseReader = databaseReaders.get(databaseFile);
             if (databaseReader == null) {
@@ -298,7 +308,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
                 }
             }
 
-            return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties);
+            return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties, ignoreMissing);
         }
     }
 
@@ -307,7 +317,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
     // it with an unchecked exception.
     private static final class AddressNotFoundRuntimeException extends RuntimeException {
 
-        public AddressNotFoundRuntimeException(Throwable cause) {
+        AddressNotFoundRuntimeException(Throwable cause) {
             super(cause);
         }
     }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 162137b5f3c..0c80bcc71fd 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -60,7 +60,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")),
                 geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
 
-        NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomPositiveLong()));
+        NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomNonNegativeLong()));
         databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, cache);
     }
 
@@ -85,6 +85,24 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testSetIgnoreMissing() throws Exception {
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+        String processorTag = randomAsciiOfLength(10);
+
+        GeoIpProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("geoip"));
+        assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
+        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
+        assertTrue(processor.isIgnoreMissing());
     }
 
     public void testCountryBuildDefaults() throws Exception {
@@ -102,6 +120,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
@@ -112,6 +131,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         GeoIpProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildDbFile() throws Exception {
@@ -124,6 +144,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
         assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildWithCountryDbAndCityFields() throws Exception {
@@ -174,6 +195,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         GeoIpProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
+        assertFalse(processor.isIgnoreMissing());
     }
 
     public void testBuildIllegalFieldOption() throws Exception {
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index 3b2f65e2814..221c06d3b68 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -20,17 +20,20 @@
 package org.elasticsearch.ingest.geoip;
 
 import com.maxmind.geoip2.DatabaseReader;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.zip.GZIPInputStream;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -40,7 +43,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testCity() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "8.8.8.8");
@@ -64,10 +67,52 @@ public class GeoIpProcessorTests extends ESTestCase {
         assertThat(geoData.get("location"), equalTo(location));
     }
 
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot extract geoip information."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
     public void testCity_withIpV6() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         String address = "2602:306:33d3:8000::3257:9652";
         Map<String, Object> document = new HashMap<>();
@@ -95,7 +140,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testCityWithMissingLocation() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "93.114.45.13");
@@ -112,7 +157,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testCountry() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "82.170.213.79");
@@ -132,7 +177,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testCountryWithMissingLocation() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+            new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "93.114.45.13");
@@ -149,7 +194,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testAddressIsNotInTheDatabase() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "127.0.0.1");
@@ -162,7 +207,7 @@ public class GeoIpProcessorTests extends ESTestCase {
     public void testInvalid() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
-                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class), false);
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "www.google.com");
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
index ed381dab0b6..33f7c67e018 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
index af764d5baf2..0d9660e9fb5 100644
--- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -39,11 +40,11 @@ final class UserAgentParser {
     private final List<UserAgentSubpattern> osPatterns = new ArrayList<>();
     private final List<UserAgentSubpattern> devicePatterns = new ArrayList<>();
     private final String name;
-    
-    public UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
+
+    UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
         this.name = name;
         this.cache = cache;
-        
+
         try {
             init(regexStream);
         } catch (IOException e) {
@@ -52,17 +53,18 @@ final class UserAgentParser {
     }
 
     private void init(InputStream regexStream) throws IOException {
-        XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(regexStream);
-        
+        // EMPTY is safe here because we don't use namedObject
+        XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, regexStream);
+
         XContentParser.Token token = yamlParser.nextToken();
-        
+
         if (token == XContentParser.Token.START_OBJECT) {
             token = yamlParser.nextToken();
-            
+
             for (; token != null; token = yamlParser.nextToken()) {
                 if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("user_agent_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"),
@@ -71,7 +73,7 @@ final class UserAgentParser {
                 }
                 else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"),
@@ -80,7 +82,7 @@ final class UserAgentParser {
                 }
                 else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) {
                     List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
-                    
+
                     for (Map<String, String> map : parserConfigurations) {
                         devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
                                 map.get("device_replacement"), null, null, null, null));
@@ -88,7 +90,7 @@ final class UserAgentParser {
                 }
             }
         }
-        
+
         if (uaPatterns.isEmpty() && osPatterns.isEmpty() && devicePatterns.isEmpty()) {
             throw new ElasticsearchParseException("not a valid regular expression file");
         }
@@ -105,38 +107,38 @@ final class UserAgentParser {
 
     private List<Map<String, String>> readParserConfigurations(XContentParser yamlParser) throws IOException {
         List <Map<String, String>> patternList = new ArrayList<>();
-        
+
         XContentParser.Token token = yamlParser.nextToken();
         if (token != XContentParser.Token.START_ARRAY) {
             throw new ElasticsearchParseException("malformed regular expression file, should continue with 'array' after 'object'");
         }
-        
+
         token = yamlParser.nextToken();
         if (token != XContentParser.Token.START_OBJECT) {
             throw new ElasticsearchParseException("malformed regular expression file, expecting 'object'");
         }
-        
+
         while (token == XContentParser.Token.START_OBJECT) {
             token = yamlParser.nextToken();
-            
+
             if (token != XContentParser.Token.FIELD_NAME) {
                 throw new ElasticsearchParseException("malformed regular expression file, should continue with 'field_name' after 'array'");
             }
-            
+
             Map<String, String> regexMap = new HashMap<>();
             for (; token == XContentParser.Token.FIELD_NAME; token = yamlParser.nextToken()) {
                 String fieldName = yamlParser.currentName();
-                
+
                 token = yamlParser.nextToken();
                 String fieldValue = yamlParser.text();
                 regexMap.put(fieldName, fieldValue);
             }
 
             patternList.add(regexMap);
-            
+
             token = yamlParser.nextToken();
         }
-        
+
         return patternList;
     }
 
@@ -151,21 +153,21 @@ final class UserAgentParser {
     List<UserAgentSubpattern> getDevicePatterns() {
         return devicePatterns;
     }
-    
+
     String getName() {
         return name;
     }
 
     public Details parse(String agentString) {
         Details details = cache.get(name, agentString);;
-        
+
         if (details == null) {
             VersionedName userAgent = findMatch(uaPatterns, agentString);
             VersionedName operatingSystem = findMatch(osPatterns, agentString);
             VersionedName device = findMatch(devicePatterns, agentString);
-            
+
             details = new Details(userAgent, operatingSystem, device);
-            
+
             cache.put(name, agentString, details);
         }
 
@@ -181,7 +183,7 @@ final class UserAgentParser {
                 return name;
             }
         }
-        
+
         return null;
     }
 
@@ -189,22 +191,22 @@ final class UserAgentParser {
         public final VersionedName userAgent;
         public final VersionedName operatingSystem;
         public final VersionedName device;
-        
-        public Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
+
+        Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
             this.userAgent = userAgent;
             this.operatingSystem = operatingSystem;
             this.device = device;
         }
     }
-    
+
     static final class VersionedName {
         public final String name;
         public final String major;
         public final String minor;
         public final String patch;
         public final String build;
-        
-        public VersionedName(String name, String major, String minor, String patch, String build) {
+
+        VersionedName(String name, String major, String minor, String patch, String build) {
             this.name = name;
             this.major = major;
             this.minor = minor;
@@ -212,7 +214,7 @@ final class UserAgentParser {
             this.build = build;
         }
     }
-    
+
     /**
      * One of: user agent, operating system, device
      */
@@ -220,7 +222,7 @@ final class UserAgentParser {
         private final Pattern pattern;
         private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement;
 
-        public UserAgentSubpattern(Pattern pattern, String nameReplacement,
+        UserAgentSubpattern(Pattern pattern, String nameReplacement,
                 String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) {
           this.pattern = pattern;
           this.nameReplacement = nameReplacement;
@@ -261,19 +263,19 @@ final class UserAgentParser {
           } else if (groupCount >= 3) {
             minor = matcher.group(3);
           }
-          
+
           if (v3Replacement != null) {
               patch = v3Replacement;
           } else if (groupCount >= 4) {
               patch = matcher.group(4);
           }
-          
+
           if (v4Replacement != null) {
               build = v4Replacement;
           } else if (groupCount >= 5) {
               build = matcher.group(5);
           }
-          
+
           return name == null ? null : new VersionedName(name, major, minor, patch, build);
         }
       }
diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
index ec18126457b..93f210c427b 100644
--- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -34,6 +34,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
@@ -44,20 +45,32 @@ public class UserAgentProcessor extends AbstractProcessor {
     private final String field;
     private final String targetField;
     private final Set<Property> properties;
-    
     private final UserAgentParser parser;
+    private final boolean ignoreMissing;
 
-    public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties) {
+    public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties,
+                              boolean ignoreMissing) {
         super(tag);
         this.field = field;
         this.targetField = targetField;
         this.parser = parser;
         this.properties = properties;
+        this.ignoreMissing = ignoreMissing;
+    }
+
+    boolean isIgnoreMissing() {
+        return ignoreMissing;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
-        String userAgent = ingestDocument.getFieldValue(field, String.class);
+        String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
+
+        if (userAgent == null && ignoreMissing) {
+            return;
+        } else if (userAgent == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent.");
+        }
 
         Details uaClient = parser.parse(userAgent);
 
@@ -99,7 +112,7 @@ public class UserAgentProcessor extends AbstractProcessor {
                     else {
                         uaDetails.put("os", "Other");
                     }
-                    
+
                     break;
                 case OS_NAME:
                     if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
@@ -168,7 +181,7 @@ public class UserAgentProcessor extends AbstractProcessor {
     public String getType() {
         return TYPE;
     }
-    
+
     String getField() {
         return field;
     }
@@ -180,7 +193,7 @@ public class UserAgentProcessor extends AbstractProcessor {
     Set<Property> getProperties() {
         return properties;
     }
-    
+
     UserAgentParser getUaParser() {
         return parser;
     }
@@ -188,7 +201,7 @@ public class UserAgentProcessor extends AbstractProcessor {
     public static final class Factory implements Processor.Factory {
 
         private final Map<String, UserAgentParser> userAgentParsers;
-        
+
         public Factory(Map<String, UserAgentParser> userAgentParsers) {
             this.userAgentParsers = userAgentParsers;
         }
@@ -200,13 +213,14 @@ public class UserAgentProcessor extends AbstractProcessor {
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
                 throw newConfigurationException(TYPE, processorTag,
                         "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)");
             }
-            
+
             final Set<Property> properties;
             if (propertyNames != null) {
                 properties = EnumSet.noneOf(Property.class);
@@ -221,7 +235,7 @@ public class UserAgentProcessor extends AbstractProcessor {
                 properties = EnumSet.allOf(Property.class);
             }
 
-            return new UserAgentProcessor(processorTag, field, targetField, parser, properties);
+            return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing);
         }
     }
 
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
index 2acac873637..cc70c1ef6ba 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class IngestUserAgentClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
index 10c6ccd7941..c1c46283076 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -89,6 +89,27 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
         assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertFalse(processor.isIgnoreMissing());
+    }
+
+    public void testBuildWithIgnoreMissing() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("ignore_missing", true);
+
+        String processorTag = randomAsciiOfLength(10);
+
+        UserAgentProcessor processor = factory.create(null, processorTag, config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("user_agent"));
+        assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
+        assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertTrue(processor.isIgnoreMissing());
     }
 
     public void testBuildTargetField() throws Exception {
diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
index d9b5eed059a..2c9e72ae9d9 100644
--- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
@@ -27,55 +27,96 @@ import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 
 public class UserAgentProcessorTests extends ESTestCase {
 
     private static UserAgentProcessor processor;
-    
+
     @BeforeClass
     public static void setupProcessor() throws IOException {
         InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yaml");
         assertNotNull(regexStream);
-        
+
         UserAgentParser parser = new UserAgentParser(randomAsciiOfLength(10), regexStream, new UserAgentCache(1000));
-        
+
         processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", parser,
-                EnumSet.allOf(UserAgentProcessor.Property.class));
+                EnumSet.allOf(UserAgentProcessor.Property.class), false);
     }
-    
+
+    public void testNullValueWithIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNonExistentWithIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), true);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        processor.execute(ingestDocument);
+        assertIngestDocument(originalIngestDocument, ingestDocument);
+    }
+
+    public void testNullWithoutIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
+            Collections.singletonMap("source_field", null));
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse user-agent."));
+    }
+
+    public void testNonExistentWithoutIgnoreMissing() throws Exception {
+        UserAgentProcessor processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", null,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false);
+        IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
+        IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
+        Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
+        assertThat(exception.getMessage(), equalTo("field [source_field] not present as part of path [source_field]"));
+    }
+
     @SuppressWarnings("unchecked")
     public void testCommonBrowser() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Chrome"));
         assertThat(target.get("major"), is("33"));
         assertThat(target.get("minor"), is("0"));
         assertThat(target.get("patch"), is("1750"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Mac OS X 10.9.2"));
         assertThat(target.get("os_name"), is("Mac OS X"));
         assertThat(target.get("os_major"), is("10"));
         assertThat(target.get("os_minor"), is("9"));
-        
+
         assertThat(target.get("device"), is("Other"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testUncommonDevice() throws Exception {
         Map<String, Object> document = new HashMap<>();
@@ -83,78 +124,78 @@ public class UserAgentProcessorTests extends ESTestCase {
                 "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10+ "
                 + "(KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Android"));
         assertThat(target.get("major"), is("3"));
         assertThat(target.get("minor"), is("0"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Android 3.0"));
         assertThat(target.get("os_name"), is("Android"));
         assertThat(target.get("os_major"), is("3"));
         assertThat(target.get("os_minor"), is("0"));
-        
+
         assertThat(target.get("device"), is("Motorola Xoom"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testSpider() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("EasouSpider"));
         assertNull(target.get("major"));
         assertNull(target.get("minor"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Other"));
         assertThat(target.get("os_name"), is("Other"));
         assertNull(target.get("os_major"));
         assertNull(target.get("os_minor"));
-        
+
         assertThat(target.get("device"), is("Spider"));
     }
-    
+
     @SuppressWarnings("unchecked")
     public void testUnknown() throws Exception {
         Map<String, Object> document = new HashMap<>();
         document.put("source_field",
             "Something I made up v42.0.1");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
-        
+
         processor.execute(ingestDocument);
         Map<String, Object> data = ingestDocument.getSourceAndMetadata();
 
         assertThat(data, hasKey("target_field"));
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
-        
+
         assertThat(target.get("name"), is("Other"));
         assertNull(target.get("major"));
         assertNull(target.get("minor"));
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
-        
+
         assertThat(target.get("os"), is("Other"));
         assertThat(target.get("os_name"), is("Other"));
         assertNull(target.get("os_major"));
         assertNull(target.get("os_minor"));
-        
+
         assertThat(target.get("device"), is("Other"));
     }
 }
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
index 88ff7609ff9..d7f3f94c818 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
@@ -20,7 +20,6 @@ package org.elasticsearch.plugin.example;
 
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
@@ -36,7 +35,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class ExampleCatAction extends AbstractCatAction {
     private final ExamplePluginConfiguration config;
 
-    @Inject
     public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) {
         super(settings);
         this.config = config;
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
index 21582ccde52..de8cb091156 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExamplePluginConfiguration.java
@@ -19,31 +19,31 @@
 
 package org.elasticsearch.plugin.example;
 
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Locale;
 
 /**
  * Example configuration.
  */
 public class ExamplePluginConfiguration {
-
     private final Settings customSettings;
 
     public static final Setting<String> TEST_SETTING =
       new Setting<String>("test", "default_value",
       (value) -> value, Setting.Property.Dynamic);
 
-    @Inject
-    public ExamplePluginConfiguration(Environment env) throws IOException {
+    public ExamplePluginConfiguration(Environment env) {
         // The directory part of the location matches the artifactId of this plugin
         Path path = env.configFile().resolve("jvm-example/example.yaml");
-        customSettings = Settings.builder().loadFromPath(path).build();
+        try {
+            customSettings = Settings.builder().loadFromPath(path).build();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to load settings, giving up", e);
+        }
 
         // asserts for tests
         assert customSettings != null;
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
index ac76858d110..431d4818791 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
@@ -19,40 +19,32 @@
 
 package org.elasticsearch.plugin.example;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesModule;
-import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+import static java.util.Collections.singletonList;
 
 /**
  * Example of a plugin.
  */
-public class JvmExamplePlugin extends Plugin {
-
-    private final Settings settings;
+public class JvmExamplePlugin extends Plugin implements ActionPlugin {
+    private final ExamplePluginConfiguration config;
 
     public JvmExamplePlugin(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public Collection<Module> createGuiceModules() {
-        return Collections.<Module>singletonList(new ConfiguredExampleModule());
-    }
-
-    @Override
-    @SuppressWarnings("rawtypes") // Plugin use a rawtype
-    public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
-        Collection<Class<? extends LifecycleComponent>> services = new ArrayList<>();
-        return services;
+        Environment environment = new Environment(settings);
+        config = new ExamplePluginConfiguration(environment);
     }
 
     @Override
@@ -60,16 +52,10 @@ public class JvmExamplePlugin extends Plugin {
         return Settings.EMPTY;
     }
 
-    /**
-     * Module declaring some example configuration and a _cat action that uses
-     * it.
-     */
-    public static class ConfiguredExampleModule extends AbstractModule {
-        @Override
-        protected void configure() {
-          bind(ExamplePluginConfiguration.class).asEagerSingleton();
-          Multibinder<AbstractCatAction> catActionMultibinder = Multibinder.newSetBinder(binder(), AbstractCatAction.class);
-          catActionMultibinder.addBinding().to(ExampleCatAction.class).asEagerSingleton();
-        }
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return singletonList(new ExampleCatAction(settings, restController, config));
     }
 }
diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java
index 1f48549aad4..3ed616cb3af 100644
--- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java
+++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugin.example;
 
+import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.BufferedReader;
@@ -34,7 +35,7 @@ public class ExampleExternalIT extends ESTestCase {
         String stringAddress = Objects.requireNonNull(System.getProperty("external.address"));
         URL url = new URL("http://" + stringAddress);
         InetAddress address = InetAddress.getByName(url.getHost());
-        try (Socket socket = new Socket(address, url.getPort());
+        try (Socket socket = new MockSocket(address, url.getPort());
              BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) {
            assertEquals("TEST", reader.readLine());
         }
diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
index b7bae90817b..d3e84316b14 100644
--- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
+++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class JvmExampleClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
index 3e9a5f13927..9ca9f677b8d 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class MapperMurmur3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
index af5886bb6f9..325ac726713 100644
--- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
+++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java
@@ -63,8 +63,8 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase {
         Supplier<QueryShardContext> queryShardContext = () -> {
             return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
-        parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(),
-                indexService.getIndexAnalyzers(), indexService.similarityService(), mapperRegistry, queryShardContext);
+        parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), indexService.getIndexAnalyzers(),
+                indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext);
     }
 
     @Override
@@ -160,8 +160,9 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase {
         Supplier<QueryShardContext> queryShardContext = () -> {
             return indexService2x.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); });
         };
-        DocumentMapperParser parser = new DocumentMapperParser(indexService2x.getIndexSettings(), indexService2x.mapperService(), indexService2x.getIndexAnalyzers(),
-            indexService2x.similarityService(), mapperRegistry, queryShardContext);
+        DocumentMapperParser parser = new DocumentMapperParser(indexService2x.getIndexSettings(), indexService2x.mapperService(),
+                indexService2x.getIndexAnalyzers(), indexService2x.xContentRegistry(), indexService2x.similarityService(), mapperRegistry,
+                queryShardContext);
 
         DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, defaultMapper.mappingSource().string());
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
index c2a7e79ff2b..b2974f139fb 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
@@ -32,14 +32,13 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TypeParsers;
 
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-
 public class SizeFieldMapper extends MetadataFieldMapper {
     public static final String NAME = "_size";
 
@@ -102,8 +101,8 @@ public class SizeFieldMapper extends MetadataFieldMapper {
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode) ?
-                        EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
+                    boolean enabled = TypeParsers.nodeBooleanValue(name, "enabled", fieldNode, parserContext);
+                    builder.enabled(enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();
                 }
             }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
index d8de3635b77..3741c5626f6 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class MapperSizeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
index c4de761c8e7..92cbb2e6e39 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
@@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.plugin.mapper.MapperSizePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -107,7 +108,7 @@ public class SizeMappingIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true"));
         final String source = "{\"f\":10}";
         indexRandom(true,
-                client().prepareIndex("test", "type", "1").setSource(source));
+                client().prepareIndex("test", "type", "1").setSource(source, XContentType.JSON));
         GetResponse getResponse = client().prepareGet("test", "type", "1").setStoredFields("_size").get();
         assertNotNull(getResponse.getField("_size"));
         assertEquals(source.length(), getResponse.getField("_size").getValue());
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index 3fa5300fb43..2cde1b1bd07 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -60,7 +61,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         boolean stored = false;
         boolean points = false;
@@ -81,7 +82,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
@@ -95,7 +96,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
             .field("field", "value")
             .endObject()
             .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source, XContentType.JSON));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index b77cebfa2f2..e7b6911447f 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -100,16 +100,6 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
         return this.client.doesContainerExist(this.accountName, this.locMode, container);
     }
 
-    public void removeContainer(String container) throws URISyntaxException, StorageException
-    {
-        this.client.removeContainer(this.accountName, this.locMode, container);
-    }
-
-    public void createContainer(String container) throws URISyntaxException, StorageException
-    {
-        this.client.createContainer(this.accountName, this.locMode, container);
-    }
-
     public void deleteFiles(String container, String path) throws URISyntaxException, StorageException
     {
         this.client.deleteFiles(this.accountName, this.locMode, container, path);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/util/SocketAccess.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/util/SocketAccess.java
new file mode 100644
index 00000000000..6202a0a46f8
--- /dev/null
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/util/SocketAccess.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.azure.blobstore.util;
+
+import com.microsoft.azure.storage.StorageException;
+import org.elasticsearch.SpecialPermission;
+
+import java.net.SocketPermission;
+import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * This plugin uses azure libraries to connect to azure storage services. For these remote calls the plugin needs
+ * {@link SocketPermission} 'connect' to establish connections. This class wraps the operations requiring access in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks.
+ */
+public final class SocketAccess {
+
+    private SocketAccess() {}
+
+    public static <T> T doPrivilegedException(PrivilegedExceptionAction<T> operation) throws StorageException, URISyntaxException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (StorageException) e.getCause();
+        }
+    }
+
+    public static void doPrivilegedVoidException(StorageRunnable action) throws StorageException, URISyntaxException {
+        SpecialPermission.check();
+        try {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                action.executeCouldThrow();
+                return null;
+            });
+        } catch (PrivilegedActionException e) {
+            Throwable cause = e.getCause();
+            if (cause instanceof StorageException) {
+                throw (StorageException) cause;
+            } else {
+                throw (URISyntaxException) cause;
+            }
+        }
+    }
+
+    @FunctionalInterface
+    public interface StorageRunnable {
+        void executeCouldThrow() throws StorageException, URISyntaxException;
+    }
+
+}
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 3e854ab9c70..6343541aed3 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -42,6 +43,9 @@ import java.util.function.Function;
  */
 public interface AzureStorageService {
 
+    ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
+    ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
+
     final class Storage {
         public static final String PREFIX = "cloud.azure.storage.";
 
@@ -58,7 +62,7 @@ public interface AzureStorageService {
         public static final Setting<String> LOCATION_MODE_SETTING =
             Setting.simpleString("repositories.azure.location_mode", Property.NodeScope);
         public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
-            Setting.byteSizeSetting("repositories.azure.chunk_size", new ByteSizeValue(-1), Property.NodeScope);
+            Setting.byteSizeSetting("repositories.azure.chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope);
         public static final Setting<Boolean> COMPRESS_SETTING =
             Setting.boolSetting("repositories.azure.compress", false, Property.NodeScope);
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index 697f05e3dee..e1eea1f57f4 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -29,6 +29,7 @@ import com.microsoft.azure.storage.blob.CloudBlockBlob;
 import com.microsoft.azure.storage.blob.ListBlobItem;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.cloud.azure.blobstore.util.SocketAccess;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@@ -42,6 +43,9 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -79,12 +83,12 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
     void createClient(AzureStorageSettings azureStorageSettings) {
         try {
             logger.trace("creating new Azure storage client using account [{}], key [{}]",
-                    azureStorageSettings.getAccount(), azureStorageSettings.getKey());
+                azureStorageSettings.getAccount(), azureStorageSettings.getKey());
 
             String storageConnectionString =
-                    "DefaultEndpointsProtocol=https;"
-                            + "AccountName="+ azureStorageSettings.getAccount() +";"
-                            + "AccountKey=" + azureStorageSettings.getKey();
+                "DefaultEndpointsProtocol=https;"
+                    + "AccountName=" + azureStorageSettings.getAccount() + ";"
+                    + "AccountKey=" + azureStorageSettings.getKey();
 
             // Retrieve storage account from connection-string.
             CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
@@ -151,7 +155,7 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         try {
             CloudBlobClient client = this.getSelectedClient(account, mode);
             CloudBlobContainer blobContainer = client.getContainerReference(container);
-            return blobContainer.exists();
+            return SocketAccess.doPrivilegedException(blobContainer::exists);
         } catch (Exception e) {
             logger.error("can not access container [{}]", container);
         }
@@ -163,7 +167,7 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         CloudBlobClient client = this.getSelectedClient(account, mode);
         CloudBlobContainer blobContainer = client.getContainerReference(container);
         logger.trace("removing container [{}]", container);
-        blobContainer.deleteIfExists();
+        SocketAccess.doPrivilegedException(blobContainer::deleteIfExists);
     }
 
     @Override
@@ -172,7 +176,7 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
             CloudBlobClient client = this.getSelectedClient(account, mode);
             CloudBlobContainer blobContainer = client.getContainerReference(container);
             logger.trace("creating container [{}]", container);
-            blobContainer.createIfNotExists();
+            SocketAccess.doPrivilegedException(blobContainer::createIfNotExists);
         } catch (IllegalArgumentException e) {
             logger.trace((Supplier<?>) () -> new ParameterizedMessage("fails creating container [{}]", container), e);
             throw new RepositoryException(container, e.getMessage(), e);
@@ -186,14 +190,16 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         // Container name must be lower case.
         CloudBlobClient client = this.getSelectedClient(account, mode);
         CloudBlobContainer blobContainer = client.getContainerReference(container);
-        if (blobContainer.exists()) {
-            // We list the blobs using a flat blob listing mode
-            for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
-                String blobName = blobNameFromUri(blobItem.getUri());
-                logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
-                deleteBlob(account, mode, container, blobName);
+        SocketAccess.doPrivilegedVoidException(() -> {
+            if (blobContainer.exists()) {
+                // We list the blobs using a flat blob listing mode
+                for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
+                    String blobName = blobNameFromUri(blobItem.getUri());
+                    logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
+                    deleteBlob(account, mode, container, blobName);
+                }
             }
-        }
+        });
     }
 
     /**
@@ -223,7 +229,7 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         CloudBlobContainer blobContainer = client.getContainerReference(container);
         if (blobContainer.exists()) {
             CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
-            return azureBlob.exists();
+            return SocketAccess.doPrivilegedException(azureBlob::exists);
         }
 
         return false;
@@ -239,7 +245,7 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         if (blobContainer.exists()) {
             logger.trace("container [{}]: blob [{}] found. removing.", container, blob);
             CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
-            azureBlob.delete();
+            SocketAccess.doPrivilegedVoidException(azureBlob::delete);
         }
     }
 
@@ -247,14 +253,16 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
     public InputStream getInputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
         logger.trace("reading container [{}], blob [{}]", container, blob);
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
+        CloudBlockBlob blockBlobReference = client.getContainerReference(container).getBlockBlobReference(blob);
+        return SocketAccess.doPrivilegedException(blockBlobReference::openInputStream);
     }
 
     @Override
     public OutputStream getOutputStream(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
         logger.trace("writing container [{}], blob [{}]", container, blob);
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        return client.getContainerReference(container).getBlockBlobReference(blob).openOutputStream();
+        CloudBlockBlob blockBlobReference = client.getContainerReference(container).getBlockBlobReference(blob);
+        return SocketAccess.doPrivilegedException(blockBlobReference::openOutputStream);
     }
 
     @Override
@@ -265,30 +273,32 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
 
         logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
-
         CloudBlobClient client = this.getSelectedClient(account, mode);
         CloudBlobContainer blobContainer = client.getContainerReference(container);
-        if (blobContainer.exists()) {
-            for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) {
-                URI uri = blobItem.getUri();
-                logger.trace("blob url [{}]", uri);
 
-                // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
-                // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
-                String blobPath = uri.getPath().substring(1 + container.length() + 1);
+        SocketAccess.doPrivilegedVoidException(() -> {
+            if (blobContainer.exists()) {
+                for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix))) {
+                    URI uri = blobItem.getUri();
+                    logger.trace("blob url [{}]", uri);
 
-                CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobPath);
+                    // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
+                    // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
+                    String blobPath = uri.getPath().substring(1 + container.length() + 1);
 
-                // fetch the blob attributes from Azure (getBlockBlobReference does not do this)
-                // this is needed to retrieve the blob length (among other metadata) from Azure Storage
-                blob.downloadAttributes();
+                    CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobPath);
 
-                BlobProperties properties = blob.getProperties();
-                String name = blobPath.substring(keyPath.length());
-                logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
-                blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
+                    // fetch the blob attributes from Azure (getBlockBlobReference does not do this)
+                    // this is needed to retrieve the blob length (among other metadata) from Azure Storage
+                    blob.downloadAttributes();
+
+                    BlobProperties properties = blob.getProperties();
+                    String name = blobPath.substring(keyPath.length());
+                    logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
+                    blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
+                }
             }
-        }
+        });
 
         return blobsBuilder.immutableMap();
     }
@@ -302,8 +312,10 @@ public class AzureStorageServiceImpl extends AbstractComponent implements AzureS
         CloudBlockBlob blobSource = blobContainer.getBlockBlobReference(sourceBlob);
         if (blobSource.exists()) {
             CloudBlockBlob blobTarget = blobContainer.getBlockBlobReference(targetBlob);
-            blobTarget.startCopy(blobSource);
-            blobSource.delete();
+            SocketAccess.doPrivilegedVoidException(() -> {
+                blobTarget.startCopy(blobSource);
+                blobSource.delete();
+            });
             logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
         }
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index 9d67eea628b..5e0de46f65a 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.node.Node;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -34,19 +35,14 @@ import java.util.Map;
 import java.util.function.Function;
 
 public final class AzureStorageSettings {
-    private static final Setting.AffixKey TIMEOUT_KEY = Setting.AffixKey.withAffix(Storage.PREFIX, "timeout");
-
-    private static final Setting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(
-        TIMEOUT_KEY,
-        (s) -> Storage.TIMEOUT_SETTING.get(s).toString(),
-        (s) -> Setting.parseTimeValue(s, TimeValue.timeValueSeconds(-1), TIMEOUT_KEY.toString()),
-        Setting.Property.NodeScope);
+    private static final Setting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(Storage.PREFIX, "timeout",
+        (key) -> Setting.timeSetting(key, Storage.TIMEOUT_SETTING, Setting.Property.NodeScope));
     private static final Setting<String> ACCOUNT_SETTING =
-        Setting.affixKeySetting(Storage.PREFIX, "account", "", Function.identity(), Setting.Property.NodeScope);
+        Setting.affixKeySetting(Storage.PREFIX, "account", (key) -> Setting.simpleString(key, Setting.Property.NodeScope));
     private static final Setting<String> KEY_SETTING =
-        Setting.affixKeySetting(Storage.PREFIX, "key", "", Function.identity(), Setting.Property.NodeScope);
+        Setting.affixKeySetting(Storage.PREFIX, "key", (key) -> Setting.simpleString(key, Setting.Property.NodeScope));
     private static final Setting<Boolean> DEFAULT_SETTING =
-        Setting.affixKeySetting(Storage.PREFIX, "default", "false", Boolean::valueOf, Setting.Property.NodeScope);
+        Setting.affixKeySetting(Storage.PREFIX, "default", (key) -> Setting.boolSetting(key, false, Setting.Property.NodeScope));
 
 
     private final String name;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index 5b938fce188..f8953833074 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
@@ -45,9 +46,9 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin {
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.singletonMap(AzureRepository.TYPE,
-            (metadata) -> new AzureRepository(metadata, env, createStorageService(env.settings())));
+            (metadata) -> new AzureRepository(metadata, env, namedXContentRegistry, createStorageService(env.settings())));
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 2ae5fbe8493..50d087e713d 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -19,12 +19,6 @@
 
 package org.elasticsearch.repositories.azure;
 
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Locale;
-import java.util.function.Function;
-
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
@@ -32,22 +26,26 @@ import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
-import org.elasticsearch.repositories.IndexId;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.repositories.RepositoryVerificationException;
+import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
-import org.elasticsearch.snapshots.SnapshotCreationException;
+import org.elasticsearch.snapshots.SnapshotId;
 
-import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getEffectiveSetting;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+
+import static org.elasticsearch.cloud.azure.storage.AzureStorageService.MAX_CHUNK_SIZE;
+import static org.elasticsearch.cloud.azure.storage.AzureStorageService.MIN_CHUNK_SIZE;
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
 
 /**
@@ -63,8 +61,6 @@ import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValu
  */
 public class AzureRepository extends BlobStoreRepository {
 
-    private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
-
     public static final String TYPE = "azure";
 
     public static final class Repository {
@@ -74,7 +70,7 @@ public class AzureRepository extends BlobStoreRepository {
         public static final Setting<String> BASE_PATH_SETTING = Setting.simpleString("base_path", Property.NodeScope);
         public static final Setting<String> LOCATION_MODE_SETTING = Setting.simpleString("location_mode", Property.NodeScope);
         public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
-            Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, Property.NodeScope);
+            Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope);
         public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
     }
 
@@ -84,20 +80,14 @@ public class AzureRepository extends BlobStoreRepository {
     private final boolean compress;
     private final boolean readonly;
 
-    public AzureRepository(RepositoryMetaData metadata, Environment environment, AzureStorageService storageService)
+    public AzureRepository(RepositoryMetaData metadata, Environment environment,
+                           NamedXContentRegistry namedXContentRegistry, AzureStorageService storageService)
         throws IOException, URISyntaxException, StorageException {
-        super(metadata, environment.settings());
+        super(metadata, environment.settings(), namedXContentRegistry);
 
         blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
         String container = getValue(metadata.settings(), settings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
-        ByteSizeValue configuredChunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
-        if (configuredChunkSize.getMb() > MAX_CHUNK_SIZE.getMb()) {
-            Setting<ByteSizeValue> setting = getEffectiveSetting(metadata.settings(), Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
-            throw new SettingsException("["  + setting.getKey() + "] must not exceed [" + MAX_CHUNK_SIZE + "] but is set to [" + configuredChunkSize + "].");
-        } else {
-            this.chunkSize = configuredChunkSize;
-        }
-
+        this.chunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
         this.compress = getValue(metadata.settings(), settings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING);
         String modeStr = getValue(metadata.settings(), settings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
         Boolean forcedReadonly = metadata.settings().getAsBoolean("readonly", null);
@@ -162,34 +152,12 @@ public class AzureRepository extends BlobStoreRepository {
 
     @Override
     public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
-        try {
-            if (!blobStore.doesContainerExist(blobStore.container())) {
-                logger.debug("container [{}] does not exist. Creating...", blobStore.container());
-                blobStore.createContainer(blobStore.container());
-            }
-            super.initializeSnapshot(snapshotId, indices, clusterMetadata);
-        } catch (StorageException | URISyntaxException e) {
-            logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage());
-            throw new SnapshotCreationException(getMetadata().name(), snapshotId, e);
+        if (blobStore.doesContainerExist(blobStore.container()) == false) {
+            throw new IllegalArgumentException("The bucket [" + blobStore.container() + "] does not exist. Please create it before " +
+                " creating an azure snapshot repository backed by it.");
         }
     }
 
-    @Override
-    public String startVerification() {
-        if (readonly == false) {
-            try {
-                if (!blobStore.doesContainerExist(blobStore.container())) {
-                    logger.debug("container [{}] does not exist. Creating...", blobStore.container());
-                    blobStore.createContainer(blobStore.container());
-                }
-            } catch (StorageException | URISyntaxException e) {
-                logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage());
-                throw new RepositoryVerificationException(getMetadata().name(), "can not initialize container " + blobStore.container(), e);
-            }
-        }
-        return super.startVerification();
-    }
-
     @Override
     public boolean isReadOnly() {
         return readonly;
diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 00000000000..19a35f84059
--- /dev/null
+++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  // azure client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
+};
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
deleted file mode 100644
index e42dcfcd298..00000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.StorageException;
-
-import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoryMissingException;
-import org.elasticsearch.test.store.MockFSDirectoryService;
-import org.elasticsearch.test.store.MockFSIndexStore;
-import org.junit.After;
-import org.junit.Before;
-
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collection;
-
-public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
-
-    private static final AzureStorageService storageService = new AzureStorageServiceMock();
-
-    public static class TestPlugin extends AzureRepositoryPlugin {
-        @Override
-        protected AzureStorageService createStorageService(Settings settings) {
-            return storageService;
-        }
-    }
-
-    protected String basePath;
-
-    public AbstractAzureRepositoryServiceIntegTestCase(String basePath) {
-        this.basePath = basePath;
-    }
-
-    /**
-     * Deletes repositories, supports wildcard notation.
-     */
-    public static void wipeRepositories(String... repositories) {
-        // if nothing is provided, delete all
-        if (repositories.length == 0) {
-            repositories = new String[]{"*"};
-        }
-        for (String repository : repositories) {
-            try {
-                client().admin().cluster().prepareDeleteRepository(repository).get();
-            } catch (RepositoryMissingException ex) {
-                // ignore
-            }
-        }
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder builder = Settings.builder()
-                .put(Storage.CONTAINER_SETTING.getKey(), "snapshots");
-        return builder.build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(TestPlugin.class, MockFSIndexStore.TestPlugin.class);
-    }
-
-    @Override
-    public Settings indexSettings() {
-        // During restore we frequently restore index to exactly the same state it was before, that might cause the same
-        // checksum file to be written twice during restore operation
-        return Settings.builder().put(super.indexSettings())
-                .put(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING.getKey(), false)
-                .put(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING.getKey(), false)
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                .build();
-    }
-
-    @Before @After
-    public final void wipe() throws StorageException, URISyntaxException {
-        wipeRepositories();
-        cleanRepositoryFiles(basePath);
-    }
-
-    /**
-     * Purge the test container
-     */
-    public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
-        String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
-        logger.info("--> remove blobs in container [{}]", container);
-        storageService.deleteFiles(null, LocationMode.PRIMARY_ONLY, container, path);
-    }
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index cf7f0cd1abb..ba377c03c47 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -147,7 +147,7 @@ public class AzureStorageServiceTests extends ESTestCase {
      * This internal class just overload createClient method which is called by AzureStorageServiceImpl.doStart()
      */
     class AzureStorageServiceMock extends AzureStorageServiceImpl {
-        public AzureStorageServiceMock(Settings settings) {
+        AzureStorageServiceMock(Settings settings) {
             super(settings);
         }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
index e82ed3a1875..b67073d7d38 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
@@ -62,7 +62,7 @@ public class AzureStorageSettingsFilterTests extends ESTestCase {
         settings.toXContent(xContentBuilder, request);
         xContentBuilder.endObject();
         String filteredSettingsString = xContentBuilder.string();
-        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString).build();
+        filteredSettings = Settings.builder().loadFromSource(filteredSettingsString, xContentBuilder.contentType()).build();
         assertThat(filteredSettings.getAsMap().keySet(), contains("cloud.azure.storage.azure1.default"));
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
deleted file mode 100644
index ccd9bf8cf81..00000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.azure;
-
-import com.microsoft.azure.storage.StorageException;
-import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
-import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
-import org.elasticsearch.cluster.metadata.RepositoryMetaData;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.ESBlobStoreTestCase;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
-
-/**
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml}
- * in order to run these tests.
- */
-@ESIntegTestCase.ThirdParty
-public class AzureBlobStoreTests extends ESBlobStoreTestCase {
-    @Override
-    protected BlobStore newBlobStore() throws IOException {
-        try {
-            Settings settings = readSettingsFromFile();
-            RepositoryMetaData metadata = new RepositoryMetaData("ittest", "azure", Settings.EMPTY);
-            AzureStorageService storageService = new AzureStorageServiceImpl(settings);
-            AzureBlobStore blobStore = new AzureBlobStore(metadata, settings, storageService);
-            blobStore.createContainer(blobStore.container());
-            return blobStore;
-        } catch (URISyntaxException | StorageException e) {
-            throw new IOException(e);
-        }
-    }
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
index 68b3783b03a..014fa88cc4d 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java
@@ -21,23 +21,19 @@ package org.elasticsearch.repositories.azure;
 
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
-import org.elasticsearch.cloud.azure.storage.AzureStorageSettings;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.net.URI;
 import java.net.URISyntaxException;
 
-import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class AzureRepositorySettingsTests extends ESTestCase {
 
@@ -47,7 +43,8 @@ public class AzureRepositorySettingsTests extends ESTestCase {
             .putArray(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
             .put(settings)
             .build();
-        return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings), new Environment(internalSettings), null);
+        return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings), new Environment(internalSettings),
+            NamedXContentRegistry.EMPTY, null);
     }
 
 
@@ -101,4 +98,30 @@ public class AzureRepositorySettingsTests extends ESTestCase {
             .put("readonly", false)
             .build()).isReadOnly(), is(false));
     }
+
+    public void testChunkSize() throws StorageException, IOException, URISyntaxException {
+        // default chunk size
+        AzureRepository azureRepository = azureRepository(Settings.EMPTY);
+        assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());
+
+        // chunk size in settings
+        int size = randomIntBetween(1, 64);
+        azureRepository = azureRepository(Settings.builder().put("chunk_size", size + "mb").build());
+        assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), azureRepository.chunkSize());
+
+        // zero bytes is not allowed
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "0").build()));
+        assertEquals("Failed to parse value [0] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // negative bytes not allowed
+        e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "-1").build()));
+        assertEquals("Failed to parse value [-1] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // greater than max chunk size not allowed
+        e = expectThrows(IllegalArgumentException.class, () ->
+            azureRepository(Settings.builder().put("chunk_size", "65mb").build()));
+        assertEquals("Failed to parse value [65mb] for setting [chunk_size] must be <= 64mb", e.getMessage());
+    }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
deleted file mode 100644
index 808ae13e678..00000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.azure;
-
-
-import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
-import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceIntegTestCase;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.snapshots.SnapshotInfo;
-import org.elasticsearch.snapshots.SnapshotShardFailure;
-import org.elasticsearch.snapshots.SnapshotState;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.is;
-
-@ESIntegTestCase.ClusterScope(
-        scope = ESIntegTestCase.Scope.SUITE,
-        supportsDedicatedMasters = false,
-        numDataNodes = 1,
-        numClientNodes = 0,
-        transportClientRatio = 0.0)
-public class AzureSnapshotRestoreServiceIntegTests extends AbstractAzureRepositoryServiceIntegTestCase {
-    public AzureSnapshotRestoreServiceIntegTests() {
-        super("/snapshot-test/repo-" + randomInt());
-    }
-
-    public void testSimpleWorkflow() {
-        Client client = client();
-        logger.info("-->  creating azure repository with path [{}]", basePath);
-        PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
-                .setType("azure").setSettings(Settings.builder()
-                        .put("base_path", basePath)
-                        .put("chunk_size", randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                ).get();
-        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-        createIndex("test-idx-1", "test-idx-2", "test-idx-3");
-        ensureGreen();
-
-        logger.info("--> indexing some data");
-        for (int i = 0; i < 100; i++) {
-            index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
-            index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
-            index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
-
-        logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster()
-                .prepareCreateSnapshot("test-repo", "test-snap")
-                .setWaitForCompletion(true)
-                .setIndices("test-idx-*", "-test-idx-3")
-                .get();
-
-        final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
-        if (snapshotInfo.shardFailures() != null) {
-            for (SnapshotShardFailure shardFailure : snapshotInfo.shardFailures()) {
-                logger.warn("shard failure during snapshot: {}", shardFailure::toString);
-            }
-        }
-        assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS));
-        assertEquals(snapshotInfo.failedShards(), 0);
-
-        logger.info("--> delete some data");
-        for (int i = 0; i < 50; i++) {
-            client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 50; i < 100; i++) {
-            client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
-        }
-        for (int i = 0; i < 100; i += 2) {
-            client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
-        }
-        refresh();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(50L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        logger.info("--> close indices");
-        client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
-
-        logger.info("--> restore all indices from the snapshot");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
-            .setWaitForCompletion(true).get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(50L));
-
-        // Test restore after index deletion
-        logger.info("--> delete indices");
-        cluster().wipeIndices("test-idx-1", "test-idx-2");
-        logger.info("--> restore one index after deletion");
-        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
-            .setIndices("test-idx-*", "-test-idx-2").get();
-        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
-        ensureGreen();
-        assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
-        ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
-        assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
-    }
-
-}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
index 2919d073978..5d3d051fb3b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryAzureClientYamlTestSuiteIT extends ESClientYamlSuiteTestC
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
index b63d03487b2..01b8d510be9 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.BlobStoreException;
+import org.elasticsearch.common.blobstore.gcs.util.SocketAccess;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
@@ -103,7 +104,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      */
     boolean doesBucketExist(String bucketName) {
         try {
-            return doPrivileged(() -> {
+            return SocketAccess.doPrivilegedIOException(() -> {
                 try {
                     Bucket bucket = client.buckets().get(bucketName).execute();
                     if (bucket != null) {
@@ -130,7 +131,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @return a map of blob names and their metadata
      */
     Map<String, BlobMetaData> listBlobs(String path) throws IOException {
-        return doPrivileged(() -> listBlobsByPath(bucket, path, path));
+        return SocketAccess.doPrivilegedIOException(() -> listBlobsByPath(bucket, path, path));
     }
 
     /**
@@ -141,7 +142,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @return a map of blob names and their metadata
      */
     Map<String, BlobMetaData> listBlobsByPrefix(String path, String prefix) throws IOException {
-        return doPrivileged(() -> listBlobsByPath(bucket, buildKey(path, prefix), path));
+        return SocketAccess.doPrivilegedIOException(() -> listBlobsByPath(bucket, buildKey(path, prefix), path));
     }
 
     /**
@@ -165,21 +166,19 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @return true if the blob exists, false otherwise
      */
     boolean blobExists(String blobName) throws IOException {
-        return doPrivileged(() -> {
-            try {
-                StorageObject blob = client.objects().get(bucket, blobName).execute();
-                if (blob != null) {
-                    return Strings.hasText(blob.getId());
-                }
-            } catch (GoogleJsonResponseException e) {
-                GoogleJsonError error = e.getDetails();
-                if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
-                    return false;
-                }
-                throw e;
+        try {
+            StorageObject blob = SocketAccess.doPrivilegedIOException(() -> client.objects().get(bucket, blobName).execute());
+            if (blob != null) {
+                return Strings.hasText(blob.getId());
             }
-            return false;
-        });
+        } catch (GoogleJsonResponseException e) {
+            GoogleJsonError error = e.getDetails();
+            if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
+                return false;
+            }
+            throw e;
+        }
+        return false;
     }
 
     /**
@@ -189,18 +188,18 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @return an InputStream
      */
     InputStream readBlob(String blobName) throws IOException {
-        return doPrivileged(() -> {
-            try {
+        try {
+            return SocketAccess.doPrivilegedIOException(() -> {
                 Storage.Objects.Get object = client.objects().get(bucket, blobName);
                 return object.executeMediaAsInputStream();
-            } catch (GoogleJsonResponseException e) {
-                GoogleJsonError error = e.getDetails();
-                if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
-                    throw new NoSuchFileException(e.getMessage());
-                }
-                throw e;
+            });
+        } catch (GoogleJsonResponseException e) {
+            GoogleJsonError error = e.getDetails();
+            if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
+                throw new NoSuchFileException(e.getMessage());
             }
-        });
+            throw e;
+        }
     }
 
     /**
@@ -210,14 +209,13 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @param blobSize    expected size of the blob to be written
      */
     void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
-        doPrivileged(() -> {
+        SocketAccess.doPrivilegedVoidIOException(() -> {
             InputStreamContent stream = new InputStreamContent(null, inputStream);
             stream.setLength(blobSize);
 
             Storage.Objects.Insert insert = client.objects().insert(bucket, null, stream);
             insert.setName(blobName);
             insert.execute();
-            return null;
         });
     }
 
@@ -230,7 +228,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
         if (!blobExists(blobName)) {
             throw new NoSuchFileException("Blob [" + blobName + "] does not exist");
         }
-        doPrivileged(() -> client.objects().delete(bucket, blobName).execute());
+        SocketAccess.doPrivilegedIOException(() -> client.objects().delete(bucket, blobName).execute());
     }
 
     /**
@@ -239,10 +237,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @param prefix prefix of the buckets to delete
      */
     void deleteBlobsByPrefix(String prefix) throws IOException {
-        doPrivileged(() -> {
-            deleteBlobs(listBlobsByPath(bucket, prefix, null).keySet());
-            return null;
-        });
+        deleteBlobs(listBlobsByPath(bucket, prefix, null).keySet());
     }
 
     /**
@@ -259,11 +254,10 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
             deleteBlob(blobNames.iterator().next());
             return;
         }
+        final List<Storage.Objects.Delete> deletions = new ArrayList<>();
+        final Iterator<String> blobs = blobNames.iterator();
 
-        doPrivileged(() -> {
-            final List<Storage.Objects.Delete> deletions = new ArrayList<>();
-            final Iterator<String> blobs = blobNames.iterator();
-
+        SocketAccess.doPrivilegedVoidIOException(() -> {
             while (blobs.hasNext()) {
                 // Create a delete request for each blob to delete
                 deletions.add(client.objects().delete(bucket, blobs.next()));
@@ -282,7 +276,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
                                 @Override
                                 public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException {
                                     logger.error("failed to delete blob [{}] in bucket [{}]: {}", delete.getObject(), delete.getBucket(), e
-                                            .getMessage());
+                                        .getMessage());
                                 }
 
                                 @Override
@@ -302,7 +296,6 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
                     }
                 }
             }
-            return null;
         });
     }
 
@@ -313,7 +306,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
      * @param targetBlob new name of the blob in the target bucket
      */
     void moveBlob(String sourceBlob, String targetBlob) throws IOException {
-        doPrivileged(() -> {
+        SocketAccess.doPrivilegedIOException(() -> {
             // There's no atomic "move" in GCS so we need to copy and delete
             client.objects().copy(bucket, sourceBlob, bucket, targetBlob, null).execute();
             client.objects().delete(bucket, sourceBlob).execute();
@@ -321,21 +314,6 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
         });
     }
 
-    /**
-     * Executes a {@link PrivilegedExceptionAction} with privileges enabled.
-     */
-    <T> T doPrivileged(PrivilegedExceptionAction<T> operation) throws IOException {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        try {
-            return AccessController.doPrivileged((PrivilegedExceptionAction<T>) operation::run);
-        } catch (PrivilegedActionException e) {
-            throw (IOException) e.getException();
-        }
-    }
-
     private String buildKey(String keyPath, String s) {
         assert s != null;
         return keyPath + s;
@@ -370,7 +348,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
         private final Storage.Objects.List list;
 
         StorageObjectsSpliterator(Storage client, String bucketName, String prefix, long pageSize) throws IOException {
-            list = client.objects().list(bucketName);
+            list = SocketAccess.doPrivilegedIOException(() -> client.objects().list(bucketName));
             list.setMaxResults(pageSize);
             if (prefix != null) {
                 list.setPrefix(prefix);
@@ -381,7 +359,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
         public boolean tryAdvance(Consumer<? super StorageObject> action) {
             try {
                 // Retrieves the next page of items
-                Objects objects = list.execute();
+                Objects objects = SocketAccess.doPrivilegedIOException(list::execute);
 
                 if ((objects == null) || (objects.getItems() == null) || (objects.getItems().isEmpty())) {
                     return false;
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/util/SocketAccess.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/util/SocketAccess.java
new file mode 100644
index 00000000000..5d2b4c6bb21
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/util/SocketAccess.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs.util;
+
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.CheckedRunnable;
+
+import java.io.IOException;
+import java.net.SocketPermission;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * This plugin uses google api/client libraries to connect to google cloud services. For these remote calls the plugin
+ * needs {@link SocketPermission} 'connect' to establish connections. This class wraps the operations requiring access
+ * in {@link AccessController#doPrivileged(PrivilegedAction)} blocks.
+ */
+public final class SocketAccess {
+
+    private SocketAccess() {}
+
+    public static <T> T doPrivilegedIOException(PrivilegedExceptionAction<T> operation) throws IOException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
+    public static void doPrivilegedVoidIOException(CheckedRunnable<IOException> action) throws IOException {
+        SpecialPermission.check();
+        try {
+            AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+                action.run();
+                return null;
+            });
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
index c0fa38e8b57..b20ed34e532 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
@@ -21,7 +21,6 @@ package org.elasticsearch.plugin.repository.gcs;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 
@@ -40,12 +39,10 @@ import com.google.api.services.storage.model.Bucket;
 import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository;
 import org.elasticsearch.repositories.gcs.GoogleCloudStorageService;
@@ -63,10 +60,7 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin
          * our plugin permissions don't allow core to "reach through" plugins to
          * change the permission. Because that'd be silly.
          */
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
             // ClassInfo put in cache all the fields of a given class
             // that are annoted with @Key; at the same time it changes
@@ -120,8 +114,8 @@ public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.singletonMap(GoogleCloudStorageRepository.TYPE,
-            (metadata) -> new GoogleCloudStorageRepository(metadata, env, createStorageService(env)));
+            (metadata) -> new GoogleCloudStorageRepository(metadata, env, namedXContentRegistry, createStorageService(env)));
     }
 }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index f7b74d5a4f8..b7473a2d5fa 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.repositories.RepositoryException;
@@ -45,6 +46,10 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 public class GoogleCloudStorageRepository extends BlobStoreRepository {
 
+    // package private for testing
+    static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
+    static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(100, ByteSizeUnit.MB);
+
     public static final String TYPE = "gcs";
 
     public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
@@ -56,7 +61,7 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     public static final Setting<Boolean> COMPRESS =
             boolSetting("compress", false, Property.NodeScope, Property.Dynamic);
     public static final Setting<ByteSizeValue> CHUNK_SIZE =
-            byteSizeSetting("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope, Property.Dynamic);
+            byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
     public static final Setting<String> APPLICATION_NAME =
             new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
     public static final Setting<String> SERVICE_ACCOUNT =
@@ -72,12 +77,13 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     private final GoogleCloudStorageBlobStore blobStore;
 
     public GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
+                                        NamedXContentRegistry namedXContentRegistry,
                                         GoogleCloudStorageService storageService) throws Exception {
-        super(metadata, environment.settings());
+        super(metadata, environment.settings(), namedXContentRegistry);
 
-        String bucket = get(BUCKET, metadata);
-        String application = get(APPLICATION_NAME, metadata);
-        String serviceAccount = get(SERVICE_ACCOUNT, metadata);
+        String bucket = getSetting(BUCKET, metadata);
+        String application = getSetting(APPLICATION_NAME, metadata);
+        String serviceAccount = getSetting(SERVICE_ACCOUNT, metadata);
 
         String basePath = BASE_PATH.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
@@ -103,8 +109,8 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
             readTimeout = timeout;
         }
 
-        this.compress = get(COMPRESS, metadata);
-        this.chunkSize = get(CHUNK_SIZE, metadata);
+        this.compress = getSetting(COMPRESS, metadata);
+        this.chunkSize = getSetting(CHUNK_SIZE, metadata);
 
         logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
                 bucket, basePath, chunkSize, compress, application);
@@ -137,7 +143,7 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     /**
      * Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty.
      */
-    static <T> T get(Setting<T> setting, RepositoryMetaData metadata) {
+    static <T> T getSetting(Setting<T> setting, RepositoryMetaData metadata) {
         T value = setting.get(metadata.settings());
         if (value == null) {
             throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is not defined for repository");
diff --git a/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
index bc7acd60602..ce9b0334638 100644
--- a/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
@@ -23,4 +23,7 @@ grant {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   permission java.net.URLPermission "http://www.googleapis.com/*", "*";
   permission java.net.URLPermission "https://www.googleapis.com/*", "*";
+
+  // gcs client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index 15cd647dc5c..eeb877dabf5 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.repositories.gcs;
 
 import com.google.api.services.storage.Storage;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.gcs.MockHttpTransport;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
@@ -80,4 +82,42 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
             return storage.get();
         }
     }
+
+    public void testChunkSize() {
+        // default chunk size
+        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE, Settings.EMPTY);
+        ByteSizeValue chunkSize = GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repositoryMetaData);
+        assertEquals(GoogleCloudStorageRepository.MAX_CHUNK_SIZE, chunkSize);
+
+        // chunk size in settings
+        int size = randomIntBetween(1, 100);
+        repositoryMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                       Settings.builder().put("chunk_size", size + "mb").build());
+        chunkSize = GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repositoryMetaData);
+        assertEquals(new ByteSizeValue(size, ByteSizeUnit.MB), chunkSize);
+
+        // zero bytes is not allowed
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "0").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [0] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // negative bytes not allowed
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "-1").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [-1] for setting [chunk_size] must be >= 1b", e.getMessage());
+
+        // greater than max chunk size not allowed
+        e = expectThrows(IllegalArgumentException.class, () -> {
+            RepositoryMetaData repoMetaData = new RepositoryMetaData("repo", GoogleCloudStorageRepository.TYPE,
+                                                                        Settings.builder().put("chunk_size", "101mb").build());
+            GoogleCloudStorageRepository.getSetting(GoogleCloudStorageRepository.CHUNK_SIZE, repoMetaData);
+        });
+        assertEquals("Failed to parse value [101mb] for setting [chunk_size] must be <= 100mb", e.getMessage());
+    }
 }
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
index 6ed036e277f..cdd86bc48b9 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryGcsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCas
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index 8cfb5043b6c..213f2fe0f07 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -368,3 +368,9 @@ thirdPartyAudit.excludes = [
   'org.apache.log4j.helpers.ISO8601DateFormat',
   'org.apache.log4j.spi.ThrowableInformation'
 ]
+
+// Gradle 2.13 bundles org.slf4j.impl.StaticLoggerBinder in its core.jar which leaks into the forbidden APIs ant task
+// Gradle 2.14+ does not bundle this class anymore so we need to properly exclude it here.
+if (GradleVersion.current() > GradleVersion.version("2.13")) {
+  thirdPartyAudit.excludes += ['org.slf4j.impl.StaticLoggerBinder']
+}
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index e485cf56b4c..4649cf858d2 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -20,11 +20,9 @@ package org.elasticsearch.repositories.hdfs;
 
 import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -32,10 +30,14 @@ import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.repositories.hdfs.HdfsBlobStore.Operation;
 
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.NoSuchFileException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.LinkedHashMap;
@@ -56,12 +58,7 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
     @Override
     public boolean blobExists(String blobName) {
         try {
-            return store.execute(new Operation<Boolean>() {
-                @Override
-                public Boolean run(FileContext fileContext) throws IOException {
-                    return fileContext.util().exists(new Path(path, blobName));
-                }
-            });
+            return store.execute(fileContext -> fileContext.util().exists(new Path(path, blobName)));
         } catch (Exception e) {
             return false;
         }
@@ -73,22 +70,14 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
             throw new NoSuchFileException("Blob [" + blobName + "] does not exist");
         }
 
-        store.execute(new Operation<Boolean>() {
-            @Override
-            public Boolean run(FileContext fileContext) throws IOException {
-                return fileContext.delete(new Path(path, blobName), true);
-            }
-        });
+        store.execute(fileContext -> fileContext.delete(new Path(path, blobName), true));
     }
 
     @Override
     public void move(String sourceBlobName, String targetBlobName) throws IOException {
-        store.execute(new Operation<Void>() {
-            @Override
-            public Void run(FileContext fileContext) throws IOException {
-                fileContext.rename(new Path(path, sourceBlobName), new Path(path, targetBlobName));
-                return null;
-            }
+        store.execute((Operation<Void>) fileContext -> {
+            fileContext.rename(new Path(path, sourceBlobName), new Path(path, targetBlobName));
+            return null;
         });
     }
 
@@ -98,12 +87,10 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
             throw new NoSuchFileException("Blob [" + blobName + "] does not exist");
         }
         // FSDataInputStream does buffering internally
-        return store.execute(new Operation<InputStream>() {
-            @Override
-            public InputStream run(FileContext fileContext) throws IOException {
-                return fileContext.open(new Path(path, blobName), bufferSize);
-            }
-        });
+        // FSDataInputStream can open connections on read() or skip() so we wrap in
+        // HDFSPrivilegedInputSteam which will ensure that underlying methods will
+        // be called with the proper privileges.
+        return store.execute(fileContext -> new HDFSPrivilegedInputSteam(fileContext.open(new Path(path, blobName), bufferSize)));
     }
 
     @Override
@@ -111,45 +98,33 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
         if (blobExists(blobName)) {
             throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
         }
-        store.execute(new Operation<Void>() {
-            @Override
-            public Void run(FileContext fileContext) throws IOException {
-                Path blob = new Path(path, blobName);
-                // we pass CREATE, which means it fails if a blob already exists.
-                // NOTE: this behavior differs from FSBlobContainer, which passes TRUNCATE_EXISTING
-                // that should be fixed there, no need to bring truncation into this, give the user an error.
-                EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK);
-                CreateOpts[] opts = { CreateOpts.bufferSize(bufferSize) };
-                try (FSDataOutputStream stream = fileContext.create(blob, flags, opts)) {
-                    int bytesRead;
-                    byte[] buffer = new byte[bufferSize];
-                    while ((bytesRead = inputStream.read(buffer)) != -1) {
-                        stream.write(buffer, 0, bytesRead);
-                        //  For safety we also hsync each write as well, because of its docs:
-                        //  SYNC_BLOCK - to force closed blocks to the disk device
-                        // "In addition Syncable.hsync() should be called after each write,
-                        //  if true synchronous behavior is required"
-                        stream.hsync();
-                    }
+        store.execute((Operation<Void>) fileContext -> {
+            Path blob = new Path(path, blobName);
+            // we pass CREATE, which means it fails if a blob already exists.
+            // NOTE: this behavior differs from FSBlobContainer, which passes TRUNCATE_EXISTING
+            // that should be fixed there, no need to bring truncation into this, give the user an error.
+            EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE, CreateFlag.SYNC_BLOCK);
+            CreateOpts[] opts = {CreateOpts.bufferSize(bufferSize)};
+            try (FSDataOutputStream stream = fileContext.create(blob, flags, opts)) {
+                int bytesRead;
+                byte[] buffer = new byte[bufferSize];
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    stream.write(buffer, 0, bytesRead);
+                    //  For safety we also hsync each write as well, because of its docs:
+                    //  SYNC_BLOCK - to force closed blocks to the disk device
+                    // "In addition Syncable.hsync() should be called after each write,
+                    //  if true synchronous behavior is required"
+                    stream.hsync();
                 }
-                return null;
             }
+            return null;
         });
     }
 
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable final String prefix) throws IOException {
-        FileStatus[] files = store.execute(new Operation<FileStatus[]>() {
-            @Override
-            public FileStatus[] run(FileContext fileContext) throws IOException {
-                return (fileContext.util().listStatus(path, new PathFilter() {
-                    @Override
-                    public boolean accept(Path path) {
-                        return prefix == null || path.getName().startsWith(prefix);
-                    }
-                }));
-            }
-        });
+        FileStatus[] files = store.execute(fileContext -> (fileContext.util().listStatus(path,
+            path -> prefix == null || path.getName().startsWith(prefix))));
         Map<String, BlobMetaData> map = new LinkedHashMap<String, BlobMetaData>();
         for (FileStatus file : files) {
             map.put(file.getPath().getName(), new PlainBlobMetaData(file.getPath().getName(), file.getLen()));
@@ -161,4 +136,51 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
     public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
+
+    /**
+     * Exists to wrap underlying InputStream methods that might make socket connections in
+     * doPrivileged blocks. This is due to the way that hdfs client libraries might open
+     * socket connections when you are reading from an InputStream.
+     */
+    private static class HDFSPrivilegedInputSteam extends FilterInputStream {
+
+        HDFSPrivilegedInputSteam(InputStream in) {
+            super(in);
+        }
+
+        public int read() throws IOException {
+            return doPrivilegedOrThrow(in::read);
+        }
+
+        public int read(byte b[]) throws IOException {
+            return doPrivilegedOrThrow(() -> in.read(b));
+        }
+
+        public int read(byte b[], int off, int len) throws IOException {
+            return doPrivilegedOrThrow(() -> in.read(b, off, len));
+        }
+
+        public long skip(long n) throws IOException {
+            return doPrivilegedOrThrow(() -> in.skip(n));
+        }
+
+        public int available() throws IOException {
+            return doPrivilegedOrThrow(() -> in.available());
+        }
+
+        public synchronized void reset() throws IOException {
+            doPrivilegedOrThrow(() -> {
+                in.reset();
+                return null;
+            });
+        }
+
+        private static  <T> T doPrivilegedOrThrow(PrivilegedExceptionAction<T> action) throws IOException {
+            try {
+                return AccessController.doPrivileged(action);
+            } catch (PrivilegedActionException e) {
+                throw (IOException) e.getCause();
+            }
+        }
+    }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
index 23404a7c360..7ce5e8d3cd8 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.blobstore.BlobStore;
 
 import java.io.IOException;
 import java.lang.reflect.ReflectPermission;
+import java.net.SocketPermission;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
@@ -46,12 +47,7 @@ final class HdfsBlobStore implements BlobStore {
     HdfsBlobStore(FileContext fileContext, String path, int bufferSize) throws IOException {
         this.fileContext = fileContext;
         this.bufferSize = bufferSize;
-        this.root = execute(new Operation<Path>() {
-          @Override
-          public Path run(FileContext fileContext) throws IOException {
-              return fileContext.makeQualified(new Path(path));
-          }
-        });
+        this.root = execute(fileContext1 -> fileContext1.makeQualified(new Path(path)));
         try {
             mkdirs(root);
         } catch (FileAlreadyExistsException ok) {
@@ -60,23 +56,17 @@ final class HdfsBlobStore implements BlobStore {
     }
 
     private void mkdirs(Path path) throws IOException {
-        execute(new Operation<Void>() {
-            @Override
-            public Void run(FileContext fileContext) throws IOException {
-                fileContext.mkdir(path, null, true);
-                return null;
-            }
+        execute((Operation<Void>) fileContext -> {
+            fileContext.mkdir(path, null, true);
+            return null;
         });
     }
 
     @Override
     public void delete(BlobPath path) throws IOException {
-        execute(new Operation<Void>() {
-            @Override
-            public Void run(FileContext fc) throws IOException {
-                fc.delete(translateToHdfsPath(path), true);
-                return null;
-            }
+        execute((Operation<Void>) fc -> {
+            fc.delete(translateToHdfsPath(path), true);
+            return null;
         });
     }
 
@@ -109,7 +99,7 @@ final class HdfsBlobStore implements BlobStore {
         }
         return path;
     }
-    
+
     interface Operation<V> {
         V run(FileContext fileContext) throws IOException;
     }
@@ -121,22 +111,14 @@ final class HdfsBlobStore implements BlobStore {
     // 1) hadoop dynamic proxy is messy with access rules
     // 2) allow hadoop to add credentials to our Subject
     <V> V execute(Operation<V> operation) throws IOException {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            // unprivileged code such as scripts do not have SpecialPermission
-            sm.checkPermission(new SpecialPermission());
-        }
+        SpecialPermission.check();
         if (closed) {
             throw new AlreadyClosedException("HdfsBlobStore is closed: " + this);
         }
         try {
-            return AccessController.doPrivileged(new PrivilegedExceptionAction<V>() {
-                @Override
-                public V run() throws IOException {
-                    return operation.run(fileContext);
-                }
-            }, null, new ReflectPermission("suppressAccessChecks"),
-                     new AuthPermission("modifyPrivateCredentials"));
+            return AccessController.doPrivileged((PrivilegedExceptionAction<V>)
+                    () -> operation.run(fileContext), null, new ReflectPermission("suppressAccessChecks"),
+                     new AuthPermission("modifyPrivateCredentials"), new SocketPermission("*", "connect"));
         } catch (PrivilegedActionException pae) {
             throw (IOException) pae.getException();
         }
@@ -146,4 +128,4 @@ final class HdfsBlobStore implements BlobStore {
     public void close() {
         closed = true;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index d4af26c3bbc..9ea53a5acf2 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -28,26 +28,18 @@ import java.util.Map;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
-import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.Repository;
 
 public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
 
     // initialize some problematic classes with elevated privileges
     static {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                return evilHadoopInit();
-            }
-        });
+        SpecialPermission.check();
+        AccessController.doPrivileged((PrivilegedAction<Void>) HdfsPlugin::evilHadoopInit);
     }
 
     @SuppressForbidden(reason = "Needs a security hack for hadoop on windows, until HADOOP-XXXX is fixed")
@@ -88,7 +80,7 @@ public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
-        return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env));
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
+        return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env, namedXContentRegistry));
     }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index b111a5d0d0a..d784e8bf093 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -45,6 +45,7 @@ import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
@@ -60,8 +61,9 @@ public final class HdfsRepository extends BlobStoreRepository {
     // TODO: why 100KB?
     private static final ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(100, ByteSizeUnit.KB);
 
-    public HdfsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
-        super(metadata, environment.settings());
+    public HdfsRepository(RepositoryMetaData metadata, Environment environment,
+                          NamedXContentRegistry namedXContentRegistry) throws IOException {
+        super(metadata, environment.settings(), namedXContentRegistry);
 
         this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
         this.compress = metadata.settings().getAsBoolean("compress", false);
@@ -88,21 +90,14 @@ public final class HdfsRepository extends BlobStoreRepository {
         if (pathSetting == null) {
             throw new IllegalArgumentException("No 'path' defined for hdfs snapshot/restore");
         }
-        
+
         int bufferSize = getMetadata().settings().getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
 
         try {
             // initialize our filecontext
-            SecurityManager sm = System.getSecurityManager();
-            if (sm != null) {
-                sm.checkPermission(new SpecialPermission());
-            }
-            FileContext fileContext = AccessController.doPrivileged(new PrivilegedAction<FileContext>() {
-                @Override
-                public FileContext run() {
-                    return createContext(uri, getMetadata().settings());
-                }
-            });
+            SpecialPermission.check();
+            FileContext fileContext = AccessController.doPrivileged((PrivilegedAction<FileContext>)
+                () -> createContext(uri, getMetadata().settings()));
             blobStore = new HdfsBlobStore(fileContext, pathSetting, bufferSize);
             logger.debug("Using file-system [{}] for URI [{}], path [{}]", fileContext.getDefaultFileSystem(), fileContext.getDefaultFileSystem().getUri(), pathSetting);
         } catch (IOException e) {
@@ -110,7 +105,7 @@ public final class HdfsRepository extends BlobStoreRepository {
         }
         super.doStart();
     }
-    
+
     // create hadoop filecontext
     @SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)")
     private static FileContext createContext(URI uri, Settings repositorySettings)  {
@@ -139,15 +134,12 @@ public final class HdfsRepository extends BlobStoreRepository {
         cfg.setBoolean("fs.hdfs.impl.disable.cache", true);
 
         // create the filecontext with our user
-        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
-            @Override
-            public FileContext run() {
-                try {
-                    AbstractFileSystem fs = AbstractFileSystem.get(uri, cfg);
-                    return FileContext.getFileContext(fs, cfg);
-                } catch (UnsupportedFileSystemException e) {
-                    throw new RuntimeException(e);
-                }
+        return Subject.doAs(subject, (PrivilegedAction<FileContext>) () -> {
+            try {
+                AbstractFileSystem fs = AbstractFileSystem.get(uri, cfg);
+                return FileContext.getFileContext(fs, cfg);
+            } catch (UnsupportedFileSystemException e) {
+                throw new RuntimeException(e);
             }
         });
     }
diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
index 85447245c96..b800f3eee46 100644
--- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy
@@ -35,4 +35,7 @@ grant {
   permission javax.security.auth.AuthPermission "getSubject";
   permission javax.security.auth.AuthPermission "doAs";
   permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
+
+  // hdfs client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
index cdc6dd96803..2bfe6843daf 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
@@ -32,28 +32,25 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.security.AccessController;
 import java.security.Principal;
 import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Collections;
 
 public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
 
     @Override
     protected BlobStore newBlobStore() throws IOException {
-        return AccessController.doPrivileged(
-                new PrivilegedAction<HdfsBlobStore>() {
-                    @Override
-                    public HdfsBlobStore run() {
-                        try {
-                            FileContext fileContext = createContext(new URI("hdfs:///"));
-                            return new HdfsBlobStore(fileContext, "temp", 1024);
-                        } catch (IOException | URISyntaxException e) {
-                            throw new RuntimeException(e);
-                        }
-            }
-        });
+        FileContext fileContext;
+        try {
+            fileContext = AccessController.doPrivileged((PrivilegedExceptionAction<FileContext>)
+                () -> createContext(new URI("hdfs:///")));
+        } catch (PrivilegedActionException e) {
+            throw new RuntimeException(e.getCause());
+        }
+        return new HdfsBlobStore(fileContext, "temp", 1024);
     }
 
     @SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)")
@@ -90,15 +87,12 @@ public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
         cfg.set("fs.AbstractFileSystem." + uri.getScheme() + ".impl", TestingFs.class.getName());
 
         // create the FileContext with our user
-        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
-            @Override
-            public FileContext run() {
-                try {
-                    TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
-                    return FileContext.getFileContext(fs, cfg);
-                } catch (UnsupportedFileSystemException e) {
-                    throw new RuntimeException(e);
-                }
+        return Subject.doAs(subject, (PrivilegedAction<FileContext>) () -> {
+            try {
+                TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
+                return FileContext.getFileContext(fs, cfg);
+            } catch (UnsupportedFileSystemException e) {
+                throw new RuntimeException(e);
             }
         });
     }
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
index 264a350d514..6e66b0d49fe 100644
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java
@@ -18,14 +18,13 @@
  */
 package org.elasticsearch.repositories.hdfs;
 
-import java.io.IOException;
-
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
+
+import java.io.IOException;
 
 public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -34,7 +33,7 @@ public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index b1369908670..ce29393bf49 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -54,11 +54,6 @@ bundlePlugin {
   }
 }
 
-test {
-  // this is needed for insecure plugins, remove if possible!
-  systemProperty 'tests.artifact', project.name
-}
-
 thirdPartyAudit.excludes = [
   // classes are missing
   'javax.servlet.ServletContextEvent', 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index ddaa45d15fd..6efc3ec82a0 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -19,142 +19,136 @@
 
 package org.elasticsearch.cloud.aws;
 
+import java.util.Locale;
+import java.util.function.Function;
+
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
-import java.util.Locale;
-import java.util.function.Function;
-
 public interface AwsS3Service extends LifecycleComponent {
 
-    // Global AWS settings (shared between discovery-ec2 and repository-s3)
+    // Legacy global AWS settings (shared between discovery-ec2 and repository-s3)
     // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
     // the code there if you change anything here.
     /**
      * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
      */
-    Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> KEY_SETTING = new Setting<>("cloud.aws.access_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
      */
-    Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> SECRET_SETTING = new Setting<>("cloud.aws.secret_key", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
      */
-    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope, Property.Shared);
+    Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https",
+        s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host",
+        Property.NodeScope, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
-        Property.Shared);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16,
+        Property.NodeScope, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
+    Setting<SecureString> PROXY_USERNAME_SETTING = new Setting<>("cloud.aws.proxy.username", "", SecureString::new,
+        Property.NodeScope, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
+    Setting<SecureString> PROXY_PASSWORD_SETTING = new Setting<>("cloud.aws.proxy.password", "", SecureString::new,
+        Property.NodeScope, Property.Filtered, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
-    /**
-     * cloud.aws.region: Region. Shared with discovery-ec2 plugin
-     */
-    Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer",
+        Property.NodeScope, Property.Deprecated, Property.Shared);
     /**
      * cloud.aws.read_timeout: Socket read timeout. Shared with discovery-ec2 plugin
      */
     Setting<TimeValue> READ_TIMEOUT = Setting.timeSetting("cloud.aws.read_timeout",
-        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Shared);
+        TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope, Property.Deprecated, Property.Shared);
 
     /**
      * Defines specific s3 settings starting with cloud.aws.s3.
+     * NOTE: These are legacy settings. Use the named client configs in {@link org.elasticsearch.repositories.s3.S3Repository}.
      */
     interface CLOUD_S3 {
         /**
          * cloud.aws.s3.access_key: AWS Access key specific for S3 API calls. Defaults to cloud.aws.access_key.
          * @see AwsS3Service#KEY_SETTING
          */
-        Setting<String> KEY_SETTING =
-            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
+        Setting<SecureString> KEY_SETTING =
+            new Setting<>("cloud.aws.s3.access_key", AwsS3Service.KEY_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.s3.secret_key: AWS Secret key specific for S3 API calls. Defaults to cloud.aws.secret_key.
          * @see AwsS3Service#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING =
-            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
+        Setting<SecureString> SECRET_SETTING =
+            new Setting<>("cloud.aws.s3.secret_key", AwsS3Service.SECRET_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.s3.protocol: Protocol for AWS API specific for S3 API calls: http or https. Defaults to cloud.aws.protocol.
          * @see AwsS3Service#PROTOCOL_SETTING
          */
         Setting<Protocol> PROTOCOL_SETTING =
             new Setting<>("cloud.aws.s3.protocol", AwsS3Service.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-                Property.NodeScope);
+                Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.s3.proxy.host: In case of proxy, define its hostname/IP specific for S3 API calls. Defaults to cloud.aws.proxy.host.
          * @see AwsS3Service#PROXY_HOST_SETTING
          */
         Setting<String> PROXY_HOST_SETTING =
             new Setting<>("cloud.aws.s3.proxy.host", AwsS3Service.PROXY_HOST_SETTING, Function.identity(),
-                Property.NodeScope);
+                Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.s3.proxy.port: In case of proxy, define its port specific for S3 API calls.  Defaults to cloud.aws.proxy.port.
          * @see AwsS3Service#PROXY_PORT_SETTING
          */
         Setting<Integer> PROXY_PORT_SETTING =
             new Setting<>("cloud.aws.s3.proxy.port", AwsS3Service.PROXY_PORT_SETTING,
-                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope);
+                s -> Setting.parseInt(s, 0, 1<<16, "cloud.aws.s3.proxy.port"), Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.s3.proxy.username: In case of proxy with auth, define the username specific for S3 API calls.
          * Defaults to cloud.aws.proxy.username.
          * @see AwsS3Service#PROXY_USERNAME_SETTING
          */
-        Setting<String> PROXY_USERNAME_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, Function.identity(),
-                Property.NodeScope);
+        Setting<SecureString> PROXY_USERNAME_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.username", AwsS3Service.PROXY_USERNAME_SETTING, SecureString::new,
+                Property.NodeScope, Property.Deprecated);
         /**
          * cloud.aws.s3.proxy.password: In case of proxy with auth, define the password specific for S3 API calls.
          * Defaults to cloud.aws.proxy.password.
          * @see AwsS3Service#PROXY_PASSWORD_SETTING
          */
-        Setting<String> PROXY_PASSWORD_SETTING =
-            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, Function.identity(),
-                Property.NodeScope, Property.Filtered);
+        Setting<SecureString> PROXY_PASSWORD_SETTING =
+            new Setting<>("cloud.aws.s3.proxy.password", AwsS3Service.PROXY_PASSWORD_SETTING, SecureString::new,
+                Property.NodeScope, Property.Filtered, Property.Deprecated);
         /**
          * cloud.aws.s3.signer: If you are using an old AWS API version, you can define a Signer. Specific for S3 API calls.
          * Defaults to cloud.aws.signer.
          * @see AwsS3Service#SIGNER_SETTING
          */
         Setting<String> SIGNER_SETTING =
-            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(), Property.NodeScope);
+            new Setting<>("cloud.aws.s3.signer", AwsS3Service.SIGNER_SETTING, Function.identity(),
+                Property.NodeScope, Property.Deprecated);
         /**
-         * cloud.aws.s3.region: Region specific for S3 API calls. Defaults to cloud.aws.region.
-         * @see AwsS3Service#REGION_SETTING
-         */
-        Setting<String> REGION_SETTING =
-            new Setting<>("cloud.aws.s3.region", AwsS3Service.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT),
-                Property.NodeScope);
-        /**
-         * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
+         * cloud.aws.s3.endpoint: Endpoint.
          */
         Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
         /**
@@ -162,9 +156,8 @@ public interface AwsS3Service extends LifecycleComponent {
          * @see AwsS3Service#READ_TIMEOUT
          */
         Setting<TimeValue> READ_TIMEOUT =
-            Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope);
+            Setting.timeSetting("cloud.aws.s3.read_timeout", AwsS3Service.READ_TIMEOUT, Property.NodeScope, Property.Deprecated);
     }
 
-    AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                    boolean useThrottleRetries, Boolean pathStyleAccess);
+    AmazonS3 client(Settings repositorySettings, Integer maxRetries, boolean useThrottleRetries, Boolean pathStyleAccess);
 }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 25a8ab403d7..cafcb6b98f0 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -19,11 +19,15 @@
 
 package org.elasticsearch.cloud.aws;
 
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.http.IdleConnectionReaper;
 import com.amazonaws.internal.StaticCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
@@ -34,16 +38,18 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.repositories.s3.S3Repository;
 
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.elasticsearch.repositories.s3.S3Repository.getValue;
-
 public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
 
+    // pkg private for tests
+    static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
+
     /**
      * (acceskey, endpoint) -&gt; client
      */
@@ -54,11 +60,12 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
     }
 
     @Override
-    public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                                        boolean useThrottleRetries, Boolean pathStyleAccess) {
-        String foundEndpoint = findEndpoint(logger, settings, endpoint, region);
+    public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
+                                              boolean useThrottleRetries, Boolean pathStyleAccess) {
+        String clientName = CLIENT_NAME.get(repositorySettings);
+        String foundEndpoint = findEndpoint(logger, repositorySettings, settings, clientName);
 
-        AWSCredentialsProvider credentials = buildCredentials(logger, settings, repositorySettings);
+        AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, settings, repositorySettings, clientName);
 
         Tuple<String, String> clientDescriptor = new Tuple<>(foundEndpoint, credentials.getCredentials().getAWSAccessKeyId());
         AmazonS3Client client = clients.get(clientDescriptor);
@@ -68,7 +75,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
 
         client = new AmazonS3Client(
             credentials,
-            buildConfiguration(logger, settings, protocol, maxRetries, foundEndpoint, useThrottleRetries));
+            buildConfiguration(logger, repositorySettings, settings, clientName, maxRetries, foundEndpoint, useThrottleRetries));
 
         if (pathStyleAccess != null) {
             client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
@@ -82,25 +89,34 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
         return client;
     }
 
-    public static ClientConfiguration buildConfiguration(Logger logger, Settings settings, Protocol protocol, Integer maxRetries,
-                                                         String endpoint, boolean useThrottleRetries) {
+    // pkg private for tests
+    static ClientConfiguration buildConfiguration(Logger logger, Settings repositorySettings, Settings settings,
+                                                         String clientName, Integer maxRetries, String endpoint,
+                                                         boolean useThrottleRetries) {
         ClientConfiguration clientConfiguration = new ClientConfiguration();
         // the response metadata cache is only there for diagnostics purposes,
         // but can force objects from every response to the old generation.
         clientConfiguration.setResponseMetadataCacheSize(0);
+        Protocol protocol = getConfigValue(repositorySettings, settings, clientName, S3Repository.PROTOCOL_SETTING,
+                                           S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
         clientConfiguration.setProtocol(protocol);
 
-        String proxyHost = CLOUD_S3.PROXY_HOST_SETTING.get(settings);
+        String proxyHost = getConfigValue(null, settings, clientName,
+                                          S3Repository.PROXY_HOST_SETTING, null, CLOUD_S3.PROXY_HOST_SETTING);
         if (Strings.hasText(proxyHost)) {
-            Integer proxyPort = CLOUD_S3.PROXY_PORT_SETTING.get(settings);
-            String proxyUsername = CLOUD_S3.PROXY_USERNAME_SETTING.get(settings);
-            String proxyPassword = CLOUD_S3.PROXY_PASSWORD_SETTING.get(settings);
+            Integer proxyPort = getConfigValue(null, settings, clientName,
+                                               S3Repository.PROXY_PORT_SETTING, null, CLOUD_S3.PROXY_PORT_SETTING);
+            try (SecureString proxyUsername = getConfigValue(null, settings, clientName,
+                                                             S3Repository.PROXY_USERNAME_SETTING, null, CLOUD_S3.PROXY_USERNAME_SETTING);
+                 SecureString proxyPassword = getConfigValue(null, settings, clientName,
+                                                             S3Repository.PROXY_PASSWORD_SETTING, null, CLOUD_S3.PROXY_PASSWORD_SETTING)) {
 
-            clientConfiguration
-                .withProxyHost(proxyHost)
-                .withProxyPort(proxyPort)
-                .withProxyUsername(proxyUsername)
-                .withProxyPassword(proxyPassword);
+                clientConfiguration
+                    .withProxyHost(proxyHost)
+                    .withProxyPort(proxyPort)
+                    .withProxyUsername(proxyUsername.toString())
+                    .withProxyPassword(proxyPassword.toString());
+            }
         }
 
         if (maxRetries != null) {
@@ -116,45 +132,43 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
             AwsSigner.configureSigner(awsSigner, clientConfiguration, endpoint);
         }
 
-        clientConfiguration.setSocketTimeout((int) CLOUD_S3.READ_TIMEOUT.get(settings).millis());
+        TimeValue readTimeout = getConfigValue(null, settings, clientName,
+                                               S3Repository.READ_TIMEOUT_SETTING, null, CLOUD_S3.READ_TIMEOUT);
+        clientConfiguration.setSocketTimeout((int)readTimeout.millis());
 
         return clientConfiguration;
     }
 
-    public static AWSCredentialsProvider buildCredentials(Logger logger, Settings settings, Settings repositorySettings) {
+    public static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger,
+                                                          Settings settings, Settings repositorySettings, String clientName) {
         AWSCredentialsProvider credentials;
-        String key = getValue(repositorySettings, settings,
-            S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
-        String secret = getValue(repositorySettings, settings,
-            S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING);
+        try (SecureString key = getConfigValue(repositorySettings, settings, clientName, S3Repository.ACCESS_KEY_SETTING,
+                                               S3Repository.Repository.KEY_SETTING, S3Repository.Repositories.KEY_SETTING);
+             SecureString secret = getConfigValue(repositorySettings, settings, clientName, S3Repository.SECRET_KEY_SETTING,
+                                                  S3Repository.Repository.SECRET_SETTING, S3Repository.Repositories.SECRET_SETTING)) {
 
-        if (key.isEmpty() && secret.isEmpty()) {
-            logger.debug("Using either environment variables, system properties or instance profile credentials");
-            credentials = new DefaultAWSCredentialsProviderChain();
-        } else {
-            logger.debug("Using basic key/secret credentials");
-            credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key, secret));
+            if (key.length() == 0 && secret.length() == 0) {
+                logger.debug("Using instance profile credentials");
+                credentials = new InstanceProfileCredentialsProvider();
+            } else {
+                logger.debug("Using basic key/secret credentials");
+                credentials = new StaticCredentialsProvider(new BasicAWSCredentials(key.toString(), secret.toString()));
+            }
         }
 
         return credentials;
     }
 
-    protected static String findEndpoint(Logger logger, Settings settings, String endpoint, String region) {
+    // pkg private for tests
+    /** Returns the endpoint the client should use, based on the available endpoint settings found. */
+    static String findEndpoint(Logger logger, Settings repositorySettings, Settings settings, String clientName) {
+        String endpoint = getConfigValue(repositorySettings, settings, clientName, S3Repository.ENDPOINT_SETTING,
+                                         S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
         if (Strings.isNullOrEmpty(endpoint)) {
-            logger.debug("no repository level endpoint has been defined. Trying to guess from repository region [{}]", region);
-            if (!region.isEmpty()) {
-                endpoint = getEndpoint(region);
-                logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
-            } else {
-                // No region has been set so we will use the default endpoint
-                if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) {
-                    endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings);
-                    logger.debug("using explicit s3 endpoint [{}]", endpoint);
-                } else if (REGION_SETTING.exists(settings) || CLOUD_S3.REGION_SETTING.exists(settings)) {
-                    region = CLOUD_S3.REGION_SETTING.get(settings);
-                    endpoint = getEndpoint(region);
-                    logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
-                }
+            // No region has been set so we will use the default endpoint
+            if (CLOUD_S3.ENDPOINT_SETTING.exists(settings)) {
+                endpoint = CLOUD_S3.ENDPOINT_SETTING.get(settings);
+                logger.debug("using explicit s3 endpoint [{}]", endpoint);
             }
         } else {
             logger.debug("using repository level endpoint [{}]", endpoint);
@@ -163,66 +177,21 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
         return endpoint;
     }
 
-    private static String getEndpoint(String region) {
-        final String endpoint;
-        switch (region) {
-            case "us-east":
-            case "us-east-1":
-                endpoint = "s3.amazonaws.com";
-                break;
-            case "us-east-2":
-                endpoint = "s3.us-east-2.amazonaws.com";
-                break;
-            case "us-west":
-            case "us-west-1":
-                endpoint = "s3-us-west-1.amazonaws.com";
-                break;
-            case "us-west-2":
-                endpoint = "s3-us-west-2.amazonaws.com";
-                break;
-            case "ap-south":
-            case "ap-south-1":
-                endpoint = "s3-ap-south-1.amazonaws.com";
-                break;
-            case "ap-southeast":
-            case "ap-southeast-1":
-                endpoint = "s3-ap-southeast-1.amazonaws.com";
-                break;
-            case "ap-southeast-2":
-                endpoint = "s3-ap-southeast-2.amazonaws.com";
-                break;
-            case "ap-northeast":
-            case "ap-northeast-1":
-                endpoint = "s3-ap-northeast-1.amazonaws.com";
-                break;
-            case "ap-northeast-2":
-                endpoint = "s3-ap-northeast-2.amazonaws.com";
-                break;
-            case "eu-west":
-            case "eu-west-1":
-                endpoint = "s3-eu-west-1.amazonaws.com";
-                break;
-            case "eu-central":
-            case "eu-central-1":
-                endpoint = "s3.eu-central-1.amazonaws.com";
-                break;
-            case "sa-east":
-            case "sa-east-1":
-                endpoint = "s3-sa-east-1.amazonaws.com";
-                break;
-            case "cn-north":
-            case "cn-north-1":
-                endpoint = "s3.cn-north-1.amazonaws.com.cn";
-                break;
-            case "us-gov-west":
-            case "us-gov-west-1":
-                endpoint = "s3-us-gov-west-1.amazonaws.com";
-                break;
-            default:
-                throw new IllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
+    /**
+     * Find the setting value, trying first with named configs,
+     * then falling back to repository and global repositories settings.
+     */
+    private static <T> T getConfigValue(Settings repositorySettings, Settings globalSettings, String clientName,
+                                        Setting.AffixSetting<T> configSetting, Setting<T> repositorySetting, Setting<T> globalSetting) {
+        Setting<T> concreteSetting = configSetting.getConcreteSettingForNamespace(clientName);
+        if (concreteSetting.exists(globalSettings)) {
+            return concreteSetting.get(globalSettings);
+        } else if (repositorySetting == null) {
+            // no repository setting, just use global setting
+            return globalSetting.get(globalSettings);
+        } else {
+            return S3Repository.getValue(repositorySettings, globalSettings, repositorySetting, globalSetting);
         }
-
-        return endpoint;
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index e73a3909d8d..1b942d65082 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -20,13 +20,14 @@
 package org.elasticsearch.cloud.aws.blobstore;
 
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.CopyObjectRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cloud.aws.util.SocketAccess;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -42,8 +43,7 @@ import java.io.OutputStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.NoSuchFileException;
 import java.security.AccessController;
-import java.security.PrivilegedActionException;
-import java.security.PrivilegedExceptionAction;
+import java.security.PrivilegedAction;
 import java.util.Map;
 
 public class S3BlobContainer extends AbstractBlobContainer {
@@ -61,13 +61,9 @@ public class S3BlobContainer extends AbstractBlobContainer {
     @Override
     public boolean blobExists(String blobName) {
         try {
-            return doPrivileged(() -> {
-                try {
-                    blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName));
-                    return true;
-                } catch (AmazonS3Exception e) {
-                    return false;
-                }
+            return SocketAccess.doPrivileged(() -> {
+                blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName));
+                return true;
             });
         } catch (AmazonS3Exception e) {
             return false;
@@ -81,7 +77,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
         int retry = 0;
         while (retry <= blobStore.numberOfRetries()) {
             try {
-                S3Object s3Object = blobStore.client().getObject(blobStore.bucket(), buildKey(blobName));
+                S3Object s3Object = SocketAccess.doPrivileged(() -> blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)));
                 return s3Object.getObjectContent();
             } catch (AmazonClientException e) {
                 if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) {
@@ -96,7 +92,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
                 }
             }
         }
-        throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() +"]");
+        throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() + "]");
     }
 
     @Override
@@ -105,7 +101,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
             throw new FileAlreadyExistsException("blob [" + blobName + "] already exists, cannot overwrite");
         }
         try (OutputStream stream = createOutput(blobName)) {
-            Streams.copy(inputStream, stream);
+            SocketAccess.doPrivilegedIOException(() -> Streams.copy(inputStream, stream));
         }
     }
 
@@ -116,7 +112,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
         }
 
         try {
-            blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName));
+            SocketAccess.doPrivilegedVoid(() -> blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)));
         } catch (AmazonClientException e) {
             throw new IOException("Exception when deleting blob [" + blobName + "]", e);
         }
@@ -125,51 +121,60 @@ public class S3BlobContainer extends AbstractBlobContainer {
     private OutputStream createOutput(final String blobName) throws IOException {
         // UploadS3OutputStream does buffering & retry logic internally
         return new DefaultS3OutputStream(blobStore, blobStore.bucket(), buildKey(blobName),
-                blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption());
+            blobStore.bufferSizeInBytes(), blobStore.numberOfRetries(), blobStore.serverSideEncryption());
     }
 
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
-        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
-        ObjectListing prevListing = null;
-        while (true) {
-            ObjectListing list;
-            if (prevListing != null) {
-                list = blobStore.client().listNextBatchOfObjects(prevListing);
-            } else {
-                if (blobNamePrefix != null) {
-                    list = blobStore.client().listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
-                } else {
-                    list = blobStore.client().listObjects(blobStore.bucket(), keyPath);
+        return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
+            MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
+            AmazonS3 client = blobStore.client();
+            SocketAccess.doPrivilegedVoid(() -> {
+                ObjectListing prevListing = null;
+                while (true) {
+                    ObjectListing list;
+                    if (prevListing != null) {
+                        list = client.listNextBatchOfObjects(prevListing);
+                    } else {
+                        if (blobNamePrefix != null) {
+                            list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
+                        } else {
+                            list = client.listObjects(blobStore.bucket(), keyPath);
+                        }
+                    }
+                    for (S3ObjectSummary summary : list.getObjectSummaries()) {
+                        String name = summary.getKey().substring(keyPath.length());
+                        blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
+                    }
+                    if (list.isTruncated()) {
+                        prevListing = list;
+                    } else {
+                        break;
+                    }
                 }
-            }
-            for (S3ObjectSummary summary : list.getObjectSummaries()) {
-                String name = summary.getKey().substring(keyPath.length());
-                blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
-            }
-            if (list.isTruncated()) {
-                prevListing = list;
-            } else {
-                break;
-            }
-        }
-        return blobsBuilder.immutableMap();
+            });
+            return blobsBuilder.immutableMap();
+        });
     }
 
     @Override
     public void move(String sourceBlobName, String targetBlobName) throws IOException {
         try {
             CopyObjectRequest request = new CopyObjectRequest(blobStore.bucket(), buildKey(sourceBlobName),
-                    blobStore.bucket(), buildKey(targetBlobName));
+                blobStore.bucket(), buildKey(targetBlobName));
 
             if (blobStore.serverSideEncryption()) {
                 ObjectMetadata objectMetadata = new ObjectMetadata();
                 objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
                 request.setNewObjectMetadata(objectMetadata);
             }
-            blobStore.client().copyObject(request);
-            blobStore.client().deleteObject(blobStore.bucket(), buildKey(sourceBlobName));
-        } catch (AmazonS3Exception e){
+
+            SocketAccess.doPrivilegedVoid(() -> {
+                blobStore.client().copyObject(request);
+                blobStore.client().deleteObject(blobStore.bucket(), buildKey(sourceBlobName));
+            });
+
+        } catch (AmazonS3Exception e) {
             throw new IOException(e);
         }
     }
@@ -182,20 +187,4 @@ public class S3BlobContainer extends AbstractBlobContainer {
     protected String buildKey(String blobName) {
         return keyPath + blobName;
     }
-
-    /**
-     * +     * Executes a {@link PrivilegedExceptionAction} with privileges enabled.
-     * +
-     */
-    <T> T doPrivileged(PrivilegedExceptionAction<T> operation) throws IOException {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        try {
-            return AccessController.doPrivileged(operation);
-        } catch (PrivilegedActionException e) {
-            throw (IOException) e.getException();
-        }
-    }
 }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
index 6a03b4bcea5..23f5605035f 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
@@ -29,6 +29,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
+import org.elasticsearch.cloud.aws.util.SocketAccess;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -38,6 +39,8 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Locale;
 
@@ -47,8 +50,6 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
 
     private final String bucket;
 
-    private final String region;
-
     private final ByteSizeValue bufferSize;
 
     private final boolean serverSideEncryption;
@@ -59,12 +60,11 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
 
     private final StorageClass storageClass;
 
-    public S3BlobStore(Settings settings, AmazonS3 client, String bucket, @Nullable String region, boolean serverSideEncryption,
+    public S3BlobStore(Settings settings, AmazonS3 client, String bucket, boolean serverSideEncryption,
                        ByteSizeValue bufferSize, int maxRetries, String cannedACL, String storageClass) {
         super(settings);
         this.client = client;
         this.bucket = bucket;
-        this.region = region;
         this.serverSideEncryption = serverSideEncryption;
         this.bufferSize = bufferSize;
         this.cannedACL = initCannedACL(cannedACL);
@@ -76,34 +76,17 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
         // Also, if invalid security credentials are used to execute this method, the
         // client is not able to distinguish between bucket permission errors and
         // invalid credential errors, and this method could return an incorrect result.
-        int retry = 0;
-        while (retry <= maxRetries) {
-            try {
-                if (!client.doesBucketExist(bucket)) {
-                    CreateBucketRequest request = null;
-                    if (region != null) {
-                        request = new CreateBucketRequest(bucket, region);
-                    } else {
-                        request = new CreateBucketRequest(bucket);
-                    }
-                    request.setCannedAcl(this.cannedACL);
-                    client.createBucket(request);
-                }
-                break;
-            } catch (AmazonClientException e) {
-                if (shouldRetry(e) && retry < maxRetries) {
-                    retry++;
-                } else {
-                    logger.debug("S3 client create bucket failed");
-                    throw e;
-                }
+        SocketAccess.doPrivilegedVoid(() -> {
+            if (client.doesBucketExist(bucket) == false) {
+                throw new IllegalArgumentException("The bucket [" + bucket + "] does not exist. Please create it before " +
+                                                   " creating an s3 snapshot repository backed by it.");
             }
-        }
+        });
     }
 
     @Override
     public String toString() {
-        return (region == null ? "" : region + "/") + bucket;
+        return bucket;
     }
 
     public AmazonS3 client() {
@@ -114,7 +97,9 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
         return bucket;
     }
 
-    public boolean serverSideEncryption() { return serverSideEncryption; }
+    public boolean serverSideEncryption() {
+        return serverSideEncryption;
+    }
 
     public int bufferSizeInBytes() {
         return bufferSize.bytesAsInt();
@@ -131,45 +116,48 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
 
     @Override
     public void delete(BlobPath path) {
-        ObjectListing prevListing = null;
-        //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
-        //we can do at most 1K objects per delete
-        //We don't know the bucket name until first object listing
-        DeleteObjectsRequest multiObjectDeleteRequest = null;
-        ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
-        while (true) {
-            ObjectListing list;
-            if (prevListing != null) {
-                list = client.listNextBatchOfObjects(prevListing);
-            } else {
-                list = client.listObjects(bucket, path.buildAsString());
-                multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
-            }
-            for (S3ObjectSummary summary : list.getObjectSummaries()) {
-                keys.add(new KeyVersion(summary.getKey()));
-                //Every 500 objects batch the delete request
-                if (keys.size() > 500) {
-                    multiObjectDeleteRequest.setKeys(keys);
-                    client.deleteObjects(multiObjectDeleteRequest);
+        AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
+            ObjectListing prevListing = null;
+            //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
+            //we can do at most 1K objects per delete
+            //We don't know the bucket name until first object listing
+            DeleteObjectsRequest multiObjectDeleteRequest = null;
+            ArrayList<KeyVersion> keys = new ArrayList<KeyVersion>();
+            while (true) {
+                ObjectListing list;
+                if (prevListing != null) {
+                    list = client.listNextBatchOfObjects(prevListing);
+                } else {
+                    list = client.listObjects(bucket, path.buildAsString());
                     multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
-                    keys.clear();
+                }
+                for (S3ObjectSummary summary : list.getObjectSummaries()) {
+                    keys.add(new KeyVersion(summary.getKey()));
+                    //Every 500 objects batch the delete request
+                    if (keys.size() > 500) {
+                        multiObjectDeleteRequest.setKeys(keys);
+                        client.deleteObjects(multiObjectDeleteRequest);
+                        multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
+                        keys.clear();
+                    }
+                }
+                if (list.isTruncated()) {
+                    prevListing = list;
+                } else {
+                    break;
                 }
             }
-            if (list.isTruncated()) {
-                prevListing = list;
-            } else {
-                break;
+            if (!keys.isEmpty()) {
+                multiObjectDeleteRequest.setKeys(keys);
+                client.deleteObjects(multiObjectDeleteRequest);
             }
-        }
-        if (!keys.isEmpty()) {
-            multiObjectDeleteRequest.setKeys(keys);
-            client.deleteObjects(multiObjectDeleteRequest);
-        }
+            return null;
+        });
     }
 
     protected boolean shouldRetry(AmazonClientException e) {
         if (e instanceof AmazonS3Exception) {
-            AmazonS3Exception s3e = (AmazonS3Exception)e;
+            AmazonS3Exception s3e = (AmazonS3Exception) e;
             if (s3e.getStatusCode() == 400 && "RequestTimeout".equals(s3e.getErrorCode())) {
                 return true;
             }
@@ -194,7 +182,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
 
         try {
             StorageClass _storageClass = StorageClass.fromValue(storageClass.toUpperCase(Locale.ENGLISH));
-            if(_storageClass.equals(StorageClass.Glacier)) {
+            if (_storageClass.equals(StorageClass.Glacier)) {
                 throw new BlobStoreException("Glacier storage class is not supported");
             }
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java
new file mode 100644
index 00000000000..91a3190d4b9
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/util/SocketAccess.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.aws.util;
+
+import org.elasticsearch.SpecialPermission;
+
+import java.io.IOException;
+import java.net.SocketPermission;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+
+/**
+ * This plugin uses aws libraries to connect to S3 repositories. For these remote calls the plugin needs
+ * {@link SocketPermission} 'connect' to establish connections. This class wraps the operations requiring access in
+ * {@link AccessController#doPrivileged(PrivilegedAction)} blocks.
+ */
+public final class SocketAccess {
+
+    private SocketAccess() {}
+
+    public static <T> T doPrivileged(PrivilegedAction<T> operation) {
+        SpecialPermission.check();
+        return AccessController.doPrivileged(operation);
+    }
+
+    public static <T> T doPrivilegedIOException(PrivilegedExceptionAction<T> operation) throws IOException {
+        SpecialPermission.check();
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getCause();
+        }
+    }
+
+    public static void doPrivilegedVoid(Runnable action) {
+        SpecialPermission.check();
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            action.run();
+            return null;
+        });
+    }
+
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 214d057d2f6..9a2e15dd9b4 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cloud.aws.InternalAwsS3Service;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
@@ -44,24 +45,18 @@ import org.elasticsearch.repositories.s3.S3Repository;
 public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
 
     static {
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                try {
-                    // kick jackson to do some static caching of declared members info
-                    Jackson.jsonNodeOf("{}");
-                    // ClientConfiguration clinit has some classloader problems
-                    // TODO: fix that
-                    Class.forName("com.amazonaws.ClientConfiguration");
-                } catch (ClassNotFoundException e) {
-                    throw new RuntimeException(e);
-                }
-                return null;
+        SpecialPermission.check();
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            try {
+                // kick jackson to do some static caching of declared members info
+                Jackson.jsonNodeOf("{}");
+                // ClientConfiguration clinit has some classloader problems
+                // TODO: fix that
+                Class.forName("com.amazonaws.ClientConfiguration");
+            } catch (ClassNotFoundException e) {
+                throw new RuntimeException(e);
             }
+            return null;
         });
     }
 
@@ -71,9 +66,9 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
     }
 
     @Override
-    public Map<String, Repository.Factory> getRepositories(Environment env) {
+    public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
         return Collections.singletonMap(S3Repository.TYPE,
-            (metadata) -> new S3Repository(metadata, env.settings(), createStorageService(env.settings())));
+            (metadata) -> new S3Repository(metadata, env.settings(), namedXContentRegistry, createStorageService(env.settings())));
     }
 
     @Override
@@ -95,7 +90,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
         AwsS3Service.PROXY_USERNAME_SETTING,
         AwsS3Service.PROXY_PASSWORD_SETTING,
         AwsS3Service.SIGNER_SETTING,
-        AwsS3Service.REGION_SETTING,
         AwsS3Service.READ_TIMEOUT,
 
         // Register S3 specific settings: cloud.aws.s3
@@ -107,7 +101,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
         AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
         AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
         AwsS3Service.CLOUD_S3.SIGNER_SETTING,
-        AwsS3Service.CLOUD_S3.REGION_SETTING,
         AwsS3Service.CLOUD_S3.ENDPOINT_SETTING,
         AwsS3Service.CLOUD_S3.READ_TIMEOUT,
 
@@ -115,7 +108,6 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
         S3Repository.Repositories.KEY_SETTING,
         S3Repository.Repositories.SECRET_SETTING,
         S3Repository.Repositories.BUCKET_SETTING,
-        S3Repository.Repositories.REGION_SETTING,
         S3Repository.Repositories.ENDPOINT_SETTING,
         S3Repository.Repositories.PROTOCOL_SETTING,
         S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING,
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 7310b527158..ab83f72c676 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -21,25 +21,31 @@ package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
+import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
+import org.elasticsearch.cloud.aws.InternalAwsS3Service;
 import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.AffixSetting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
 import java.util.Locale;
-import java.util.function.Function;
 
 /**
  * Shared file system implementation of the BlobStoreRepository
@@ -47,7 +53,6 @@ import java.util.function.Function;
  * Shared file system repository supports the following settings
  * <dl>
  * <dt>{@code bucket}</dt><dd>S3 bucket</dd>
- * <dt>{@code region}</dt><dd>S3 region. Defaults to us-east</dd>
  * <dt>{@code base_path}</dt><dd>Specifies the path within bucket to repository data. Defaults to root directory.</dd>
  * <dt>{@code concurrent_streams}</dt><dd>Number of concurrent read/write stream (per repository on each node). Defaults to 5.</dd>
  * <dt>{@code chunk_size}</dt><dd>Large file can be divided into chunks. This parameter specifies the chunk size. Defaults to not chucked.</dd>
@@ -58,40 +63,76 @@ public class S3Repository extends BlobStoreRepository {
 
     public static final String TYPE = "s3";
 
+    // prefix for s3 client settings
+    private static final String PREFIX = "s3.client.";
+
+    /** The access key (ie login id) for connecting to s3. */
+    public static final AffixSetting<SecureString> ACCESS_KEY_SETTING = Setting.affixKeySetting(PREFIX, "access_key",
+        key -> SecureSetting.secureString(key, Repositories.KEY_SETTING, false));
+
+    /** The secret key (ie password) for connecting to s3. */
+    public static final AffixSetting<SecureString> SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key",
+        key -> SecureSetting.secureString(key, Repositories.SECRET_SETTING, false));
+
+    /** An override for the s3 endpoint to connect to. */
+    public static final AffixSetting<String> ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint",
+        key -> new Setting<>(key, Repositories.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope));
+
+    /** The protocol to use to connec to to s3. */
+    public static final AffixSetting<Protocol> PROTOCOL_SETTING = Setting.affixKeySetting(PREFIX, "protocol",
+        key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
+
+    /** The host name of a proxy to connect to s3 through. */
+    public static final AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
+        key -> Setting.simpleString(key, Property.NodeScope));
+
+    /** The port of a proxy to connect to s3 through. */
+    public static final AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
+        key -> Setting.intSetting(key, 80, 0, 1<<16, Property.NodeScope));
+
+    /** The username of a proxy to connect to s3 through. */
+    public static final AffixSetting<SecureString> PROXY_USERNAME_SETTING = Setting.affixKeySetting(PREFIX, "proxy.username",
+        key -> SecureSetting.secureString(key, AwsS3Service.PROXY_USERNAME_SETTING, false));
+
+    /** The password of a proxy to connect to s3 through. */
+    public static final AffixSetting<SecureString> PROXY_PASSWORD_SETTING = Setting.affixKeySetting(PREFIX, "proxy.password",
+        key -> SecureSetting.secureString(key, AwsS3Service.PROXY_PASSWORD_SETTING, false));
+
+    /** The socket timeout for connecting to s3. */
+    public static final AffixSetting<TimeValue> READ_TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "read_timeout",
+        key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope));
+
     /**
      * Global S3 repositories settings. Starting with: repositories.s3
+     * NOTE: These are legacy settings. Use the named client config settings above.
      */
     public interface Repositories {
         /**
          * repositories.s3.access_key: AWS Access key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.access_key.
          * @see CLOUD_S3#KEY_SETTING
          */
-        Setting<String> KEY_SETTING =
-            new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
+        Setting<SecureString> KEY_SETTING = new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
          * @see CLOUD_S3#SECRET_SETTING
          */
-        Setting<String> SECRET_SETTING =
-            new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
-        /**
-         * repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
-         * @see CLOUD_S3#REGION_SETTING
-         */
-        Setting<String> REGION_SETTING =
-            new Setting<>("repositories.s3.region", CLOUD_S3.REGION_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        Setting<SecureString> SECRET_SETTING = new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, SecureString::new,
+            Property.NodeScope, Property.Filtered, Property.Deprecated);
+
         /**
          * repositories.s3.endpoint: Endpoint specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.endpoint.
          * @see CLOUD_S3#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING =
-            new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING, s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        Setting<String> ENDPOINT_SETTING = new Setting<>("repositories.s3.endpoint", CLOUD_S3.ENDPOINT_SETTING,
+            s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.protocol: Protocol specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.protocol.
          * @see CLOUD_S3#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING =
-            new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING, s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope);
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("repositories.s3.protocol", CLOUD_S3.PROTOCOL_SETTING,
+            s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope, Property.Deprecated);
         /**
          * repositories.s3.bucket: The name of the bucket to be used for snapshots.
          */
@@ -174,36 +215,25 @@ public class S3Repository extends BlobStoreRepository {
      * If undefined, they use the repositories.s3.xxx equivalent setting.
      */
     public interface Repository {
-        /**
-         * access_key
-         * @see  Repositories#KEY_SETTING
-         */
-        Setting<String> KEY_SETTING = Setting.simpleString("access_key");
-        /**
-         * secret_key
-         * @see  Repositories#SECRET_SETTING
-         */
-        Setting<String> SECRET_SETTING = Setting.simpleString("secret_key");
-        /**
-         * bucket
-         * @see  Repositories#BUCKET_SETTING
-         */
+        Setting<SecureString> KEY_SETTING = new Setting<>("access_key", "", SecureString::new,
+            Property.Filtered, Property.Deprecated);
+
+
+        Setting<SecureString> SECRET_SETTING = new Setting<>("secret_key", "", SecureString::new,
+            Property.Filtered, Property.Deprecated);
+
         Setting<String> BUCKET_SETTING = Setting.simpleString("bucket");
         /**
          * endpoint
          * @see  Repositories#ENDPOINT_SETTING
          */
-        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint");
+        Setting<String> ENDPOINT_SETTING = Setting.simpleString("endpoint", Property.Deprecated);
         /**
          * protocol
          * @see  Repositories#PROTOCOL_SETTING
          */
-        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)));
-        /**
-         * region
-         * @see  Repositories#REGION_SETTING
-         */
-        Setting<String> REGION_SETTING = new Setting<>("region", "", s -> s.toLowerCase(Locale.ROOT));
+        Setting<Protocol> PROTOCOL_SETTING = new Setting<>("protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
+            Property.Deprecated);
         /**
          * server_side_encryption
          * @see  Repositories#SERVER_SIDE_ENCRYPTION_SETTING
@@ -273,18 +303,15 @@ public class S3Repository extends BlobStoreRepository {
     /**
      * Constructs an s3 backed repository
      */
-    public S3Repository(RepositoryMetaData metadata, Settings settings, AwsS3Service s3Service) throws IOException {
-        super(metadata, settings);
+    public S3Repository(RepositoryMetaData metadata, Settings settings,
+                        NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) throws IOException {
+        super(metadata, settings, namedXContentRegistry);
 
         String bucket = getValue(metadata.settings(), settings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING);
         if (bucket == null) {
             throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
         }
 
-        String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
-        Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
-        String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
-
         boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
         ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
         Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
@@ -310,14 +337,13 @@ public class S3Repository extends BlobStoreRepository {
             pathStyleAccess = getValue(metadata.settings(), settings, Repository.PATH_STYLE_ACCESS_SETTING, Repositories.PATH_STYLE_ACCESS_SETTING);
         }
 
-        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
+        logger.debug("using bucket [{}], chunk_size [{}], server_side_encryption [{}], " +
             "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}], path_style_access [{}]",
-            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
+            bucket, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
             storageClass, pathStyleAccess);
 
-        blobStore = new S3BlobStore(settings,
-            s3Service.client(metadata.settings(), endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess),
-                bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
+        AmazonS3 client = s3Service.client(metadata.settings(), maxRetries, useThrottleRetries, pathStyleAccess);
+        blobStore = new S3BlobStore(settings, client, bucket, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
         String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
         if (Strings.hasLength(basePath)) {
diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
index 1f09cada2e5..d8fca1fc899 100644
--- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
@@ -34,4 +34,7 @@ grant {
   // TODO: get these fixed in aws sdk
   // See https://github.com/aws/aws-sdk-java/issues/766
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+
+  // s3 client opens socket connections for to access repository
+  permission java.net.SocketPermission "*", "connect";
 };
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
index 2bd14847f7e..e11dade7953 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AwsS3ServiceImplTests.java
@@ -23,7 +23,8 @@ import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.s3.S3Repository;
 import org.elasticsearch.test.ESTestCase;
@@ -35,61 +36,89 @@ import static org.hamcrest.Matchers.is;
 public class AwsS3ServiceImplTests extends ESTestCase {
 
     public void testAWSCredentialsWithSystemProviders() {
-        AWSCredentialsProvider credentialsProvider = InternalAwsS3Service.buildCredentials(logger, Settings.EMPTY, Settings.EMPTY);
-        assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
+        AWSCredentialsProvider credentialsProvider =
+            InternalAwsS3Service.buildCredentials(logger, deprecationLogger, Settings.EMPTY, Settings.EMPTY, "default");
+        assertThat(credentialsProvider, instanceOf(InstanceProfileCredentialsProvider.class));
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        Settings settings = Settings.builder()
-            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
-            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
-            .build();
+    public void testAwsCredsDefaultSettings() {
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.access_key", "aws_key");
+        secureSettings.setString("s3.client.default.secret_key", "aws_secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "aws_key", "aws_secret");
+    }
+
+    public void testAwsCredsExplicitConfigSettings() {
+        Settings repositorySettings = Settings.builder().put(InternalAwsS3Service.CLIENT_NAME.getKey(), "myconfig").build();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.myconfig.access_key", "aws_key");
+        secureSettings.setString("s3.client.myconfig.secret_key", "aws_secret");
+        secureSettings.setString("s3.client.default.access_key", "wrong_key");
+        secureSettings.setString("s3.client.default.secret_key", "wrong_secret");
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
         launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "aws_key", "aws_secret");
     }
 
-    public void testAWSCredentialsWithElasticsearchS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsSettingsBackcompat() {
+        Settings settings = Settings.builder()
+            .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
+            .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
+            .build();
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "aws_key", "aws_secret");
+        assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated");
+    }
+
+    public void testAWSCredentialsWithElasticsearchS3SettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
             .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
             .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "s3_key", "s3_secret");
+        assertWarnings("[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsAndS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsAndS3SettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
             .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
             .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3_key")
             .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3_secret")
             .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "s3_key", "s3_secret");
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "s3_key", "s3_secret");
+        assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
+        assertWarnings("[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
             .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
             .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
+        assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
             .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@@ -98,20 +127,28 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
-        launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repositories_key", "repositories_secret");
+        launchAWSCredentialsWithElasticsearchSettingsTest(Settings.EMPTY, settings, "repositories_key", "repositories_secret");
+        assertWarnings("[" + AwsS3Service.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SECRET_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repositories.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repositories.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
         Settings settings = Settings.builder()
             .put(S3Repository.Repositories.KEY_SETTING.getKey(), "repositories_key")
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
         launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertWarnings("[" + S3Repository.Repository.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repository.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsAndRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
         Settings settings = Settings.builder()
             .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
             .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@@ -119,10 +156,12 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
         launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertWarnings("[" + S3Repository.Repository.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repository.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettings() {
-        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null);
+    public void testAWSCredentialsWithElasticsearchAwsAndS3AndRepositoriesSettingsAndRepositorySettingsBackcompat() {
+        Settings repositorySettings = generateRepositorySettings("repository_key", "repository_secret", null, null);
         Settings settings = Settings.builder()
             .put(AwsS3Service.KEY_SETTING.getKey(), "aws_key")
             .put(AwsS3Service.SECRET_SETTING.getKey(), "aws_secret")
@@ -132,23 +171,40 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(S3Repository.Repositories.SECRET_SETTING.getKey(), "repositories_secret")
             .build();
         launchAWSCredentialsWithElasticsearchSettingsTest(repositorySettings, settings, "repository_key", "repository_secret");
+        assertWarnings("[" + S3Repository.Repository.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + S3Repository.Repository.SECRET_SETTING.getKey() + "] setting was deprecated");
     }
 
     protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings singleRepositorySettings, Settings settings,
                                                                      String expectedKey, String expectedSecret) {
-        AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, settings, singleRepositorySettings).getCredentials();
+        String configName = InternalAwsS3Service.CLIENT_NAME.get(singleRepositorySettings);
+        AWSCredentials credentials = InternalAwsS3Service.buildCredentials(logger, deprecationLogger, settings,
+            singleRepositorySettings, configName).getCredentials();
         assertThat(credentials.getAWSAccessKeyId(), is(expectedKey));
         assertThat(credentials.getAWSSecretKey(), is(expectedSecret));
     }
 
     public void testAWSDefaultConfiguration() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
-        launchAWSConfigurationTest(Settings.EMPTY, repositorySettings, Protocol.HTTPS, null, -1, null, null, null, 3, false,
+        launchAWSConfigurationTest(Settings.EMPTY, Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, null, 3, false,
             ClientConfiguration.DEFAULT_SOCKET_TIMEOUT);
     }
 
     public void testAWSConfigurationWithAwsSettings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
+        secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
+        Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("s3.client.default.protocol", "http")
+            .put("s3.client.default.proxy.host", "aws_proxy_host")
+            .put("s3.client.default.proxy.port", 8080)
+            .put("s3.client.default.read_timeout", "10s")
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
+            "aws_proxy_password", null, 3, false, 10000);
+    }
+
+    public void testAWSConfigurationWithAwsSettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
             .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
@@ -158,12 +214,18 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
             .put(AwsS3Service.READ_TIMEOUT.getKey(), "10s")
             .build();
-        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username",
             "aws_proxy_password", "AWS3SignerType", 3, false, 10000);
+        assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated");
     }
 
-    public void testAWSConfigurationWithAwsAndS3Settings() {
-        Settings repositorySettings = generateRepositorySettings(null, null, "eu-central", null, null);
+    public void testAWSConfigurationWithAwsAndS3SettingsBackcompat() {
         Settings settings = Settings.builder()
             .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "http")
             .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "aws_proxy_host")
@@ -171,6 +233,7 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "aws_proxy_username")
             .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "aws_proxy_password")
             .put(AwsS3Service.SIGNER_SETTING.getKey(), "AWS3SignerType")
+            .put(AwsS3Service.READ_TIMEOUT.getKey(), "5s")
             .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "https")
             .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3_proxy_host")
             .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 8081)
@@ -179,8 +242,39 @@ public class AwsS3ServiceImplTests extends ESTestCase {
             .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "NoOpSignerType")
             .put(AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey(), "10s")
             .build();
-        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, "s3_proxy_host", 8081, "s3_proxy_username",
             "s3_proxy_password", "NoOpSignerType", 3, false, 10000);
+        assertWarnings("[" + AwsS3Service.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.SIGNER_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.READ_TIMEOUT.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey() + "] setting was deprecated",
+                       "[" + AwsS3Service.CLOUD_S3.READ_TIMEOUT.getKey() + "] setting was deprecated");
+    }
+
+    public void testGlobalMaxRetries() {
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
+            .build();
+        launchAWSConfigurationTest(settings, Settings.EMPTY, Protocol.HTTPS, null, -1, null,
+            null, null, 10, false, 50000);
+    }
+
+    public void testRepositoryMaxRetries() {
+        Settings repositorySettings = generateRepositorySettings(null, null, null, 20);
+        Settings settings = Settings.builder()
+            .put(S3Repository.Repositories.MAX_RETRIES_SETTING.getKey(), 10)
+            .build();
+        launchAWSConfigurationTest(settings, repositorySettings, Protocol.HTTPS, null, -1, null,
+            null, null, 20, false, 50000);
     }
 
     protected void launchAWSConfigurationTest(Settings settings,
@@ -194,15 +288,13 @@ public class AwsS3ServiceImplTests extends ESTestCase {
                                               Integer expectedMaxRetries,
                                               boolean expectedUseThrottleRetries,
                                               int expectedReadTimeout) {
-        Protocol protocol = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.PROTOCOL_SETTING, S3Repository.Repositories.PROTOCOL_SETTING);
         Integer maxRetries = S3Repository.getValue(singleRepositorySettings, settings,
             S3Repository.Repository.MAX_RETRIES_SETTING, S3Repository.Repositories.MAX_RETRIES_SETTING);
         Boolean useThrottleRetries = S3Repository.getValue(singleRepositorySettings, settings,
             S3Repository.Repository.USE_THROTTLE_RETRIES_SETTING, S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
 
-        ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, settings, protocol, maxRetries, null,
-            useThrottleRetries);
+        ClientConfiguration configuration = InternalAwsS3Service.buildConfiguration(logger, singleRepositorySettings, settings,
+            "default", maxRetries, null, useThrottleRetries);
 
         assertThat(configuration.getResponseMetadataCacheSize(), is(0));
         assertThat(configuration.getProtocol(), is(expectedProtocol));
@@ -216,11 +308,8 @@ public class AwsS3ServiceImplTests extends ESTestCase {
         assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
     }
 
-    private static Settings generateRepositorySettings(String key, String secret, String region, String endpoint, Integer maxRetries) {
+    private static Settings generateRepositorySettings(String key, String secret, String endpoint, Integer maxRetries) {
         Settings.Builder builder = Settings.builder();
-        if (region != null) {
-            builder.put(S3Repository.Repository.REGION_SETTING.getKey(), region);
-        }
         if (endpoint != null) {
             builder.put(S3Repository.Repository.ENDPOINT_SETTING.getKey(), endpoint);
         }
@@ -237,74 +326,32 @@ public class AwsS3ServiceImplTests extends ESTestCase {
     }
 
     public void testDefaultEndpoint() {
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), Settings.EMPTY, "");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), Settings.EMPTY,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), Settings.EMPTY, "");
+    }
+
+    public void testEndpointSetting() {
+        Settings settings = Settings.builder()
+            .put("s3.client.default.endpoint", "s3.endpoint")
+            .build();
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), settings, "s3.endpoint");
+    }
+
+    public void testEndpointSettingBackcompat() {
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", "repository.endpoint", null),
             Settings.EMPTY, "repository.endpoint");
-    }
-
-    public void testSpecificEndpoint() {
+        assertWarnings("[" + S3Repository.Repository.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
         Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "ec2.endpoint")
+            .put(S3Repository.Repositories.ENDPOINT_SETTING.getKey(), "repositories.endpoint")
             .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "ec2.endpoint");
-        // Endpoint has precedence on region. Whatever region we set, we won't use it
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "ec2.endpoint");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
+        assertEndpoint(generateRepositorySettings("repository_key", "repository_secret", null, null), settings,
+            "repositories.endpoint");
+        assertWarnings("[" + S3Repository.Repositories.ENDPOINT_SETTING.getKey() + "] setting was deprecated");
     }
 
-    public void testRegionWithAwsSettings() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "s3-eu-west-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    public void testRegionWithAwsAndS3Settings() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), randomFrom("eu-west", "eu-west-1"))
-            .put(InternalAwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), randomFrom("us-west", "us-west-1"))
-            .build();
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings,
-            "s3-us-west-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    public void testInvalidRegion() {
-        Settings settings = Settings.builder()
-            .put(InternalAwsS3Service.REGION_SETTING.getKey(), "does-not-exist")
-            .build();
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, null, null), settings, null);
-        });
-        assertThat(e.getMessage(), containsString("No automatic endpoint could be derived from region"));
-
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", "eu-central", null, null), settings,
-            "s3.eu-central-1.amazonaws.com");
-        launchAWSEndpointTest(generateRepositorySettings("repository_key", "repository_secret", null, "repository.endpoint", null),
-            settings, "repository.endpoint");
-    }
-
-    protected void launchAWSEndpointTest(Settings singleRepositorySettings, Settings settings,
-                                         String expectedEndpoint) {
-        String region = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.REGION_SETTING, S3Repository.Repositories.REGION_SETTING);
-        String endpoint = S3Repository.getValue(singleRepositorySettings, settings,
-            S3Repository.Repository.ENDPOINT_SETTING, S3Repository.Repositories.ENDPOINT_SETTING);
-
-        String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, settings, endpoint, region);
+    private void assertEndpoint(Settings repositorySettings, Settings settings,
+                                  String expectedEndpoint) {
+        String configName = InternalAwsS3Service.CLIENT_NAME.get(repositorySettings);
+        String foundEndpoint = InternalAwsS3Service.findEndpoint(logger, repositorySettings, settings, configName);
         assertThat(foundEndpoint, is(expectedEndpoint));
     }
 
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
index e7a57958a89..554e913c676 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
@@ -18,14 +18,14 @@
  */
 package org.elasticsearch.cloud.aws;
 
-import com.amazonaws.Protocol;
+import java.util.IdentityHashMap;
+
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
 
-import java.util.IdentityHashMap;
-
 public class TestAwsS3Service extends InternalAwsS3Service {
     public static class TestPlugin extends S3RepositoryPlugin {
         @Override
@@ -42,9 +42,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
 
 
     @Override
-    public synchronized AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
-                                        boolean useThrottleRetries, Boolean pathStyleAccess) {
-        return cachedWrapper(super.client(repositorySettings, endpoint, protocol, region, maxRetries, useThrottleRetries, pathStyleAccess));
+    public synchronized AmazonS3 client(Settings repositorySettings, Integer maxRetries,
+                                              boolean useThrottleRetries, Boolean pathStyleAccess) {
+        return cachedWrapper(super.client(repositorySettings, maxRetries, useThrottleRetries, pathStyleAccess));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
index bca1c1d8a18..98dd43b6f4d 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
@@ -33,7 +33,7 @@ public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase {
         MockAmazonS3 client = new MockAmazonS3();
         String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
 
-        return new S3BlobStore(Settings.EMPTY, client, bucket, null, false,
+        return new S3BlobStore(Settings.EMPTY, client, bucket, false,
             new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard");
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 4d7f30ed9db..d38c61f4a37 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -197,12 +197,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
 
         Settings settings = internalCluster().getInstance(Settings.class);
         Settings bucket = settings.getByPrefix("repositories.s3.");
-        AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(
-            repositorySettings,
-            null,
-            S3Repository.Repositories.PROTOCOL_SETTING.get(settings),
-            S3Repository.Repositories.REGION_SETTING.get(settings),
-            null, randomBoolean(), null);
+        AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
 
         String bucketName = bucket.get("bucket");
         logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@@ -288,7 +283,6 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
         PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo")
                 .setType("s3").setSettings(Settings.builder()
                     .put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath)
-                    .put(S3Repository.Repository.REGION_SETTING.getKey(), bucketSettings.get("region"))
                     .put(S3Repository.Repository.KEY_SETTING.getKey(), bucketSettings.get("access_key"))
                     .put(S3Repository.Repository.SECRET_SETTING.getKey(), bucketSettings.get("secret_key"))
                     .put(S3Repository.Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket"))
@@ -346,7 +340,6 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
                 .setType("s3").setSettings(Settings.builder()
                     .put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath)
                     .put(S3Repository.Repository.BUCKET_SETTING.getKey(), bucketSettings.get("bucket"))
-                    .put(S3Repository.Repository.REGION_SETTING.getKey(), bucketSettings.get("region"))
                     ).get();
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
@@ -467,16 +460,12 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
                 settings.getByPrefix("repositories.s3.external-bucket.")
         };
         for (Settings bucket : buckets) {
-            String endpoint = bucket.get("endpoint", S3Repository.Repositories.ENDPOINT_SETTING.get(settings));
-            Protocol protocol = S3Repository.Repositories.PROTOCOL_SETTING.get(settings);
-            String region = bucket.get("region", S3Repository.Repositories.REGION_SETTING.get(settings));
             String bucketName = bucket.get("bucket");
 
             // We check that settings has been set in elasticsearch.yml integration test file
             // as described in README
             assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
-            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, endpoint, protocol, region, null,
-                randomBoolean(), null);
+            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(Settings.EMPTY, null, randomBoolean(), null);
             try {
                 ObjectListing prevListing = null;
                 //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
@@ -513,7 +502,7 @@ public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
                     client.deleteObjects(multiObjectDeleteRequest);
                 }
             } catch (Exception ex) {
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}] in [{}]", bucketName, region), ex);
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("Failed to delete S3 repository [{}]", bucketName), ex);
             }
         }
     }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
index 9c567a570fe..30056f67d2f 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class RepositoryS3ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
index 14595d13448..79fc453e0ed 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -21,15 +21,16 @@ package org.elasticsearch.repositories.s3;
 
 import java.io.IOException;
 
-import com.amazonaws.Protocol;
 import com.amazonaws.services.s3.AbstractAmazonS3;
 import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
@@ -49,7 +50,7 @@ public class S3RepositoryTests extends ESTestCase {
     }
 
     private static class DummyS3Service extends AbstractLifecycleComponent implements AwsS3Service {
-        public DummyS3Service() {
+        DummyS3Service() {
             super(Settings.EMPTY);
         }
         @Override
@@ -59,7 +60,7 @@ public class S3RepositoryTests extends ESTestCase {
         @Override
         protected void doClose() {}
         @Override
-        public AmazonS3 client(Settings repositorySettings, String endpoint, Protocol protocol, String region, Integer maxRetries,
+        public AmazonS3 client(Settings repositorySettings, Integer maxRetries,
                                boolean useThrottleRetries, Boolean pathStyleAccess) {
             return new DummyS3Client();
         }
@@ -69,10 +70,16 @@ public class S3RepositoryTests extends ESTestCase {
         Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build();
         Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build();
 
-        assertEquals("key1", getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("key1", getValue(localSettings, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("key2", getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
-        assertEquals("", getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key1".toCharArray()),
+                     getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key1".toCharArray()),
+                     getValue(localSettings, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("key2".toCharArray()),
+                     getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals(new SecureString("".toCharArray()),
+                     getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertWarnings("[" + Repository.KEY_SETTING.getKey() + "] setting was deprecated",
+                       "[" + Repositories.KEY_SETTING.getKey() + "] setting was deprecated");
     }
 
     public void testInvalidChunkBufferSizeSettings() throws IOException {
@@ -94,7 +101,7 @@ public class S3RepositoryTests extends ESTestCase {
         RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
             .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
             .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
-        new S3Repository(metadata, Settings.EMPTY, new DummyS3Service());
+        new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
     }
 
     private void assertInvalidBuffer(int bufferMB, int chunkMB, Class<? extends Exception> clazz, String msg) throws IOException {
@@ -102,20 +109,24 @@ public class S3RepositoryTests extends ESTestCase {
             .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
             .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
 
-        Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, new DummyS3Service()));
+        Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY,
+            new DummyS3Service()));
         assertThat(e.getMessage(), containsString(msg));
     }
 
     public void testBasePathSetting() throws IOException {
         RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
             .put(Repository.BASE_PATH_SETTING.getKey(), "/foo/bar").build());
-        S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, new DummyS3Service());
+        S3Repository s3repo = new S3Repository(metadata, Settings.EMPTY, NamedXContentRegistry.EMPTY, new DummyS3Service());
         assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
-
+        assertWarnings("S3 repository base_path" +
+                " trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
         metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.EMPTY);
         Settings settings = Settings.builder().put(Repositories.BASE_PATH_SETTING.getKey(), "/foo/bar").build();
-        s3repo = new S3Repository(metadata, settings, new DummyS3Service());
+        s3repo = new S3Repository(metadata, settings, NamedXContentRegistry.EMPTY, new DummyS3Service());
         assertEquals("foo/bar/", s3repo.basePath().buildAsString()); // make sure leading `/` is removed and trailing is added
+        assertWarnings("S3 repository base_path" +
+                " trimming the leading `/`, and leading `/` will not be supported for the S3 repository in future releases");
     }
 
     public void testDefaultBufferSize() {
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
index 17294986d2d..fe60a16b4b5 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
@@ -1,7 +1,12 @@
 # Integration tests for Repository S3 component
 #
 "S3 repository can be registered":
+    - skip:
+        features: warnings
     - do:
+        warnings:
+          - "[access_key] setting was deprecated in Elasticsearch and it will be removed in a future release! See the breaking changes lists in the documentation for details"
+          - "[secret_key] setting was deprecated in Elasticsearch and it will be removed in a future release! See the breaking changes lists in the documentation for details"
         snapshot.create_repository:
           repository: test_repo_s3_1
           verify: false
diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
index fe2c32723e2..19499f56e79 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/index/store/SmbDirectoryWrapper.java
@@ -59,7 +59,7 @@ public final class SmbDirectoryWrapper extends FilterDirectory {
          */
         static final int CHUNK_SIZE = 8192;
 
-        public SmbFSIndexOutput(String name) throws IOException {
+        SmbFSIndexOutput(String name) throws IOException {
             super("SmbFSIndexOutput(path=\"" + fsDirectory.getDirectory().resolve(name) + "\")", name, new FilterOutputStream(Channels.newOutputStream(Files.newByteChannel(fsDirectory.getDirectory().resolve(name), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.READ, StandardOpenOption.WRITE))) {
                 // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
                 @Override
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
index 0b9de745cac..0216083c95f 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class StoreSmbClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index b151485ef3f..fceeac283fc 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -1,3 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 /* This project runs the core REST tests against a 2 node cluster where one of the nodes has a different minor.
@@ -16,9 +36,9 @@ apply plugin: 'elasticsearch.rest-test'
 integTest {
   includePackaged = true
   cluster {
-    numNodes = 2
-    numBwcNodes = 1
-    bwcVersion = "6.0.0-alpha1-SNAPSHOT"
+    numNodes = 4
+    numBwcNodes = 2
+    bwcVersion = "5.3.0-SNAPSHOT"
     setting 'logger.org.elasticsearch', 'DEBUG'
   }
 }
diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
index af77b216bc4..cac063fce78 100644
--- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
+++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java
@@ -21,10 +21,10 @@ package org.elasticsearch.backwards;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -36,7 +36,7 @@ public class Backwards50ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java
new file mode 100644
index 00000000000..3dd98441592
--- /dev/null
+++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/IndexingIT.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.backwards;
+
+import org.apache.http.HttpHost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.seqno.SeqNoStats;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+
+public class IndexingIT extends ESRestTestCase {
+
+    private ObjectPath objectPath(Response response) throws IOException {
+        String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+        String contentType = response.getHeader("Content-Type");
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+        return ObjectPath.createFromXContent(xContentType.xContent(), body);
+    }
+
+    private void assertOK(Response response) {
+        assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201)));
+    }
+
+    private void ensureGreen() throws IOException {
+        Map<String, String> params = new HashMap<>();
+        params.put("wait_for_status", "green");
+        params.put("wait_for_no_relocating_shards", "true");
+        assertOK(client().performRequest("GET", "_cluster/health", params));
+    }
+
+    private void createIndex(String name, Settings settings) throws IOException {
+        assertOK(client().performRequest("PUT", name, Collections.emptyMap(),
+            new StringEntity("{ \"settings\": " + Strings.toString(settings) + " }", ContentType.APPLICATION_JSON)));
+    }
+
+    private void updateIndexSetting(String name, Settings.Builder settings) throws IOException {
+        updateIndexSetting(name, settings.build());
+    }
+    private void updateIndexSetting(String name, Settings settings) throws IOException {
+        assertOK(client().performRequest("PUT", name + "/_settings", Collections.emptyMap(),
+            new StringEntity(Strings.toString(settings))));
+    }
+
+    protected int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
+        for (int i = 0; i < numDocs; i++) {
+            final int id = idStart + i;
+            assertOK(client().performRequest("PUT", index + "/test/" + id, emptyMap(),
+                new StringEntity("{\"test\": \"test_" + id + "\"}", ContentType.APPLICATION_JSON)));
+        }
+        return numDocs;
+    }
+
+    public void testSeqNoCheckpoints() throws Exception {
+        Nodes nodes = buildNodeAndVersions();
+        assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
+        logger.info("cluster discovered: {}", nodes.toString());
+        final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList());
+        final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
+        Settings.Builder settings = Settings.builder()
+            .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
+            .put("index.routing.allocation.include._name", bwcNames);
+
+        final boolean checkGlobalCheckpoints = nodes.getMaster().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED);
+        logger.info("master version is [{}], global checkpoints will be [{}]", nodes.getMaster().getVersion(),
+            checkGlobalCheckpoints ? "checked" : "not be checked");
+        if (checkGlobalCheckpoints) {
+            settings.put(IndexSettings.INDEX_SEQ_NO_CHECKPOINT_SYNC_INTERVAL.getKey(), "100ms");
+        }
+        final String index = "test";
+        createIndex(index, settings.build());
+        try (RestClient newNodeClient = buildClient(restClientSettings(),
+            nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) {
+            int numDocs = 0;
+            final int numberOfInitialDocs = 1 + randomInt(5);
+            logger.info("indexing [{}] docs initially", numberOfInitialDocs);
+            numDocs += indexDocs(index, 0, numberOfInitialDocs);
+            assertSeqNoOnShards(nodes, checkGlobalCheckpoints, 0, newNodeClient);
+            logger.info("allowing shards on all nodes");
+            updateIndexSetting(index, Settings.builder().putNull("index.routing.allocation.include._name"));
+            ensureGreen();
+            assertOK(client().performRequest("POST", index + "/_refresh"));
+            for (final String bwcName : bwcNamesList) {
+                assertCount(index, "_only_nodes:" + bwcName, numDocs);
+            }
+            final int numberOfDocsAfterAllowingShardsOnAllNodes = 1 + randomInt(5);
+            logger.info("indexing [{}] docs after allowing shards on all nodes", numberOfDocsAfterAllowingShardsOnAllNodes);
+            numDocs += indexDocs(index, numDocs, numberOfDocsAfterAllowingShardsOnAllNodes);
+            assertSeqNoOnShards(nodes, checkGlobalCheckpoints, 0, newNodeClient);
+            Shard primary = buildShards(nodes, newNodeClient).stream().filter(Shard::isPrimary).findFirst().get();
+            logger.info("moving primary to new node by excluding {}", primary.getNode().getNodeName());
+            updateIndexSetting(index, Settings.builder().put("index.routing.allocation.exclude._name", primary.getNode().getNodeName()));
+            ensureGreen();
+            int numDocsOnNewPrimary = 0;
+            final int numberOfDocsAfterMovingPrimary = 1 + randomInt(5);
+            logger.info("indexing [{}] docs after moving primary", numberOfDocsAfterMovingPrimary);
+            numDocsOnNewPrimary += indexDocs(index, numDocs, numberOfDocsAfterMovingPrimary);
+            numDocs += numberOfDocsAfterMovingPrimary;
+            assertSeqNoOnShards(nodes, checkGlobalCheckpoints, numDocsOnNewPrimary, newNodeClient);
+            /*
+             * Dropping the number of replicas to zero, and then increasing it to one triggers a recovery thus exercising any BWC-logic in
+             * the recovery code.
+             */
+            logger.info("setting number of replicas to 0");
+            updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 0));
+            final int numberOfDocsAfterDroppingReplicas = 1 + randomInt(5);
+            logger.info("indexing [{}] docs after setting number of replicas to 0", numberOfDocsAfterDroppingReplicas);
+            numDocsOnNewPrimary += indexDocs(index, numDocs, numberOfDocsAfterDroppingReplicas);
+            numDocs += numberOfDocsAfterDroppingReplicas;
+            logger.info("setting number of replicas to 1");
+            updateIndexSetting(index, Settings.builder().put("index.number_of_replicas", 1));
+            ensureGreen();
+            assertOK(client().performRequest("POST", index + "/_refresh"));
+            // the number of documents on the primary and on the recovered replica should match the number of indexed documents
+            assertCount(index, "_primary", numDocs);
+            assertCount(index, "_replica", numDocs);
+            assertSeqNoOnShards(nodes, checkGlobalCheckpoints, numDocsOnNewPrimary, newNodeClient);
+        }
+    }
+
+    private void assertCount(final String index, final String preference, final int expectedCount) throws IOException {
+        final Response response = client().performRequest("GET", index + "/_count", Collections.singletonMap("preference", preference));
+        assertOK(response);
+        final int actualCount = Integer.parseInt(objectPath(response).evaluate("count").toString());
+        assertThat(actualCount, equalTo(expectedCount));
+    }
+
+    private void assertSeqNoOnShards(Nodes nodes, boolean checkGlobalCheckpoints, int numDocs, RestClient client) throws Exception {
+        assertBusy(() -> {
+            try {
+                List<Shard> shards = buildShards(nodes, client);
+                Shard primaryShard = shards.stream().filter(Shard::isPrimary).findFirst().get();
+                assertNotNull("failed to find primary shard", primaryShard);
+                final long expectedGlobalCkp;
+                final long expectMaxSeqNo;
+                logger.info("primary resolved to node {}", primaryShard.getNode());
+                if (primaryShard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                    expectMaxSeqNo = numDocs - 1;
+                    expectedGlobalCkp = numDocs - 1;
+                } else {
+                    expectedGlobalCkp = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+                    expectMaxSeqNo = SequenceNumbersService.NO_OPS_PERFORMED;
+                }
+                for (Shard shard : shards) {
+                    if (shard.getNode().getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                        final SeqNoStats seqNoStats = shard.getSeqNoStats();
+                        logger.info("stats for {}, primary [{}]: [{}]", shard.getNode(), shard.isPrimary(), seqNoStats);
+                        assertThat("max_seq no on " + shard.getNode() + " is wrong", seqNoStats.getMaxSeqNo(), equalTo(expectMaxSeqNo));
+                        assertThat("localCheckpoint no on " + shard.getNode() + " is wrong",
+                            seqNoStats.getLocalCheckpoint(), equalTo(expectMaxSeqNo));
+                        if (checkGlobalCheckpoints) {
+                            assertThat("globalCheckpoint no on " + shard.getNode() + " is wrong",
+                                seqNoStats.getGlobalCheckpoint(), equalTo(expectedGlobalCkp));
+                        }
+                    } else {
+                        logger.info("skipping seq no test on {}", shard.getNode());
+                    }
+                }
+            } catch (IOException e) {
+                throw new AssertionError("unexpected io exception", e);
+            }
+        });
+    }
+
+    private List<Shard> buildShards(Nodes nodes, RestClient client) throws IOException {
+        Response response = client.performRequest("GET", "test/_stats", singletonMap("level", "shards"));
+        List<Object> shardStats = objectPath(response).evaluate("indices.test.shards.0");
+        ArrayList<Shard> shards = new ArrayList<>();
+        for (Object shard : shardStats) {
+            final String nodeId = ObjectPath.evaluate(shard, "routing.node");
+            final Boolean primary = ObjectPath.evaluate(shard, "routing.primary");
+            final Node node = nodes.getSafe(nodeId);
+            final SeqNoStats seqNoStats;
+            if (node.getVersion().onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
+                Integer maxSeqNo = ObjectPath.evaluate(shard, "seq_no.max_seq_no");
+                Integer localCheckpoint = ObjectPath.evaluate(shard, "seq_no.local_checkpoint");
+                Integer globalCheckpoint = ObjectPath.evaluate(shard, "seq_no.global_checkpoint");
+                seqNoStats = new SeqNoStats(maxSeqNo, localCheckpoint, globalCheckpoint);
+            } else {
+                seqNoStats = null;
+            }
+            shards.add(new Shard(node, primary, seqNoStats));
+        }
+        return shards;
+    }
+
+    private Nodes buildNodeAndVersions() throws IOException {
+        Response response = client().performRequest("GET", "_nodes");
+        ObjectPath objectPath = objectPath(response);
+        Map<String, Object> nodesAsMap = objectPath.evaluate("nodes");
+        Nodes nodes = new Nodes();
+        for (String id : nodesAsMap.keySet()) {
+            nodes.add(new Node(
+                id,
+                objectPath.evaluate("nodes." + id + ".name"),
+                Version.fromString(objectPath.evaluate("nodes." + id + ".version")),
+                HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address"))));
+        }
+        response = client().performRequest("GET", "_cluster/state");
+        nodes.setMasterNodeId(objectPath(response).evaluate("master_node"));
+        return nodes;
+    }
+
+    final class Nodes extends HashMap<String, Node> {
+
+        private String masterNodeId = null;
+
+        public Node getMaster() {
+            return get(masterNodeId);
+        }
+
+        public void setMasterNodeId(String id) {
+            if (get(id) == null) {
+                throw new IllegalArgumentException("node with id [" + id + "] not found. got:" + toString());
+            }
+            masterNodeId = id;
+        }
+
+        public void add(Node node) {
+            put(node.getId(), node);
+        }
+
+        public List<Node> getNewNodes() {
+            Version bwcVersion = getBWCVersion();
+            return values().stream().filter(n -> n.getVersion().after(bwcVersion)).collect(Collectors.toList());
+        }
+
+        public List<Node> getBWCNodes() {
+            Version bwcVersion = getBWCVersion();
+            return values().stream().filter(n -> n.getVersion().equals(bwcVersion)).collect(Collectors.toList());
+        }
+
+        public Version getBWCVersion() {
+            if (isEmpty()) {
+                throw new IllegalStateException("no nodes available");
+            }
+            return Version.fromId(values().stream().map(node -> node.getVersion().id).min(Integer::compareTo).get());
+        }
+
+        public Node getSafe(String id) {
+            Node node = get(id);
+            if (node == null) {
+                throw new IllegalArgumentException("node with id [" + id + "] not found");
+            }
+            return node;
+        }
+
+        @Override
+        public String toString() {
+            return "Nodes{" +
+                "masterNodeId='" + masterNodeId + "'\n" +
+                values().stream().map(Node::toString).collect(Collectors.joining("\n")) +
+                '}';
+        }
+    }
+
+    final class Node {
+        private final String id;
+        private final String nodeName;
+        private final Version version;
+        private final HttpHost publishAddress;
+
+        Node(String id, String nodeName, Version version, HttpHost publishAddress) {
+            this.id = id;
+            this.nodeName = nodeName;
+            this.version = version;
+            this.publishAddress = publishAddress;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public String getNodeName() {
+            return nodeName;
+        }
+
+        public HttpHost getPublishAddress() {
+            return publishAddress;
+        }
+
+        public Version getVersion() {
+            return version;
+        }
+
+        @Override
+        public String toString() {
+            return "Node{" +
+                "id='" + id + '\'' +
+                ", nodeName='" + nodeName + '\'' +
+                ", version=" + version +
+                '}';
+        }
+    }
+
+    final class Shard {
+        private final Node node;
+        private final boolean Primary;
+        private final SeqNoStats seqNoStats;
+
+        Shard(Node node, boolean primary, SeqNoStats seqNoStats) {
+            this.node = node;
+            Primary = primary;
+            this.seqNoStats = seqNoStats;
+        }
+
+        public Node getNode() {
+            return node;
+        }
+
+        public boolean isPrimary() {
+            return Primary;
+        }
+
+        public SeqNoStats getSeqNoStats() {
+            return seqNoStats;
+        }
+
+        @Override
+        public String toString() {
+            return "Shard{" +
+                "node=" + node +
+                ", Primary=" + Primary +
+                ", seqNoStats=" + seqNoStats +
+                '}';
+        }
+    }
+}
diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
index cba9334fbca..472fc872616 100644
--- a/qa/evil-tests/build.gradle
+++ b/qa/evil-tests/build.gradle
@@ -42,7 +42,7 @@ thirdPartyAudit.excludes = [
   'com.google.common.cache.Striped64$Cell',
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
   'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
-  
+
   // missing class
   'com.ibm.icu.lang.UCharacter',
 ]
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
index 23da3a99df5..2c00030973b 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.bootstrap;
 
+import java.util.Map;
+
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
 
 public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
 
@@ -39,8 +41,10 @@ public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 true,
                 output -> {},
                 (foreground, pidFile, quiet, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(1));
-                    assertThat(esSettings, hasEntry("path.home", value));
+                    Map<String, String> settings = esSettings.settings().getAsMap();
+                    assertThat(settings.size(), equalTo(2));
+                    assertThat(settings, hasEntry("path.home", value));
+                    assertThat(settings, hasKey("path.logs")); // added by env initialization
                 });
 
         System.clearProperty("es.path.home");
@@ -50,8 +54,10 @@ public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 true,
                 output -> {},
                 (foreground, pidFile, quiet, esSettings) -> {
-                    assertThat(esSettings.size(), equalTo(1));
-                    assertThat(esSettings, hasEntry("path.home", commandLineValue));
+                    Map<String, String> settings = esSettings.settings().getAsMap();
+                    assertThat(settings.size(), equalTo(2));
+                    assertThat(settings, hasEntry("path.home", commandLineValue));
+                    assertThat(settings, hasKey("path.logs")); // added by env initialization
                 },
                 "-Epath.home=" + commandLineValue);
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
index 7ee2120c36f..9cd6ec630ad 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java
@@ -34,9 +34,11 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -151,4 +153,33 @@ public class EvilLoggerConfigurationTests extends ESTestCase {
         assertThat(e, hasToString(containsString("no log4j2.properties found; tried")));
     }
 
+    public void testLoggingLevelsFromSettings() throws IOException, UserException {
+        final Level rootLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Level fooLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Level barLevel = randomFrom(Level.TRACE, Level.DEBUG, Level.INFO, Level.WARN, Level.ERROR);
+        final Path configDir = getDataPath("minimal");
+        final Settings settings = Settings.builder()
+            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put("logger.level", rootLevel.name())
+            .put("logger.foo", fooLevel.name())
+            .put("logger.bar", barLevel.name())
+            .build();
+        final Environment environment = new Environment(settings);
+        LogConfigurator.configure(environment);
+
+        final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+        final Configuration config = ctx.getConfiguration();
+        final Map<String, LoggerConfig> loggerConfigs = config.getLoggers();
+        assertThat(loggerConfigs.size(), equalTo(3));
+        assertThat(loggerConfigs, hasKey(""));
+        assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel));
+        assertThat(loggerConfigs, hasKey("foo"));
+        assertThat(loggerConfigs.get("foo").getLevel(), equalTo(fooLevel));
+        assertThat(loggerConfigs, hasKey("bar"));
+        assertThat(loggerConfigs.get("bar").getLevel(), equalTo(barLevel));
+
+        assertThat(ctx.getLogger(randomAsciiOfLength(16)).getLevel(), equalTo(rootLevel));
+    }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
index dd04d2de50e..f02ce6031c3 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java
@@ -29,9 +29,11 @@ import org.apache.logging.log4j.core.appender.CountingNoOpAppender;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.elasticsearch.cli.UserException;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.hamcrest.RegexMatcher;
 
@@ -70,7 +72,11 @@ public class EvilLoggerTests extends ESTestCase {
         testLogger.info("This is an info message");
         testLogger.debug("This is a debug message");
         testLogger.trace("This is a trace message");
-        final String path = System.getProperty("es.logs") + ".log";
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                ".log";
         final List<String> events = Files.readAllLines(PathUtils.get(path));
         assertThat(events.size(), equalTo(5));
         final String location = "org.elasticsearch.common.logging.EvilLoggerTests.testLocationInfoTest";
@@ -88,7 +94,11 @@ public class EvilLoggerTests extends ESTestCase {
         final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger("deprecation"));
 
         deprecationLogger.deprecated("This is a deprecation message");
-        final String deprecationPath = System.getProperty("es.logs") + "_deprecation.log";
+        final String deprecationPath =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                "_deprecation.log";
         final List<String> deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath));
         assertThat(deprecationEvents.size(), equalTo(1));
         assertLogLine(
@@ -96,6 +106,7 @@ public class EvilLoggerTests extends ESTestCase {
             Level.WARN,
             "org.elasticsearch.common.logging.DeprecationLogger.deprecated",
             "This is a deprecation message");
+        assertWarnings("This is a deprecation message");
     }
 
     public void testFindAppender() throws IOException, UserException {
@@ -122,7 +133,11 @@ public class EvilLoggerTests extends ESTestCase {
         final Exception e = new Exception("exception");
         logger.info(new ParameterizedMessage("{}", "test"), e);
 
-        final String path = System.getProperty("es.logs") + ".log";
+        final String path =
+            System.getProperty("es.logs.base_path") +
+                System.getProperty("file.separator") +
+                System.getProperty("es.logs.cluster_name") +
+                ".log";
         final List<String> events = Files.readAllLines(PathUtils.get(path));
 
         final StringWriter sw = new StringWriter();
@@ -140,54 +155,39 @@ public class EvilLoggerTests extends ESTestCase {
         }
     }
 
+    public void testProperties() throws IOException, UserException {
+        final Settings.Builder builder = Settings.builder().put("cluster.name", randomAsciiOfLength(16));
+        if (randomBoolean()) {
+            builder.put("node.name", randomAsciiOfLength(16));
+        }
+        final Settings settings = builder.build();
+        setupLogging("minimal", settings);
 
-    public void testLog4jShutdownHack() {
-        final AtomicBoolean denied = new AtomicBoolean();
-        final SecurityManager sm = System.getSecurityManager();
-        try {
-            System.setSecurityManager(new SecurityManager() {
-                @Override
-                public void checkPermission(Permission perm) {
-                    // just grant all permissions to Log4j, except we deny MBeanServerPermission
-                    // "createMBeanServer" as this will trigger the Log4j bug
-                    if (perm instanceof MBeanServerPermission && "createMBeanServer".equals(perm.getName())) {
-                        // without the hack in place, Log4j will try to get an MBean server which we will deny
-                        // with the hack in place, this permission should never be requested by Log4j
-                        denied.set(true);
-                        throw new AccessControlException("denied");
-                    }
-                }
+        assertNotNull(System.getProperty("es.logs.base_path"));
 
-                @Override
-                public void checkPropertyAccess(String key) {
-                    /*
-                     * grant access to all properties; this is so that Log4j can check if its usage
-                     * of JMX is disabled or not by reading log4j2.disable.jmx but there are other
-                     * properties that Log4j will try to read as well and its simpler to just grant
-                     * them all
-                     */
-                }
-            });
-
-            // this will trigger the bug without the hack
-            LoggerContext context = (LoggerContext) LogManager.getContext(false);
-            Configurator.shutdown(context);
-
-            // Log4j should have never requested permissions to create an MBean server
-            assertFalse(denied.get());
-        } finally {
-            System.setSecurityManager(sm);
+        assertThat(System.getProperty("es.logs.cluster_name"), equalTo(ClusterName.CLUSTER_NAME_SETTING.get(settings).value()));
+        if (Node.NODE_NAME_SETTING.exists(settings)) {
+            assertThat(System.getProperty("es.logs.node_name"), equalTo(Node.NODE_NAME_SETTING.get(settings)));
+        } else {
+            assertNull(System.getProperty("es.logs.node_name"));
         }
     }
 
     private void setupLogging(final String config) throws IOException, UserException {
+        setupLogging(config, Settings.EMPTY);
+    }
+
+    private void setupLogging(final String config, final Settings settings) throws IOException, UserException {
+        assert !Environment.PATH_CONF_SETTING.exists(settings);
+        assert !Environment.PATH_HOME_SETTING.exists(settings);
         final Path configDir = getDataPath(config);
         // need to set custom path.conf so we can use a custom log4j2.properties file for the test
-        final Settings settings = Settings.builder()
-                .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
-                .build();
-        final Environment environment = new Environment(settings);
+        final Settings mergedSettings = Settings.builder()
+            .put(settings)
+            .put(Environment.PATH_CONF_SETTING.getKey(), configDir.toAbsolutePath())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        final Environment environment = new Environment(mergedSettings);
         LogConfigurator.configure(environment);
     }
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 70c5f633da1..7d3115e25db 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -60,19 +60,17 @@ import java.nio.file.attribute.PosixFileAttributes;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.UserPrincipal;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
 
 @LuceneTestCase.SuppressFileSystems("*")
@@ -110,7 +108,7 @@ public class InstallPluginCommandTests extends ESTestCase {
             private final FileSystem fileSystem;
             private final Function<String, Path> temp;
 
-            public Parameter(FileSystem fileSystem, String root) {
+            Parameter(FileSystem fileSystem, String root) {
                 this(fileSystem, s -> {
                     try {
                         return Files.createTempDirectory(fileSystem.getPath(root), s);
@@ -120,7 +118,7 @@ public class InstallPluginCommandTests extends ESTestCase {
                 });
             }
 
-            public Parameter(FileSystem fileSystem, Function<String, Path> temp) {
+            Parameter(FileSystem fileSystem, Function<String, Path> temp) {
                 this.fileSystem = fileSystem;
                 this.temp = temp;
             }
@@ -207,8 +205,7 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     static MockTerminal installPlugin(String pluginUrl, Path home, boolean jarHellCheck) throws Exception {
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", home.toString());
+        Environment env = new Environment(Settings.builder().put("path.home", home).build());
         MockTerminal terminal = new MockTerminal();
         new InstallPluginCommand() {
             @Override
@@ -217,7 +214,7 @@ public class InstallPluginCommandTests extends ESTestCase {
                     super.jarHellCheck(candidate, pluginsDir);
                 }
             }
-        }.execute(terminal, pluginUrl, true, settings);
+        }.execute(terminal, pluginUrl, true, env);
         return terminal;
     }
 
@@ -674,19 +671,29 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertThat(terminal.getOutput(), not(containsString("100%")));
     }
 
+    public void testPluginAlreadyInstalled() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPlugin("fake", pluginDir);
+        installPlugin(pluginZip, env.v1());
+        final UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1(), randomBoolean()));
+        assertThat(
+            e.getMessage(),
+            equalTo("plugin directory [" + env.v2().pluginsFile().resolve("fake") + "] already exists; " +
+                "if you need to update the plugin, uninstall it first using command 'remove fake'"));
+    }
+
     private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         // if batch is enabled, we also want to add a security policy
         String pluginZip = createPlugin("fake", pluginDir, isBatch);
 
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", env.v1().toString());
         new InstallPluginCommand() {
             @Override
             void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
             }
-        }.execute(terminal, pluginZip, isBatch, settings);
+        }.execute(terminal, pluginZip, isBatch, env.v2());
     }
 
     // TODO: test checksum (need maven/official below)
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
index d7d3efb9742..997fbeffadd 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -61,10 +61,9 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     static MockTerminal removePlugin(String name, Path home) throws Exception {
-        Map<String, String> settings = new HashMap<>();
-        settings.put("path.home", home.toString());
+        Environment env = new Environment(Settings.builder().put("path.home", home).build());
         MockTerminal terminal = new MockTerminal();
-        new RemovePluginCommand().execute(terminal, name, settings);
+        new RemovePluginCommand().execute(terminal, name, env);
         return terminal;
     }
 
@@ -154,6 +153,12 @@ public class RemovePluginCommandTests extends ESTestCase {
         }
     }
 
+    public void testMissingPluginName() throws Exception {
+        UserException e = expectThrows(UserException.class, () -> removePlugin(null, home));
+        assertEquals(ExitCodes.USAGE, e.exitCode);
+        assertEquals("plugin name is required", e.getMessage());
+    }
+
     private String expectedConfigDirPreservedMessage(final Path configDir) {
         return "-> Preserving plugin config files [" + configDir + "] in case of upgrade, delete manually if not needed";
     }
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
index 6cfb6a4ec37..8b956421458 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties
@@ -5,7 +5,7 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 appender.file.type = File
 appender.file.name = file
-appender.file.fileName = ${sys:es.logs}.log
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = [%p][%l] %marker%m%n
 
@@ -21,7 +21,7 @@ logger.test.additivity = false
 
 appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
-appender.deprecation_file.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
 appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
 
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
index 3f4958adee8..fd7af2ce731 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties
@@ -5,7 +5,7 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 appender.file.type = File
 appender.file.name = file
-appender.file.fileName = ${sys:es.logs}.log
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = [%p][%l] %marker%m%n
 
@@ -15,7 +15,7 @@ rootLogger.appenderRef.file.ref = file
 
 appender.deprecation_file.type = File
 appender.deprecation_file.name = deprecation_file
-appender.deprecation_file.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
 appender.deprecation_file.layout.type = PatternLayout
 appender.deprecation_file.layout.pattern = [%p][%l] %marker%m%n
 
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
index edb143d5fc5..5fa85b5d156 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/location_info/log4j2.properties
@@ -5,7 +5,7 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 appender.file.type = File
 appender.file.name = file
-appender.file.fileName = ${sys:es.logs}.log
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = [%p][%l] %marker%m%n
 
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties
new file mode 100644
index 00000000000..f245dde979c
--- /dev/null
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/minimal/log4j2.properties
@@ -0,0 +1,7 @@
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
index 1f18b38d91e..5dfa369a3bd 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/prefix/log4j2.properties
@@ -5,7 +5,7 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%m%n
 
 appender.file.type = File
 appender.file.name = file
-appender.file.fileName = ${sys:es.logs}.log
+appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = %marker%m%n
 
diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle
new file mode 100644
index 00000000000..b48cd9a3dd1
--- /dev/null
+++ b/qa/multi-cluster-search/build.gradle
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.test.RestIntegTestTask
+
+apply plugin: 'elasticsearch.standalone-test'
+
+task remoteClusterTest(type: RestIntegTestTask) {
+  mustRunAfter(precommit)
+  cluster {
+    distribution = 'zip'
+    numNodes = 2
+    clusterName = 'remote-cluster'
+  }
+  systemProperty 'tests.rest.suite', 'remote_cluster'
+}
+
+task mixedClusterTest(type: RestIntegTestTask) {
+  dependsOn(remoteClusterTest)
+  cluster {
+    distribution = 'zip'
+    setting 'search.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
+    setting 'search.remote.connections_per_cluster', 1
+
+  }
+  systemProperty 'tests.rest.suite', 'multi_cluster'
+  finalizedBy 'remoteClusterTest#node0.stop','remoteClusterTest#node1.stop'
+}
+
+task integTest {
+  dependsOn = [mixedClusterTest]
+}
+
+test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
+
+check.dependsOn(integTest)
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java
similarity index 50%
rename from core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java
rename to qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java
index 3ecae762a2a..e8c3592e214 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineClosedException.java
+++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/upgrades/MultiClusterSearchYamlTestSuiteIT.java
@@ -17,33 +17,31 @@
  * under the License.
  */
 
-package org.elasticsearch.index.engine;
+package org.elasticsearch.upgrades;
 
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.shard.IndexShardClosedException;
-import org.elasticsearch.index.shard.ShardId;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 
 import java.io.IOException;
 
-/**
- * An engine is already closed.
- * <p>
- * Note, the relationship between shard and engine indicates that engine closed is shard closed, and
- * we might get something slipping through the shard and into the engine while the shard is closing.
- *
- *
- */
-public class EngineClosedException extends IndexShardClosedException {
+@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
+public class MultiClusterSearchYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
-    public EngineClosedException(ShardId shardId) {
-        super(shardId);
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
     }
 
-    public EngineClosedException(ShardId shardId, Throwable t) {
-        super(shardId, t);
+    public MultiClusterSearchYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
     }
 
-    public EngineClosedException(StreamInput in) throws IOException{
-        super(in);
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException {
+        return createParameters();
     }
 }
+
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml
new file mode 100644
index 00000000000..31b8fbd251e
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yaml
@@ -0,0 +1,131 @@
+---
+"Index data and search on the mixed cluster":
+
+  - do:
+    indices.create:
+      index: test_index
+      body:
+        settings:
+          index:
+            number_of_shards: 2
+            number_of_replicas: 0
+
+  - do:
+    bulk:
+      refresh: true
+      body:
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "local_cluster", "filter_field": 0}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "local_cluster", "filter_field": 1}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "local_cluster", "filter_field": 0}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "local_cluster", "filter_field": 1}'
+          - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+          - '{"f1": "local_cluster", "filter_field": 0}'
+
+  - do:
+    search:
+      index: test_index,my_remote_cluster:test_index
+      body:
+        aggs:
+          cluster:
+            terms:
+              field: f1.keyword
+
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 11 }
+  - length: { aggregations.cluster.buckets: 2 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+  - match: { aggregations.cluster.buckets.1.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.1.doc_count: 5 }
+
+  - do:
+    search:
+      index: test_index,my_remote_cluster:test_index
+      body:
+        query:
+          term:
+            f1: remote_cluster
+        aggs:
+          cluster:
+            terms:
+              field: f1.keyword
+
+  - match: { _shards.total: 5 }
+  - match: { hits.total: 6}
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+    search:
+      index: my_remote_cluster:test_index
+      body:
+        aggs:
+          cluster:
+            terms:
+              field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6}
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+    search:
+      index: test_index
+      body:
+        aggs:
+          cluster:
+            terms:
+              field: f1.keyword
+
+  - match: { _shards.total: 2 }
+  - match: { hits.total: 5}
+  - match: { hits.hits.0._index: "test_index"}
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "local_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 5 }
+
+---
+"Add transient remote cluster based on the preset cluster":
+  - do:
+    cluster.get_settings:
+      include_defaults: true
+
+  - set: { defaults.search.remote.my_remote_cluster.seeds.0: remote_ip }
+
+  - do:
+    cluster.put_settings:
+      flat_settings: true
+      body:
+        transient:
+          search.remote.test_remote_cluster.seeds: $remote_ip
+
+  - match: {transient: {search.remote.test_remote_cluster.seeds: $remote_ip}}
+
+  - do:
+    search:
+      index: test_remote_cluster:test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - match: { hits.hits.0._index: "test_remote_cluster:test_index" }
+
+---
+"Search an filtered alias on the remote cluster":
+
+  - do:
+    search:
+      index: my_remote_cluster:aliased_test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.filter_field: 1 }
+  - match: { hits.hits.0._index: "my_remote_cluster:test_index" }
diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml
new file mode 100644
index 00000000000..de4ae736f94
--- /dev/null
+++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yaml
@@ -0,0 +1,60 @@
+---
+"Index data and search on the old cluster":
+
+  - do:
+    indices.create:
+      index: test_index
+      body:
+        settings:
+          index:
+            number_of_shards: 3
+            number_of_replicas: 0
+        aliases:
+          aliased_test_index: # we use this alias in the multi cluster test to very filtered aliases work
+            filter:
+              term:
+                filter_field : 1
+
+  - do:
+    bulk:
+       refresh: true
+       body:
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 0}'
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 1}'
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 0}'
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 1}'
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 0}'
+         - '{"index": {"_index": "test_index", "_type": "test_type"}}'
+         - '{"f1": "remote_cluster", "filter_field": 0}'
+
+
+  - do:
+    search:
+      index: test_index
+      body:
+        aggs:
+          cluster:
+            terms:
+              field: f1.keyword
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 6 }
+  - length: { aggregations.cluster.buckets: 1 }
+  - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
+  - match: { aggregations.cluster.buckets.0.doc_count: 6 }
+
+  - do:
+    search:
+      index: aliased_test_index
+
+  - match: { _shards.total: 3 }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.filter_field: 1 }
+  - match: { hits.hits.0._index: "test_index" }
+
+
diff --git a/qa/no-bootstrap-tests/build.gradle b/qa/no-bootstrap-tests/build.gradle
index 16ac5e27693..25731a99dee 100644
--- a/qa/no-bootstrap-tests/build.gradle
+++ b/qa/no-bootstrap-tests/build.gradle
@@ -23,4 +23,3 @@
  */
 
 apply plugin: 'elasticsearch.standalone-test'
-
diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index e17e2454108..0d80c6ffd19 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -25,10 +25,11 @@ task oldClusterTest(type: RestIntegTestTask) {
   mustRunAfter(precommit)
   cluster {
     distribution = 'zip'
-    bwcVersion = '6.0.0-alpha1-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
+    bwcVersion = '5.3.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
     numBwcNodes = 2
     numNodes = 2
     clusterName = 'rolling-upgrade'
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'old_cluster'
 }
@@ -40,6 +41,7 @@ task mixedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'mixed_cluster'
   finalizedBy 'oldClusterTest#node0.stop'
@@ -52,6 +54,7 @@ task upgradedClusterTest(type: RestIntegTestTask) {
     clusterName = 'rolling-upgrade'
     unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
     dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
+    setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
   }
   systemProperty 'tests.rest.suite', 'upgraded_cluster'
   // only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
index f58d618adf3..125566d385a 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
@@ -21,10 +21,10 @@ package org.elasticsearch.upgrades;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -36,12 +36,17 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
         return true;
     }
 
+    @Override
+    protected boolean preserveReposUponCompletion() {
+        return true;
+    }
+
     public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
index a2b40cc54f7..906ae3caf05 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yaml
@@ -26,6 +26,13 @@
           - '{"index": {"_index": "test_index", "_type": "test_type"}}'
           - '{"f1": "v5_mixed", "f2": 9}'
 
+ - do:
+     index:
+       index: test_index
+       type: test_type
+       id: d10
+       body: {"f1": "v6_mixed", "f2": 10}
+
  - do:
      indices.flush:
         index: test_index
@@ -34,4 +41,128 @@
      search:
         index: test_index
 
- - match: { hits.total: 10 } # 5 docs from old cluster, 5 docs from mixed cluster
+ - match: { hits.total: 11 } # 5 docs from old cluster, 6 docs from mixed cluster
+
+ - do:
+     delete:
+       index: test_index
+       type: test_type
+       id: d10
+
+ - do:
+     indices.flush:
+        index: test_index
+
+ - do:
+     search:
+        index: test_index
+
+ - match: { hits.total: 10 }
+
+---
+"Verify custom cluster metadata still exists during upgrade":
+  - do:
+      snapshot.get_repository:
+        repository: my_repo
+  - is_true: my_repo
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
+---
+"Test rolling upgrade for stored scripts between the old namespace and the new namespace":
+  - skip:
+      version: " - 6.0.0"
+      reason: test fails when using bwc version 5.3.0-SNAPSHOT
+      features: warnings
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 2
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "match_all": {
+            }
+          }
+        }
+  - match: { hits.total: 3 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value",
+                "lang": "painless"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
index f1f90cf9d22..96ed75dbed7 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yaml
@@ -32,3 +32,147 @@
         index: test_index
 
  - match: { hits.total: 5 }
+
+---
+"Add stuff to cluster state so that we can verify that it remains to exist during and after the rolling upgrade":
+  - do:
+      snapshot.create_repository:
+        repository: my_repo
+        body:
+          type: url
+          settings:
+            url: "http://snapshot.test"
+  - match: { "acknowledged": true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+            ]
+          }
+  - match: { "acknowledged": true }
+
+---
+"Test rolling upgrade for stored scripts between the old namespace and the new namespace":
+  - skip:
+      version: " - 6.0.0"
+      reason: test fails when using bwc version 5.3.0-SNAPSHOT
+      features: warnings
+
+  - do:
+     indices.create:
+        index: stored_index
+        body:
+          settings:
+            index:
+              number_of_replicas: 0
+
+  - do:
+     bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value1", "num": 1.0}'
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value2", "num": 2.0}'
+          - '{"index": {"_index": "stored_index", "_type": "test"}}'
+          - '{"value": "value3", "num": 3.0}'
+
+  - do:
+     indices.flush:
+        index: stored_index
+
+  - do:
+      put_script:
+        id: "greater"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value > 1.0"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      put_script:
+        id: "value"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      put_script:
+        id: "value"
+        lang: "expression"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value",
+                "lang": "painless"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
index 03dcdc583d3..ef352a5e50d 100644
--- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
+++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yaml
@@ -35,3 +35,441 @@
         index: test_index
 
  - match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs
+
+---
+"Verify custom cluster metadata still exists after rolling upgrade":
+  - do:
+      snapshot.get_repository:
+        repository: my_repo
+  - is_true: my_repo
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { my_pipeline.description: "_description" }
+
+---
+"Test rolling upgrade for stored scripts between the old namespace and the new namespace":
+  - skip:
+      version: " - 6.0.0"
+      reason: test fails when using bwc version 5.3.0-SNAPSHOT
+      features: warnings
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        wait_for_nodes: 2
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "match_all": {
+            }
+          }
+        }
+  - match: { hits.total: 3 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value",
+                "lang": "painless"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated, use request content instead'
+      put_script:
+        id: "greater"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value > 1.0"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+          - 'specifying lang [painless] as part of the url path is deprecated, use request content instead'
+      put_script:
+        id: "value"
+        lang: "painless"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+          - 'specifying lang [expression] as part of the url path is deprecated, use request content instead'
+          - 'stored script [value] already exists using a different lang [painless], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        id: "value"
+        lang: "expression"
+        body: {
+          "script": "doc['num'].value"
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "greater"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "greater" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value > 1.0" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "painless"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { "found": true }
+  - match: { "_id": "value" }
+  - match: { "lang": "expression"}
+  - match: { "script": "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying the field [lang] for executing stored scripts is deprecated; use only the field [stored] to specify an <id>'
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater",
+                "lang": "painless"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value"
+              }
+            },
+            "script_expressions": {
+              "script": {
+                "stored": "value",
+                "lang": "expression"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
+  - match: { hits.hits.0.fields.script_expressions.0: 2.0 }
+  - match: { hits.hits.1.fields.script_expressions.0: 3.0 }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      delete_script:
+        id: "value"
+        lang: "painless"
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "expression"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'stored script [value] already exists using a different lang [expression], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "painless"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "painless"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'stored script [value] already exists using a different lang [painless], the new namespace for stored scripts will only use (id) instead of (lang, id)'
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "expression"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "expression"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "expression"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      delete_script:
+        lang: "value"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      get_script:
+        lang: "value"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      put_script:
+        lang: "value"
+        body: {
+          "script": {
+            "code": "doc['num'].value",
+            "lang": "painless"
+          }
+        }
+  - match: { acknowledged: true }
+
+  - do:
+      get_script:
+        lang: "value"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { script.lang: "painless"}
+  - match: { script.code: "doc['num'].value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [expression] as part of the url path is deprecated'
+      catch: missing
+      get_script:
+        id: "value"
+        lang: "expression"
+  - match: { found: false }
+  - match: { _id: "value" }
+
+  - do:
+      warnings:
+        - 'specifying lang [painless] as part of the url path is deprecated'
+      get_script:
+        id: "value"
+        lang: "painless"
+  - match: { found: true }
+  - match: { _id: "value" }
+  - match: { lang: "painless"}
+  - match: { script: "doc['num'].value" }
+
+  - do:
+      search:
+        index: stored_index
+        body: {
+          "query": {
+            "script": {
+              "script": {
+                "stored": "greater"
+              }
+            }
+          },
+          "script_fields": {
+            "script_painless": {
+              "script": {
+                "stored": "value"
+              }
+            }
+          },
+          "sort": {
+            "num": {
+              "order": "asc"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0.fields.script_painless.0: 2.0 }
+  - match: { hits.hits.1.fields.script_painless.0: 3.0 }
diff --git a/qa/smoke-test-client/build.gradle b/qa/smoke-test-client/build.gradle
index 260516a5bf6..888d9325242 100644
--- a/qa/smoke-test-client/build.gradle
+++ b/qa/smoke-test-client/build.gradle
@@ -17,10 +17,11 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 // TODO: this test works, but it isn't really a rest test...should we have another plugin for "non rest test that just needs N clusters?"
 
 dependencies {
     testCompile project(path: ':client:transport', configuration: 'runtime') // randomly swapped in as a transport
-}
\ No newline at end of file
+}
diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle
index 0bdacc1d48a..d15a4b5d900 100644
--- a/qa/smoke-test-http/build.gradle
+++ b/qa/smoke-test-http/build.gradle
@@ -17,8 +17,10 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
+apply plugin: 'elasticsearch.test-with-dependencies'
 
 dependencies {
     testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http
-}
\ No newline at end of file
+}
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
index 4c39d80a674..752e18dc917 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ContextAndHeaderTransportIT.java
@@ -22,7 +22,6 @@ package org.elasticsearch.http;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -317,7 +316,7 @@ public class ContextAndHeaderTransportIT extends HttpSmokeTestCase {
         }
 
         @Override
-        protected boolean apply(String action, ActionRequest request, ActionListener listener) {
+        protected boolean apply(String action, ActionRequest request, ActionListener<?> listener) {
             requests.add(new RequestAndHeaders(threadPool.getThreadContext().getHeaders(), request));
             return true;
         }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
index 2cf681e9e5a..380937ed010 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsDisabledIT.java
@@ -19,18 +19,18 @@
 
 package org.elasticsearch.http;
 
+import java.io.IOException;
+import java.util.Collections;
+
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
-import java.io.IOException;
-import java.util.Collections;
-
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 
 /**
@@ -49,14 +49,13 @@ public class DetailedErrorsDisabledIT extends HttpSmokeTestCase {
     }
 
     public void testThatErrorTraceParamReturns400() throws IOException {
-        try {
-            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"));
-            fail("request should have failed");
-        } catch(ResponseException e) {
-            Response response = e.getResponse();
-            assertThat(response.getHeader("Content-Type"), is("application/json"));
-            assertThat(EntityUtils.toString(e.getResponse().getEntity()), is("{\"error\":\"error traces in responses are disabled.\"}"));
-            assertThat(response.getStatusLine().getStatusCode(), is(400));
-        }
+        ResponseException e = expectThrows(ResponseException.class, () ->
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true")));
+
+        Response response = e.getResponse();
+        assertThat(response.getHeader("Content-Type"), is("application/json; charset=UTF-8"));
+        assertThat(EntityUtils.toString(e.getResponse().getEntity()),
+                   containsString("\"error\":\"error traces in responses are disabled.\""));
+        assertThat(response.getStatusLine().getStatusCode(), is(400));
     }
 }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
index cd3cf1a38d8..d0b80595a26 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/DetailedErrorsEnabledIT.java
@@ -22,7 +22,6 @@ package org.elasticsearch.http;
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -52,7 +51,7 @@ public class DetailedErrorsEnabledIT extends HttpSmokeTestCase {
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
-            assertThat(response.getHeader("Content-Type"), containsString("application/json"));
+            assertThat(response.getHeader("Content-Type"), containsString("application/json; charset=UTF-8"));
             assertThat(EntityUtils.toString(response.getEntity()),
                     not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
                     + "nested: ActionRequestValidationException[Validation Failed: 1:")));
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
index ce0a79691a7..bc60b7aa574 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationHeaderRestAction.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -69,7 +68,6 @@ public class TestDeprecationHeaderRestAction extends BaseRestHandler {
     public static final String DEPRECATED_ENDPOINT = "[/_test_cluster/deprecated_settings] exists for deprecated tests";
     public static final String DEPRECATED_USAGE = "[deprecated_settings] usage is deprecated. use [settings] instead";
 
-    @Inject
     public TestDeprecationHeaderRestAction(Settings settings, RestController controller) {
         super(settings);
 
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
index d047713f0c7..a264227999e 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestDeprecationPlugin.java
@@ -18,15 +18,23 @@
  */
 package org.elasticsearch.http;
 
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
@@ -36,8 +44,10 @@ import static java.util.Collections.singletonList;
 public class TestDeprecationPlugin extends Plugin implements ActionPlugin, SearchPlugin {
 
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return Collections.singletonList(TestDeprecationHeaderRestAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return Collections.singletonList(new TestDeprecationHeaderRestAction(settings, restController));
     }
 
     @Override
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
index 398f990a5cf..421c6d9bade 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderPlugin.java
@@ -19,17 +19,27 @@
 
 package org.elasticsearch.http;
 
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.List;
+import java.util.function.Supplier;
 
 import static java.util.Collections.singletonList;
 
 public class TestResponseHeaderPlugin extends Plugin implements ActionPlugin {
     @Override
-    public List<Class<? extends RestHandler>> getRestHandlers() {
-        return singletonList(TestResponseHeaderRestAction.class);
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+            IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier<DiscoveryNodes> nodesInCluster) {
+        return singletonList(new TestResponseHeaderRestAction(settings, restController));
     }
 }
diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
index a38fd3d015e..d99dd5d5245 100644
--- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
+++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/TestResponseHeaderRestAction.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.http;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -31,8 +30,6 @@ import org.elasticsearch.rest.RestStatus;
 import java.io.IOException;
 
 public class TestResponseHeaderRestAction extends BaseRestHandler {
-
-    @Inject
     public TestResponseHeaderRestAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_protected", this);
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index 08dfbf8ae7a..4c4d9c2da12 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
diff --git a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
index beb7499bf7f..7f4c2c4a4d5 100644
--- a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestIngestDisabledClientYamlTestSuiteIT extends ESClientYamlSu
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index df90bf5b982..2cfa3af434e 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
index 0bd7b9ac029..d1e1adabfd0 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestIngestWithAllDepsClientYamlTestSuiteIT extends ESClientYam
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
index c87441472f2..729347ee197 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
@@ -73,13 +73,18 @@
 
 ---
 "Test script processor with stored script":
+  - skip:
+      features: warnings
+
   - do:
       put_script:
-        id: "sum_bytes"
-        lang: "painless"
+        lang: "sum_bytes"
         body: >
           {
-            "script" : "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"
+            "script": {
+              "lang": "painless",
+              "code" : "ctx.bytes_total = ctx.bytes_in + ctx.bytes_out"
+            }
           }
   - match: { acknowledged: true }
 
diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle
index f39f790ba09..fc196fd52a4 100644
--- a/qa/smoke-test-multinode/build.gradle
+++ b/qa/smoke-test-multinode/build.gradle
@@ -1,4 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 integTest {
diff --git a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
index 225e12f65fd..69abd0b3cc2 100644
--- a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java
@@ -21,12 +21,11 @@ package org.elasticsearch.smoketest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -38,7 +37,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle
index ab69b02fc8c..6fd722e409c 100644
--- a/qa/smoke-test-plugins/build.gradle
+++ b/qa/smoke-test-plugins/build.gradle
@@ -19,6 +19,7 @@
 
 import org.elasticsearch.gradle.MavenFilteringHack
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 ext.pluginsCount = 0
@@ -40,4 +41,3 @@ processTestResources {
   inputs.properties(expansions)
   MavenFilteringHack.filter(it, expansions)
 }
-
diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
index 0f5636095e9..2ae7be8fb1e 100644
--- a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -35,7 +34,7 @@ public class SmokeTestPluginsClientYamlTestSuiteIT extends ESClientYamlSuiteTest
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-reindex-with-painless/build.gradle b/qa/smoke-test-reindex-with-painless/build.gradle
index c857db85bfa..7092c0a7b48 100644
--- a/qa/smoke-test-reindex-with-painless/build.gradle
+++ b/qa/smoke-test-reindex-with-painless/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 integTest {
diff --git a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
index fb4f27210d8..5366eaf4bd1 100644
--- a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java
@@ -24,7 +24,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -34,7 +33,7 @@ public class SmokeTestReindexWithPainlessClientYamlTestSuiteIT extends ESClientY
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return ESClientYamlSuiteTestCase.createParameters();
     }
 }
diff --git a/qa/smoke-test-tribe-node/build.gradle b/qa/smoke-test-tribe-node/build.gradle
index 6e108e87043..94789b17fdb 100644
--- a/qa/smoke-test-tribe-node/build.gradle
+++ b/qa/smoke-test-tribe-node/build.gradle
@@ -21,6 +21,7 @@ import org.elasticsearch.gradle.test.ClusterConfiguration
 import org.elasticsearch.gradle.test.ClusterFormationTasks
 import org.elasticsearch.gradle.test.NodeInfo
 
+apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
 List<NodeInfo> oneNodes
diff --git a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
index 6013913bdc4..211043ed4b0 100644
--- a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
+++ b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java
@@ -21,9 +21,9 @@ package org.elasticsearch.tribe;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
 
 import java.io.IOException;
 
@@ -46,7 +46,7 @@ public class TribeClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     }
 
     @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> parameters() throws IOException {
         return createParameters();
     }
 
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
index 7a5ba032951..0168329eca5 100644
--- a/qa/vagrant/versions
+++ b/qa/vagrant/versions
@@ -2,3 +2,5 @@
 5.0.1
 5.0.2
 5.1.1
+5.1.2
+5.2.0
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json b/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json
new file mode 100644
index 00000000000..fec522be4a1
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json
@@ -0,0 +1,29 @@
+{
+  "description": "Parameters that are accepted by all API endpoints.",
+  "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html",
+  "params": {
+    "pretty": {
+      "type": "boolean",
+      "description": "Pretty format the returned JSON response.",
+      "default": false
+    },
+    "human": {
+      "type": "boolean",
+      "description": "Return human readable values for statistics.",
+      "default": true
+    },
+    "error_trace": {
+      "type": "boolean",
+      "description": "Include the stack trace of returned errors.",
+      "default": false
+    },
+    "source": {
+      "type": "string",
+      "description": "The URL-encoded request definition. Useful for libraries that do not accept a request body for non-POST requests."
+    },
+    "filter_path": {
+      "type": "string",
+      "description": "A comma-separated list of filters used to reduce the respone."
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
index 2bccb20f36e..36400ed2f7c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
@@ -16,11 +16,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json
deleted file mode 100644
index 584f33685d3..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count_percolate.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
-  "count_percolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
-    "methods": ["GET", "POST"],
-    "url": {
-      "path": "/{index}/{type}/_percolate/count",
-      "paths": ["/{index}/{type}/_percolate/count", "/{index}/{type}/{id}/_percolate/count"],
-      "parts": {
-        "index": {
-          "type": "string",
-          "required": true,
-          "description": "The index of the document being count percolated."
-        },
-        "type": {
-          "type": "string",
-          "required": true,
-          "description": "The type of the document being count percolated."
-        },
-        "id": {
-          "type": "string",
-          "required": false,
-          "description": "Substitute the document in the request body with a document that is known by the specified id. On top of the id, the index and type parameter will be used to retrieve the document from within the cluster."
-        }
-      },
-      "params": {
-        "routing": {
-          "type": "list",
-          "description": "A comma-separated list of specific routing values"
-        },
-        "preference": {
-          "type": "string",
-          "description": "Specify the node or shard the operation should be performed on (default: random)"
-        },
-        "ignore_unavailable": {
-          "type": "boolean",
-          "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-        },
-        "allow_no_indices": {
-          "type": "boolean",
-          "description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-        },
-        "expand_wildcards": {
-          "type": "enum",
-          "options": ["open", "closed","none","all"],
-          "default": "open",
-          "description": "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "percolate_index": {
-          "type": "string",
-          "description": "The index to count percolate the document into. Defaults to index."
-        },
-        "percolate_type": {
-          "type": "string",
-          "description": "The type to count percolate document into. Defaults to type."
-        },
-        "version": {
-          "type": "number",
-          "description": "Explicit version number for concurrency control"
-        },
-        "version_type": {
-          "type": "enum",
-          "options": ["internal", "external", "external_gte", "force"],
-          "description": "Specific version type"
-        }
-      }
-    },
-    "body": {
-      "description": "The count percolator request definition using the percolate DSL",
-      "required": false
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
index 2c9742057fe..7be743e8e78 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
@@ -3,8 +3,8 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
     "methods": ["DELETE"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
index 163f147aa42..c253c6e1fdb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script.json
@@ -3,8 +3,8 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
     "methods": ["GET"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index ed96200687c..e58af6d50b2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -40,11 +40,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-          "type": "boolean",
-          "description": "Whether to return version and creation date values in human-readable format.",
-          "default": false
-        },
         "include_defaults": {
           "type": "boolean",
           "description": "Whether to return all default setting for each of the indices.",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index 17a1cd03118..706cce5277a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -38,11 +38,6 @@
           "type": "boolean",
           "description": "Return local information, do not retrieve the state from master node (default: false)"
         },
-        "human": {
-          "type": "boolean",
-          "description": "Whether to return version and creation date values in human-readable format.",
-          "default": false
-        },
         "include_defaults": {
           "type": "boolean",
           "description": "Whether to return all default setting for each of the indices.",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
index 63d9bbf418b..3c114a450ee 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_upgrade.json
@@ -25,11 +25,6 @@
             "options" : ["open","closed","none","all"],
             "default" : "open",
             "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
         }
       }
     },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
index 2b94f1ad3d6..80b7f1a0009 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.recovery.json
@@ -21,11 +21,6 @@
                     "type": "boolean",
                     "description": "Display only those recoveries that are currently on-going",
                     "default": false
-                },
-                "human": {
-                    "type": "boolean",
-                    "description": "Whether to return time and byte values in human-readable format.",
-                    "default": false
                 }
             }
         },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
index cc51bdab2c3..e4d87d4384f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.segments.json
@@ -26,11 +26,6 @@
             "default" : "open",
             "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "operation_threading": {
           "description" : "TODO: ?"
         },
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index de1e9246f7b..223e5a14ff6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -17,7 +17,7 @@
         },
         "metric" : {
           "type" : "list",
-          "options" : ["_all", "completion", "docs", "fielddata", "query_cache", "flush", "get", "indexing", "merge", "percolate", "request_cache", "refresh", "search", "segments", "store", "warmer", "suggest"],
+          "options" : ["_all", "completion", "docs", "fielddata", "query_cache", "flush", "get", "indexing", "merge", "request_cache", "refresh", "search", "segments", "store", "warmer", "suggest"],
           "description" : "Limit the information returned the specific metrics."
         }
       },
@@ -38,11 +38,6 @@
           "type" : "list",
           "description" : "A comma-separated list of search groups for `search` index metric"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "level": {
           "type" : "enum",
           "description": "Return stats aggregated at cluster, index or shard level",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json
deleted file mode 100644
index 7cbf4f61e43..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/mpercolate.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
-  "mpercolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
-    "methods": ["GET", "POST"],
-    "url": {
-      "path": "/_mpercolate",
-      "paths": ["/_mpercolate", "/{index}/_mpercolate", "/{index}/{type}/_mpercolate"],
-      "parts": {
-        "index": {
-          "type": "string",
-          "description": "The index of the document being count percolated to use as default"
-        },
-        "type": {
-          "type" : "string",
-          "description" : "The type of the document being percolated to use as default."
-        }
-      },
-      "params": {
-        "ignore_unavailable": {
-          "type": "boolean",
-          "description": "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-        },
-        "allow_no_indices": {
-          "type": "boolean",
-          "description": "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-        },
-        "expand_wildcards": {
-          "type": "enum",
-          "options": ["open", "closed","none","all"],
-          "default": "open",
-          "description": "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        }
-      }
-    },
-    "body": {
-      "description": "The percolate request definitions (header & body pair), separated by newlines",
-      "required": true,
-      "serialize" : "bulk"
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
index 12f0d11c5fc..c86d35dc1a9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
@@ -21,11 +21,6 @@
           "type": "boolean",
           "description": "Return settings in flat format (default: false)"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
index a5910c9f328..7b426d03918 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
@@ -20,7 +20,7 @@
         },
         "index_metric" : {
           "type" : "list",
-          "options" : ["_all", "completion", "docs", "fielddata", "query_cache", "flush", "get", "indexing", "merge", "percolate", "request_cache", "refresh", "search", "segments", "store", "warmer", "suggest"],
+          "options" : ["_all", "completion", "docs", "fielddata", "query_cache", "flush", "get", "indexing", "merge", "request_cache", "refresh", "search", "segments", "store", "warmer", "suggest"],
           "description" : "Limit the information returned for `indices` metric to the specific index metrics. Isn't used if `indices` (or `all`) metric isn't specified."
         },
         "node_id": {
@@ -45,11 +45,6 @@
           "type" : "boolean",
           "description" : "A comma-separated list of search groups for `search` index metric"
         },
-        "human": {
-            "type": "boolean",
-            "description": "Whether to return time and byte values in human-readable format.",
-            "default": false
-        },
         "level": {
           "type" : "enum",
           "description": "Return indices stats aggregated at index, node or shard level",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json
deleted file mode 100644
index e58655dea5a..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/percolate.json
+++ /dev/null
@@ -1,85 +0,0 @@
-{
-  "percolate": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/search-percolate.html",
-    "methods": ["GET", "POST"],
-    "url": {
-      "path": "/{index}/{type}/_percolate",
-      "paths": ["/{index}/{type}/_percolate", "/{index}/{type}/{id}/_percolate"],
-      "parts": {
-        "index": {
-          "type" : "string",
-          "required" : true,
-          "description" : "The index of the document being percolated."
-        },
-        "type": {
-          "type" : "string",
-          "required" : true,
-          "description" : "The type of the document being percolated."
-        },
-        "id": {
-          "type" : "string",
-          "required" : false,
-          "description" : "Substitute the document in the request body with a document that is known by the specified id. On top of the id, the index and type parameter will be used to retrieve the document from within the cluster."
-        }
-      },
-      "params": {
-        "routing": {
-          "type" : "list",
-          "description" : "A comma-separated list of specific routing values"
-        },
-        "preference": {
-          "type" : "string",
-          "description" : "Specify the node or shard the operation should be performed on (default: random)"
-        },
-        "ignore_unavailable": {
-          "type" : "boolean",
-          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-        },
-        "allow_no_indices": {
-          "type" : "boolean",
-          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-        },
-        "expand_wildcards": {
-          "type" : "enum",
-          "options" : ["open","closed","none","all"],
-          "default" : "open",
-          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "percolate_index": {
-          "type" : "string",
-          "description" : "The index to percolate the document into. Defaults to index."
-        },
-        "percolate_type": {
-          "type" : "string",
-          "description" : "The type to percolate document into. Defaults to type."
-        },
-        "percolate_routing": {
-          "type" : "string",
-          "description" : "The routing value to use when percolating the existing document."
-        },
-        "percolate_preference": {
-          "type" : "string",
-          "description" : "Which shard to prefer when executing the percolate request."
-        },
-        "percolate_format": {
-          "type" : "enum",
-          "options" : ["ids"],
-          "description" : "Return an array of matching query IDs instead of objects"
-        },
-        "version" : {
-          "type" : "number",
-          "description" : "Explicit version number for concurrency control"
-        },
-        "version_type": {
-          "type" : "enum",
-          "options" : ["internal", "external", "external_gte", "force"],
-          "description" : "Specific version type"
-        }
-      }
-    },
-    "body": {
-      "description" : "The percolator request definition using the percolate DSL",
-      "required" : false
-    }
-  }
-}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
index c5676392625..93fef4c030b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
@@ -3,8 +3,8 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/_scripts/{lang}/{id}",
-      "paths": [ "/_scripts/{lang}/{id}" ],
+      "path": "/_scripts/{lang}",
+      "paths": [ "/_scripts/{lang}", "/_scripts/{lang}/{id}" ],
       "parts": {
         "id": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml
index 4906975bfab..6326b9464ca 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml
@@ -4,14 +4,14 @@
       bulk:
         refresh: true
         body: |
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"index": {"_index": "bulk_50_refresh_1", "_type": "test_type", "_id": "bulk_50_refresh_id1"}}
           {"f1": "v1", "f2": 42}
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"index": {"_index": "bulk_50_refresh_1", "_type": "test_type", "_id": "bulk_50_refresh_id2"}}
           {"f1": "v2", "f2": 47}
 
   - do:
       count:
-        index: test_index
+        index: bulk_50_refresh_1
   - match: {count: 2}
 
 ---
@@ -20,14 +20,14 @@
       bulk:
         refresh: ""
         body: |
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"index": {"_index": "bulk_50_refresh_2", "_type": "test_type", "_id": "bulk_50_refresh_id3"}}
           {"f1": "v1", "f2": 42}
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"index": {"_index": "bulk_50_refresh_2", "_type": "test_type", "_id": "bulk_50_refresh_id4"}}
           {"f1": "v2", "f2": 47}
 
   - do:
       count:
-        index: test_index
+        index: bulk_50_refresh_2
   - match: {count: 2}
 
 
@@ -37,12 +37,12 @@
       bulk:
         refresh: wait_for
         body: |
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"index": {"_index": "bulk_50_refresh_3", "_type": "test_type", "_id": "bulk_50_refresh_id5"}}
           {"f1": "v1", "f2": 42}
-          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"index": {"_index": "bulk_50_refresh_3", "_type": "test_type", "_id": "bulk_50_refresh_id6"}}
           {"f1": "v2", "f2": 47}
 
   - do:
       count:
-        index: test_index
+        index: bulk_50_refresh_3
   - match: {count: 2}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
index 9225ad69ea4..380caec9318 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.health/10_basic.yaml
@@ -57,7 +57,7 @@
 
   - do:
       cat.health:
-        ts: 0
+        ts: false
 
   - match:
       $body: |
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
index 9f4de56c863..f48f73cb478 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml
@@ -61,8 +61,8 @@
 ---
 "Test cat nodes output with full_id set":
   - skip:
-    version: " - 5.0.0"
-    reason:  The full_id setting was rejected in 5.0.0 see #21266
+      version: " - 5.0.0"
+      reason:  The full_id setting was rejected in 5.0.0 see #21266
 
 
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
index 8c7cd83b0e0..f0cb8117cb9 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
@@ -1,5 +1,9 @@
 ---
 "Help":
+  - skip:
+      version: " - 5.99.99"
+      reason: seq no stats were added in 6.0.0
+
   - do:
       cat.shards:
         help: true
@@ -51,6 +55,7 @@
                     merges.total_time                .+   \n
                     refresh.total                    .+   \n
                     refresh.time                     .+   \n
+                    refresh.listeners                .+   \n
                     search.fetch_current             .+   \n
                     search.fetch_time                .+   \n
                     search.fetch_total               .+   \n
@@ -164,6 +169,13 @@
       $body: |
                /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/
 
+---
+"Test cat shards with shadow replicas":
+  - skip:
+      version: " - 5.1.99"
+      reason: deprecation was added in 5.2.0
+      features: "warnings"
+
   - do:
       indices.create:
         index: index3
@@ -173,6 +185,9 @@
             number_of_replicas: "1"
             shadow_replicas: true
             shared_filesystem: false
+      warnings:
+        - "[index.shadow_replicas] setting was deprecated in Elasticsearch and it will be removed in a future release! See the breaking changes lists in the documentation for details"
+        - "[index.shared_filesystem] setting was deprecated in Elasticsearch and it will be removed in a future release! See the breaking changes lists in the documentation for details"
 
   - do:
       cat.shards:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
index f0195788e07..6ec8651fe8d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cluster.allocation_explain/10_basic.yaml
@@ -1,45 +1,44 @@
 ---
 "cluster shard allocation explanation test":
+  - skip:
+      version: " - 5.1.99"
+      reason: explain API response output was changed starting in 5.2.0
+
   - do:
       # there aren't any unassigned shards to explain
-      catch: /unable to find any shards to explain/
+      catch: /illegal_state_exception/
       cluster.allocation_explain: {}
 
   - do:
       indices.create:
         index: test
 
+  - match: { acknowledged: true }
+
   - do:
       cluster.state:
         metric: [ master_node ]
 
-  - set: {master_node: node_id}
-
-    # This relies on there only being a single node in the test cluster, which
-    # is currently true, but if this changes in the future this test will need
-    # to be changed
   - do:
       cluster.allocation_explain:
         body: { "index": "test", "shard": 0, "primary": true }
 
-  - match: { assigned: true }
-  # - match: { assigned_node_id: $node_id }
-  - is_true: assigned_node_id
-  - match: { shard.index: "test" }
-  - match: { shard.id: 0 }
-  - match: { shard.primary: true }
-  # unfortunately can't test these because they break with multi-node backwords
-  # compat REST tests
-  # - is_true: nodes.$node_id.node_name
-  # - match: { nodes.$node_id.node_attributes.testattr: "test" }
-  # - match: { nodes.$node_id.node_attributes.portsfile: "true" }
-  # - match: { nodes.$node_id.final_decision: "CURRENTLY_ASSIGNED" }
-  # - match: { nodes.$node_id.weight: 0.0 }
-  # - match: { nodes.$node_id.decisions.0.decider: "same_shard" }
-  # - match: { nodes.$node_id.decisions.0.decision: "NO" }
+  - match: { current_state: "started" }
+  - is_true: current_node.id
+  - match: { index: "test" }
+  - match: { shard: 0 }
+  - match: { primary: true }
+  - is_true: can_remain_on_current_node
+  - is_true: can_rebalance_cluster
+  - is_true: can_rebalance_to_other_node
+  - is_true: rebalance_explanation
 
 ---
 "cluster shard allocation explanation test with empty request":
+  - skip:
+      version: " - 5.1.99"
+      reason: explain API response output was changed starting in 5.2.0
+
   - do:
       indices.create:
         index: test
@@ -49,22 +48,15 @@
       cluster.state:
         metric: [ master_node ]
 
-  - set: {master_node: node_id}
-
   - do:
       cluster.allocation_explain:
         include_disk_info: true
 
-  - match: { assigned: false }
+  - match: { current_state: "unassigned" }
   - match: { unassigned_info.reason: "INDEX_CREATED" }
   - is_true: unassigned_info.at
-  - match: { shard.index: "test" }
-  - match: { shard.id: 0 }
-  - match: { shard.primary: false }
+  - match: { index: "test" }
+  - match: { shard: 0 }
+  - match: { primary: false }
   - is_true: cluster_info
-  # - is_true: nodes.$node_id.node_name
-  # - match: { nodes.$node_id.node_attributes.testattr: "test" }
-  # - match: { nodes.$node_id.node_attributes.portsfile: "true" }
-  # - match: { nodes.$node_id.final_decision: "NO" }
-  # - match: { nodes.$node_id.decisions.0.decider: "same_shard" }
-  # - match: { nodes.$node_id.decisions.0.decision: "NO" }
+  - is_true: can_allocate
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
index 70f402691a3..2c8965f5c6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml
@@ -6,8 +6,6 @@
           body:
             mappings:
               test:
-                _all:
-                  enabled: false
                 properties:
                   number:
                     type: integer
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
index 5cdc1a20583..deb5e1e6850 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
@@ -66,17 +66,17 @@
 "refresh=wait_for waits until changes are visible in search":
  - do:
       index:
-          index:   test_1
+          index:   create_60_refresh_1
           type:    test
-          id:      1
+          id:      create_60_refresh_id1
           body:    { foo: bar }
           refresh: wait_for
  - is_false: forced_refresh
 
  - do:
       search:
-          index:   test_1
+          index:   create_60_refresh_1
           type:    test
           body:
-            query:   { term: { _id: 1 }}
+            query:   { term: { _id: create_60_refresh_id1 }}
  - match: { hits.total: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
index 9ea6bc033de..589d2cf77a3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
@@ -115,33 +115,33 @@
 "refresh=wait_for waits until changes are visible in search":
  - do:
       index:
-          index:   test_1
+          index:   delete_50_refresh_1
           type:    test
-          id:      1
+          id:      delete_50_refresh_id1
           body:    { foo: bar }
           refresh: true
  - is_true: forced_refresh
 
  - do:
       search:
-          index:   test_1
+          index:   delete_50_refresh_1
           type:    test
           body:
-            query:   { term: { _id: 1 }}
+            query:   { term: { _id: delete_50_refresh_id1 }}
  - match: { hits.total: 1 }
 
  - do:
      delete:
-         index:   test_1
+         index:   delete_50_refresh_1
          type:    test
-         id:      1
+         id:      delete_50_refresh_id1
          refresh: wait_for
  - is_false: forced_refresh
 
  - do:
       search:
-          index:   test_1
+          index:   delete_50_refresh_1
           type:    test
           body:
-            query:   { term: { _id: 1 }}
+            query:   { term: { _id: delete_50_refresh_id1 }}
  - match: { hits.total: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
index aa45376b52f..df8c697e4a1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/exists/60_realtime_refresh.yaml
@@ -26,7 +26,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - is_false: ''
  
@@ -35,7 +35,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: ''
 
@@ -44,7 +44,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - is_true: ''
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
index 78ef8c4bc89..5ccf97891af 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml
@@ -6,8 +6,6 @@
           body:
             mappings:
               test:
-                _all:
-                  enabled: false
                 properties:
                   number:
                     type: integer
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
index 35cc19224ec..4e6cbd61623 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
@@ -12,6 +12,12 @@ setup:
                     type: long
                   bar:
                     type: long
+                  geo:
+                    type: geo_point
+                  geo_shape:
+                    type: geo_shape
+                    tree: quadtree
+                    precision: 1m
 
   - do:
       indices.create:
@@ -26,20 +32,27 @@ setup:
                     type: long
                   bar:
                     type: text
+                  geo:
+                    type: geo_point
+                    index: no
+                  geo_shape:
+                    type: geo_shape
+                    tree: quadtree
+                    precision: 1m
 
   - do:
       index:
           index:  test_1
           type:   test
           id:     id_1
-          body:   { foo: "bar", number: 123, bar: 123 }
+          body:   { foo: "bar", number: 123, bar: 123, geo: { lat: 48.858093, lon: 2.294694} }
 
   - do:
       index:
           index:  test_2
           type:   test
           id:     id_10
-          body: { foo: "babar", number: 456, bar: "123" }
+          body: { foo: "babar", number: 456, bar: "123", geo: { lat: 48.858093, lon: 2.294694},  geo_shape:  {type: "linestring", coordinates : [[-77.03653, 38.897676], [-77.009051, 38.889939]] } }
 
   - do:
       indices.refresh: {}
@@ -68,6 +81,37 @@ setup:
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
   - match: { indices._all.fields.number.type: "integer" }
+
+  - is_false: conflicts
+
+---
+"Geo field stats":
+  - skip:
+      version: " - 5.3.0"
+      reason:  geo_point fields don't return min/max for versions greater than 5.2.0
+
+  - do:
+      field_stats:
+          fields:  [geo, geo_shape]
+
+  - match: { indices._all.fields.geo.type: "geo_point" }
+  - match: { indices._all.fields.geo.max_doc: 2 }
+  - match: { indices._all.fields.geo.doc_count: -1 }
+  - match: { indices._all.fields.geo.searchable: true }
+  - match: { indices._all.fields.geo.aggregatable: true }
+  - is_false: indices._all.fields.geo.min_value
+  - is_false: indices._all.fields.geo.max_value
+  - is_false: indices._all.fields.geo.min_value_as_string
+  - is_false: indices._all.fields.geo.max_value_as_string
+  - match: { indices._all.fields.geo_shape.type: "string" }
+  - match: { indices._all.fields.geo_shape.max_doc: 1 }
+  - match: { indices._all.fields.geo_shape.doc_count: -1 }
+  - match: { indices._all.fields.geo_shape.searchable: true }
+  - match: { indices._all.fields.geo_shape.aggregatable: false }
+  - is_false: indices._all.fields.geo_shape.min_value
+  - is_false: indices._all.fields.geo_shape.max_value
+  - is_false: indices._all.fields.geo_shape.min_value_as_string
+  - is_false: indices._all.fields.geo_shape.max_value_as_string
   - is_false: conflicts
 
 ---
@@ -116,6 +160,60 @@ setup:
   - match: { indices.test_2.fields.number.type: "integer" }
   - is_false: conflicts
 
+
+---
+"Geo field stats with level set to indices":
+  - skip:
+      version: " - 5.3.0"
+      reason:  geo_point fields don't return min/max for versions greater than 5.2.0
+
+  - do:
+      field_stats:
+          fields:  [geo, geo_shape]
+          level: indices
+
+  - match: { indices.test_1.fields.geo.max_doc: 1 }
+  - match: { indices.test_1.fields.geo.doc_count: 1 }
+  - is_true: indices.test_1.fields.geo.min_value
+  - is_true: indices.test_1.fields.geo.max_value
+  - is_true: indices.test_1.fields.geo.min_value_as_string
+  - is_true: indices.test_1.fields.geo.max_value_as_string
+  - match: { indices.test_1.fields.geo.searchable: true }
+  - match: { indices.test_1.fields.geo.aggregatable: true }
+  - match: { indices.test_1.fields.geo.type: "geo_point" }
+  - is_true: indices.test_2.fields.geo
+  - match: { indices.test_2.fields.geo_shape.max_doc: 1 }
+  - match: { indices.test_2.fields.geo_shape.doc_count: -1 }
+  - is_false: indices.test_2.fields.geo_shape.min_value
+  - is_false: indices.test_2.fields.geo_shape.max_value
+  - is_false: indices.test_2.fields.geo_shape.min_value_as_string
+  - is_false: indices.test_2.fields.geo_shape.max_value_as_string
+  - match: { indices.test_2.fields.geo_shape.searchable: true }
+  - match: { indices.test_2.fields.geo_shape.aggregatable: false }
+  - match: { indices.test_2.fields.geo_shape.type: "string" }
+  - is_false: conflicts
+
+---
+"Geopoint field stats":
+  - skip:
+      version: " - 5.3.0"
+      reason:  geo_point type not handled for versions earlier than 6.0.0
+
+  - do:
+      field_stats:
+          fields:  [geo]
+          level: indices
+
+  - match: { indices.test_2.fields.geo.max_doc: 1 }
+  - match: { indices.test_2.fields.geo.doc_count: -1 }
+  - is_false: indices.test_2.fields.geo.min_value
+  - is_false: indices.test_2.fields.geo.max_value
+  - match: { indices.test_2.fields.geo.searchable: false }
+  - match: { indices.test_2.fields.geo.aggregatable: true }
+  - match: { indices.test_2.fields.geo.type: "geo_point" }
+  - is_true: indices.test_2.fields.geo
+  - is_false: conflicts
+
 ---
 "Field stats with filtering":
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
index 0c705de5e7e..7d02b4667ef 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/60_realtime_refresh.yaml
@@ -27,14 +27,14 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - do:
       get:
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: found
 
@@ -43,7 +43,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - is_true: found
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
index 20009c87b28..fbf3948d7b1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/60_realtime_refresh.yaml
@@ -27,14 +27,14 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - do:
       get_source:
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - match:   { '': {foo: bar}}
 
@@ -43,8 +43,8 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
 
  - match:   { '': {foo: bar}}
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
index 2175128debb..5d20406abee 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
@@ -67,17 +67,17 @@
 "refresh=wait_for waits until changes are visible in search":
  - do:
       index:
-          index:   test_1
+          index:   index_60_refresh_1
           type:    test
-          id:      1
+          id:      index_60_refresh_id1
           body:    { foo: bar }
           refresh: wait_for
  - is_false: forced_refresh
 
  - do:
       search:
-          index:   test_1
+          index:   index_60_refresh_1
           type:    test
           body:
-            query:   { term: { _id: 1 }}
+            query:   { term: { _id: index_60_refresh_id1 }}
  - match: { hits.total: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_template/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_template/10_basic.yaml
index 4e21b818d62..0746c9e805a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_template/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_template/10_basic.yaml
@@ -12,8 +12,8 @@ setup:
 "Get template":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.get_template:
@@ -26,8 +26,8 @@ setup:
 "Get all templates":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
@@ -47,8 +47,8 @@ setup:
 "Get template with local flag":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.get_template:
@@ -61,8 +61,8 @@ setup:
 "Get template with flat settings and master timeout":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.get_template:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
index 8c5ba1c8d5e..01bd7afc582 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yaml
@@ -2,8 +2,8 @@
 "Put template":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
@@ -26,8 +26,8 @@
 "Put multiple template":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
@@ -50,8 +50,8 @@
 "Put template with aliases":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
@@ -78,8 +78,8 @@
 "Put template create":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
@@ -114,8 +114,8 @@
 "Test Put Versioned Template":
 
   - skip:
-    version: " - 5.99.99"
-    reason:  this uses a new API that has been added in 6.0
+      version: " - 5.99.99"
+      reason:  this uses a new API that has been added in 6.0
 
   - do:
       indices.put_template:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
index 9d2245b4b40..abb30eb970e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.upgrade/10_basic.yaml
@@ -19,8 +19,8 @@
 ---
 "Upgrade indices ignore unavailable":
   - skip:
-    version: " - 5.0.0"
-    reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
 
   - do:
       indices.create:
@@ -44,8 +44,8 @@
 "Upgrade indices allow no indices":
 
   - skip:
-    version: " - 5.0.0"
-    reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
 
   - do:
       indices.upgrade:
@@ -59,8 +59,8 @@
 "Upgrade indices disallow no indices":
 
   - skip:
-    version: " - 5.0.0"
-    reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
 
   - do:
       catch: missing
@@ -73,8 +73,8 @@
 "Upgrade indices disallow unavailable":
 
   - skip:
-    version: " - 5.0.0"
-    reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
+      version: " - 5.0.0"
+      reason: ignore_unavailable was added as a bugfix in 5.0.1 see #21281
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
index 3f96009c12a..3b5418169aa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml
@@ -6,8 +6,6 @@
           body:
             mappings:
               test:
-                _all:
-                  enabled: false
                 properties:
                   field:
                     type: text
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
index c688b4b0e93..0cb7b71cf43 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/mget/60_realtime_refresh.yaml
@@ -25,7 +25,7 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 0
+          realtime: false
           body:
             ids:    [1]
 
@@ -35,7 +35,7 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 1
+          realtime: true
           body:
             ids:    [1]
 
@@ -45,8 +45,8 @@
       mget:
           index:    test_1
           type:     test
-          realtime: 0
-          refresh:  1
+          realtime: false
+          refresh:  true
           body:
             ids:    [1]
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index c52723a1413..52379390d47 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -18,9 +18,26 @@ setup:
                     type: long
                   double:
                     type: double
+                  number:
+                    type: long
+                  scaled_float:
+                    type: scaled_float
+                    scaling_factor: 100
                   date:
                     type: date
 
+  - do:
+      indices.create:
+          index: test_2
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  number:
+                    type: double
+
   - do:
       cluster.health:
         wait_for_status: green
@@ -140,6 +157,7 @@ setup:
   - do:
       catch: request
       search:
+        index: test_1
         body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
 
 
@@ -282,6 +300,56 @@ setup:
 
   - match: { aggregations.double_terms.buckets.1.doc_count: 1 }
 
+---
+"Scaled float test":
+  - skip:
+      version: " - 5.2.0"
+      reason:  scaled_float were considered as longs in aggregations, this was fixed in 5.2.0
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "scaled_float": 9.99 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "scaled_float": 9.994 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "scaled_float": 8.99 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "scaled_float_terms" : { "terms" : { "field" : "scaled_float" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.scaled_float_terms.buckets: 2 }
+
+  - match: { aggregations.scaled_float_terms.buckets.0.key: 9.99 }
+
+  - is_false: aggregations.scaled_float_terms.buckets.0.key_as_string
+
+  - match: { aggregations.scaled_float_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.scaled_float_terms.buckets.1.key: 8.99 }
+
+  - is_false: aggregations.scaled_float_terms.buckets.1.key_as_string
+
+  - match: { aggregations.scaled_float_terms.buckets.1.doc_count: 1 }
+
 ---
 "Date test":
   - do:
@@ -356,9 +424,9 @@ setup:
 "Partitioned string test":
 
   - skip:
-    version: " - 5.1.99"
-    reason:  Partitioning is a 5.2.0 feature
-    
+      version: " - 5.1.99"
+      reason:  Partitioning is a 5.2.0 feature
+
   - do:
       index:
         index: test_1
@@ -415,8 +483,8 @@ setup:
 "Partitioned integer test":
 
   - skip:
-    version: " - 5.1.99"
-    reason:  Partitioning is a 5.2.0 feature
+      version: " - 5.1.99"
+      reason:  Partitioning is a 5.2.0 feature
 
   - do:
       index:
@@ -465,3 +533,217 @@ setup:
   - match: { aggregations.str_terms.buckets.0.key: 1234 }
 
   - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+
+---
+"Unmapped strings":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "string_terms" : { "terms" : { "field" : "unmapped_string", "value_type" : "string", "missing": "abc" } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.string_terms.buckets: 1 }
+
+  - match: { aggregations.string_terms.buckets.0.key: "abc" }
+
+  - match: { aggregations.string_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped booleans":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "boolean_terms" : { "terms" : { "field" : "unmapped_boolean", "value_type" : "boolean", "missing": true } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.boolean_terms.buckets: 1 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key: 1 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key_as_string: "true" }
+
+  - match: { aggregations.boolean_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped dates":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "unmapped_date", "value_type" : "date", "missing": "2016-05-11" } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.date_terms.buckets: 1 }
+
+  - match: { aggregations.date_terms.buckets.0.key: 1462924800000 }
+
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-11T00:00:00.000Z" }
+
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped longs":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "long_terms" : { "terms" : { "field" : "unmapped_long", "value_type" : "long", "missing": 3 } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.long_terms.buckets: 1 }
+
+  - match: { aggregations.long_terms.buckets.0.key: 3 }
+
+  - match: { aggregations.long_terms.buckets.0.doc_count: 1 }
+
+---
+"Unmapped doubles":
+
+  - skip:
+      version: " - 5.1.99"
+      reason: Unmapped fields handling with value_type was added in 5.2
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "double_terms" : { "terms" : { "field" : "unmapped_double", "value_type" : "double", "missing": 3.5 } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.double_terms.buckets: 1 }
+
+  - match: { aggregations.double_terms.buckets.0.key: 3.5 }
+
+  - match: { aggregations.double_terms.buckets.0.doc_count: 1 }
+
+---
+"Mixing longs and doubles":
+
+  - skip:
+      version: " - 5.99.99"
+      reason:  in 6.0 longs and doubles are compatible within a terms agg (longs are promoted to doubles)
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: {"number": 100}
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: {"number": 10}
+
+  - do:
+      index:
+        index: test_2
+        type: test
+        id: 3
+        body: {"number": 100.0}
+
+  - do:
+      index:
+        index: test_2
+        type: test
+        id: 1
+        body: {"number": 10.0}
+
+  - do:
+      index:
+        index: test_2
+        type: test
+        id: 2
+        body: {"number": 14.6}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "number_terms" : { "terms" : { "field" : "number" } } } }
+
+  - match: { hits.total: 5 }
+
+  - length: { aggregations.number_terms.buckets: 3 }
+
+  - match: { aggregations.number_terms.buckets.0.key: 10.0 }
+
+  - match: { aggregations.number_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.number_terms.buckets.1.key: 100.0 }
+
+  - match: { aggregations.number_terms.buckets.1.doc_count: 2 }
+
+  - match: { aggregations.number_terms.buckets.2.key: 14.6 }
+
+  - match: { aggregations.number_terms.buckets.2.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml
index 2152e75f7e6..63cca00509c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/50_filter.yaml
@@ -36,8 +36,8 @@ setup:
 ---
 "Filter aggs with terms lookup ensure not cached":
   - skip:
-    version: " - 5.0.0"
-    reason:  This using filter aggs that needs rewriting, this was fixed in 5.0.1
+      version: " - 5.0.0"
+      reason:  This using filter aggs that needs rewriting, this was fixed in 5.0.1
 
   - do:
       search:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_filters.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_filters.yaml
index 3d265699d46..57b320970b6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_filters.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/60_filters.yaml
@@ -39,8 +39,8 @@ setup:
 ---
 "Filters aggs with wrapper query":
   - skip:
-    version: " - 5.1.1"
-    reason:  Using filters aggs that needs rewriting, this was fixed in 5.1.2
+      version: " - 5.1.1"
+      reason:  Using filters aggs that needs rewriting, this was fixed in 5.1.2
 
   - do:
       search:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml
new file mode 100644
index 00000000000..143ff9e3d13
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/70_adjacency_matrix.yaml
@@ -0,0 +1,66 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+            mappings:
+              post:
+                properties:
+                  num:
+                    type: integer
+
+  - do:
+        index:
+          index: test
+          type: test
+          id: 1
+          body: { "num": [1, 2] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 2
+        body: { "num": [2, 3] }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 3
+        body: { "num": [3, 4] }
+
+  - do:
+      indices.refresh: {}
+
+
+---
+"Filters intersections":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  Adjacency Matrix is a 5.3.0 feature
+      
+  - do:
+      search:
+        body: { "size": 0, "aggs": { "conns": { "adjacency_matrix": {  "filters": { "1": { "term": { "num": 1 } }, "2": { "term": { "num": 2 } }, "4": { "term": { "num": 4 } } } } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.conns.buckets: 4 }
+
+  - match: { aggregations.conns.buckets.0.doc_count: 1 }
+  - match: { aggregations.conns.buckets.0.key: "1" }
+
+  - match: { aggregations.conns.buckets.1.doc_count: 1 }
+  - match: { aggregations.conns.buckets.1.key: "1&2" }
+
+  - match: { aggregations.conns.buckets.2.doc_count: 2 }
+  - match: { aggregations.conns.buckets.2.key: "2" }
+
+  - match: { aggregations.conns.buckets.3.doc_count: 1 }
+  - match: { aggregations.conns.buckets.3.key: "4" }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml
new file mode 100644
index 00000000000..72f782e68d1
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/10_unified.yaml
@@ -0,0 +1,39 @@
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            mappings:
+              unified:
+                "properties":
+                  "text":
+                     "type": "text"
+                     "fields":
+                        "fvh":
+                          "type": "text"
+                          "term_vector": "with_positions_offsets"
+                        "postings":
+                          "type": "text"
+                          "index_options": "offsets"
+  - do:
+      index:
+        index: test
+        type:  unified
+        id:    1
+        body:
+            "text" : "The quick brown fox is brown."
+  - do:
+      indices.refresh: {}
+
+---
+"Basic":
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new highlighter that has been added in 5.3
+  - do:
+      search:
+        body: { "query" : {"multi_match" : { "query" : "quick brown fox", "fields" : [ "text*"] } }, "highlight" : { "type" : "unified", "fields" : { "*" : {} } } }
+
+  - match: {hits.hits.0.highlight.text.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.0.highlight.text\.fvh.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
+  - match: {hits.hits.0.highlight.text\.postings.0: "The <em>quick</em> <em>brown</em> <em>fox</em> is <em>brown</em>."}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml
new file mode 100644
index 00000000000..a015f8649cf
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yaml
@@ -0,0 +1,203 @@
+setup:
+  - do:
+      indices.create:
+          index:  test
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     1
+          body:   { numeric_group: 1, sort: 10 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     2
+          body:   { numeric_group: 1, sort: 6 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     3
+          body:   { numeric_group: 1, sort: 24 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     4
+          body:   { numeric_group: 25, sort: 10 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     5
+          body:   { numeric_group: 25, sort: 5 }
+  - do:
+      index:
+          index:  test
+          type:   test
+          id:     6
+          body:   { numeric_group: 3, sort: 36 }
+  - do:
+      indices.refresh:
+        index: test
+
+---
+"field collapsing":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ sort: desc }]
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._type: test }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - match: {hits.hits.0.sort: [36] }
+  - match: {hits.hits.0._id: "6" }
+  - is_false: hits.hits.0.inner_hits
+  - match: {hits.hits.1._index: test }
+  - match: {hits.hits.1._type: test }
+  - match: {hits.hits.1.fields.numeric_group: [1] }
+  - match: {hits.hits.1.sort: [24] }
+  - match: {hits.hits.1._id: "3" }
+  - is_false: hits.hits.1.inner_hits
+  - match: {hits.hits.2._index: test }
+  - match: {hits.hits.2._type: test }
+  - match: {hits.hits.2.fields.numeric_group: [25] }
+  - match: {hits.hits.2.sort: [10] }
+  - match: {hits.hits.2._id: "4" }
+  - is_false: hits.hits.2.inner_hits
+
+---
+"field collapsing and from":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          from: 2
+          collapse: { field: numeric_group }
+          sort: [{ sort: desc }]
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0._index: test }
+  - match: {hits.hits.0._type: test }
+  - match: {hits.hits.0.fields.numeric_group: [25]}
+  - match: {hits.hits.0.sort: [10] }
+  - match: {hits.hits.0._id: "4" }
+  - is_false: hits.hits.0.inner_hits
+
+---
+"field collapsing and inner_hits":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group, inner_hits: { name: sub_hits, size: 2, sort: [{ sort: asc }] } }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0._type: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1._type: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+
+---
+"field collapsing and scroll":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      catch:      /cannot use \`collapse\` in a scroll context/
+      search:
+        index: test
+        type:  test
+        scroll: 1s
+        body:
+          collapse: { field: numeric_group }
+
+---
+"field collapsing and search_after":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      catch:      /cannot use \`collapse\` in conjunction with \`search_after\`/
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          search_after: [6]
+          sort: [{ sort: desc }]
+
+---
+"field collapsing and rescore":
+
+  - skip:
+      version: " - 5.2.99"
+      reason:  this uses a new API that has been added in 6.0
+
+  - do:
+      catch:      /cannot use \`collapse\` in conjunction with \`rescore\`/
+      search:
+        index: test
+        type:  test
+        body:
+          collapse: { field: numeric_group }
+          rescore:
+            window_size: 20
+            query:
+              rescore_query:
+                match_all: {}
+              query_weight: 1
+              rescore_query_weight: 2
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
index 8533cfd2668..fa9ff925d2e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml
@@ -6,8 +6,6 @@
           body:
             mappings:
               test:
-                _all:
-                  enabled: false
                 properties:
                   number:
                     type: integer
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
index 43db8e5206f..b12b9d09b6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.create/10_basic.yaml
@@ -30,3 +30,10 @@ setup:
   - match: { snapshot.state : SUCCESS }
   - match: { snapshot.shards.successful: 1 }
   - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.delete:
+        repository: test_repo_create_1
+        snapshot: test_snapshot
+
+  - match: { acknowledged: true }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml
index 5788453a739..b944fe43791 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get_repository/10_basic.yaml
@@ -5,17 +5,17 @@ setup:
       snapshot.create_repository:
         repository: test_repo_get_1
         body:
-          type: url
+          type: fs
           settings:
-            url: "http://snapshot.test1"
+            location: "test_repo_get_1_loc"
 
   - do:
       snapshot.create_repository:
         repository: test_repo_get_2
         body:
-          type: url
+          type: fs
           settings:
-            url: "http://snapshot.test2"
+            location: "test_repo_get_1_loc"
 
 ---
 "Get all repositories":
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
index cc04ea32a6a..168c913b75c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/20_issue7121.yaml
@@ -33,7 +33,7 @@
         index: testidx
         type:  doc
         id:    1
-        realtime: 0
+        realtime: false
 
   - match: { _index: "testidx" }
   - match: { _type: "doc" }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
index 12457861c98..26f441207ac 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/termvectors/30_realtime.yaml
@@ -26,7 +26,7 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 0
+          realtime: false
 
  - is_false: found
 
@@ -35,6 +35,6 @@
           index:    test_1
           type:     test
           id:       1
-          realtime: 1
+          realtime: true
 
  - is_true: found
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
index af25730fd7a..7585b9f3e0b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/22_doc_as_upsert.yaml
@@ -8,7 +8,7 @@
           id:               1
           body:
             doc:            { foo: bar, count: 1 }
-            doc_as_upsert:  1
+            doc_as_upsert:  true
 
   - do:
       get:
@@ -27,7 +27,7 @@
           id:               1
           body:
             doc:            { count: 2 }
-            doc_as_upsert:  1
+            doc_as_upsert:  true
 
   - do:
       get:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
index c5fe68e26c9..62d8fd125ff 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
@@ -81,26 +81,26 @@
 "refresh=wait_for waits until changes are visible in search":
  - do:
       index:
-          index:   test_1
+          index:   update_60_refresh_1
           type:    test
-          id:      1
+          id:      update_60_refresh_id1
           body:    { foo: bar }
           refresh: true
  - is_true: forced_refresh
 
  - do:
       search:
-          index:   test_1
+          index:   update_60_refresh_1
           type:    test
           body:
-            query:   { term: { _id: 1 }}
+            query:   { term: { _id: update_60_refresh_id1 }}
  - match: { hits.total: 1 }
 
  - do:
      update:
-         index:   test_1
+         index:   update_60_refresh_1
          type:    test
-         id:      1
+         id:      update_60_refresh_id1
          refresh: wait_for
          body:
            doc: { test: asdf }
@@ -108,7 +108,7 @@
 
  - do:
       search:
-          index:   test_1
+          index:   update_60_refresh_1
           type:    test
           body:
             query:   { match: { test: asdf } }
diff --git a/settings.gradle b/settings.gradle
index 5a2c89567ff..56a88c3d934 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -7,6 +7,7 @@ List projects = [
   'core',
   'docs',
   'client:rest',
+  'client:rest-high-level',
   'client:sniffer',
   'client:transport',
   'client:test',
@@ -31,6 +32,7 @@ List projects = [
   'modules:reindex',
   'modules:rank-eval',
   'modules:percolator',
+  'modules:repository-url',
   'plugins:analysis-icu',
   'plugins:analysis-kuromoji',
   'plugins:analysis-phonetic',
@@ -56,6 +58,7 @@ List projects = [
   'qa:evil-tests',
   'qa:no-bootstrap-tests',
   'qa:rolling-upgrade',
+  'qa:multi-cluster-search',
   'qa:smoke-test-client',
   'qa:smoke-test-http',
   'qa:smoke-test-ingest-with-all-dependencies',
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 8be06574df8..6756495e0a1 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -32,6 +32,7 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "org.elasticsearch:securemock:${versions.securemock}"
+  compile "org.elasticsearch:mocksocket:${versions.mocksocket}"
 }
 
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
index 6a5c764375c..bdf36bbb85a 100644
--- a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
@@ -42,6 +42,7 @@ import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
 import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
 import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
 import org.elasticsearch.index.analysis.ElisionTokenFilterFactory;
+import org.elasticsearch.index.analysis.FlattenGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
 import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
 import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
@@ -82,6 +83,7 @@ import org.elasticsearch.index.analysis.StandardTokenizerFactory;
 import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory;
 import org.elasticsearch.index.analysis.StemmerTokenFilterFactory;
 import org.elasticsearch.index.analysis.StopTokenFilterFactory;
+import org.elasticsearch.index.analysis.SynonymGraphTokenFilterFactory;
 import org.elasticsearch.index.analysis.SynonymTokenFilterFactory;
 import org.elasticsearch.index.analysis.ThaiTokenizerFactory;
 import org.elasticsearch.index.analysis.TrimTokenFilterFactory;
@@ -240,12 +242,14 @@ public class AnalysisFactoryTestCase extends ESTestCase {
         .put("stop",                      StopTokenFilterFactory.class)
         .put("swedishlightstem",          StemmerTokenFilterFactory.class)
         .put("synonym",                   SynonymTokenFilterFactory.class)
+        .put("synonymgraph",              SynonymGraphTokenFilterFactory.class)
         .put("trim",                      TrimTokenFilterFactory.class)
         .put("truncate",                  TruncateTokenFilterFactory.class)
         .put("turkishlowercase",          LowerCaseTokenFilterFactory.class)
         .put("type",                      KeepTypesFilterFactory.class)
         .put("uppercase",                 UpperCaseTokenFilterFactory.class)
         .put("worddelimiter",             WordDelimiterTokenFilterFactory.class)
+        .put("flattengraph",              FlattenGraphTokenFilterFactory.class)
 
         // TODO: these tokenfilters are not yet exposed: useful?
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionMetadataTestCase.java
similarity index 69%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionMetadataTestCase.java
index cb9ec0c273b..b68797381d1 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionMetadataTestCase.java
@@ -17,16 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
-public abstract class AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
+public abstract class AbstractAsyncBulkByScrollActionMetadataTestCase<
+                Request extends AbstractBulkByScrollRequest<Request>,
+                Response extends BulkByScrollResponse>
+        extends AbstractAsyncBulkByScrollActionTestCase<Request, Response> {
 
     protected ScrollableHitSource.BasicHit doc() {
         return new ScrollableHitSource.BasicHit("index", "type", "id", 0);
     }
 
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action();
+    protected abstract AbstractAsyncBulkByScrollAction<Request> action();
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionTestCase.java
similarity index 87%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionTestCase.java
index 3df61f36915..907f8965632 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractAsyncBulkByScrollActionTestCase.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.tasks.TaskId;
@@ -27,9 +27,9 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
 
-public abstract class AbstractAsyncBulkIndexByScrollActionTestCase<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
+public abstract class AbstractAsyncBulkByScrollActionTestCase<
+                Request extends AbstractBulkByScrollRequest<Request>,
+                Response extends BulkByScrollResponse>
         extends ESTestCase {
     protected ThreadPool threadPool;
     protected WorkingBulkByScrollTask task;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestTestCase.java
similarity index 96%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestTestCase.java
index 86b36f11a97..8a12a32ad0b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/AbstractBulkByScrollRequestTestCase.java
@@ -17,8 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
+import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.tasks.TaskId;
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkIndexByScrollResponseMatcher.java
similarity index 96%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
rename to test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkIndexByScrollResponseMatcher.java
index 7c8ad56a48a..2902e02a30c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
+++ b/test/framework/src/main/java/org/elasticsearch/action/bulk/byscroll/BulkIndexByScrollResponseMatcher.java
@@ -17,8 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.index.reindex;
+package org.elasticsearch.action.bulk.byscroll;
 
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
@@ -29,7 +30,7 @@ import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
-public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexByScrollResponse> {
+public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkByScrollResponse> {
 
     private Matcher<Long> createdMatcher = equalTo(0L);
     private Matcher<Long> updatedMatcher = equalTo(0L);
@@ -130,7 +131,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
     }
 
     @Override
-    protected boolean matchesSafely(BulkIndexByScrollResponse item) {
+    protected boolean matchesSafely(BulkByScrollResponse item) {
         return updatedMatcher.matches(item.getUpdated()) &&
                 createdMatcher.matches(item.getCreated()) &&
                 deletedMatcher.matches(item.getDeleted()) &&
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index ed8725fa008..d0558116448 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -25,6 +25,7 @@ import org.elasticsearch.SecureSM;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.network.IfConfig;
 import org.elasticsearch.plugins.PluginInfo;
 import org.junit.Assert;
 
@@ -89,6 +90,9 @@ public class BootstrapForTesting {
             throw new RuntimeException("found jar hell in test classpath", e);
         }
 
+        // Log ifconfig output before SecurityManager is installed
+        IfConfig.logIfNecessary();
+
         // install security manager if requested
         if (systemPropertyAsBoolean("tests.security.manager", true)) {
             try {
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
index dd2320a6855..7b575d0b7a6 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Path;
@@ -32,7 +35,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 abstract class ESElasticsearchCliTestCase extends ESTestCase {
 
     interface InitConsumer {
-        void accept(final boolean foreground, final Path pidFile, final boolean quiet, final Map<String, String> esSettings);
+        void accept(boolean foreground, Path pidFile, boolean quiet, Environment initialEnv);
     }
 
     void runTest(
@@ -42,13 +45,21 @@ abstract class ESElasticsearchCliTestCase extends ESTestCase {
             final InitConsumer initConsumer,
             String... args) throws Exception {
         final MockTerminal terminal = new MockTerminal();
+        Path home = createTempDir();
         try {
             final AtomicBoolean init = new AtomicBoolean();
             final int status = Elasticsearch.main(args, new Elasticsearch() {
                 @Override
-                void init(final boolean daemonize, final Path pidFile, final boolean quiet, final Map<String, String> esSettings) {
+                protected Environment createEnv(Terminal terminal, Map<String, String> settings) {
+                    Settings realSettings = Settings.builder()
+                        .put("path.home", home)
+                        .put(settings).build();
+                    return new Environment(realSettings);
+                }
+                @Override
+                void init(final boolean daemonize, final Path pidFile, final boolean quiet, Environment initialEnv) {
                     init.set(true);
-                    initConsumer.accept(!daemonize, pidFile, quiet, esSettings);
+                    initConsumer.accept(!daemonize, pidFile, quiet, initialEnv);
                 }
 
                 @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
index 98155967514..4128c9d9efc 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
@@ -71,7 +71,7 @@ public class FileTestUtils {
         }
         Files.createDirectories(destDir);
 
-        try (final ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
+        try (ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
             ZipEntry entry;
             while ((entry = zipInput.getNextEntry()) != null) {
                 final String entryPath;
diff --git a/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
new file mode 100644
index 00000000000..c9821ca700d
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/common/settings/MockSecureSettings.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.settings;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A mock implementation of secure settings for tests to use.
+ */
+public class MockSecureSettings implements SecureSettings {
+
+    private Map<String, SecureString> secureStrings = new HashMap<>();
+
+    @Override
+    public boolean isLoaded() {
+        return true;
+    }
+
+    @Override
+    public boolean hasSetting(String setting) {
+        return secureStrings.containsKey(setting);
+    }
+
+    @Override
+    public SecureString getString(String setting) {
+        return secureStrings.get(setting);
+    }
+
+    public void setString(String setting, String value) {
+        secureStrings.put(setting, new SecureString(value.toCharArray()));
+    }
+
+    @Override
+    public void close() throws IOException {}
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index 854260f02a4..b01ee7cc5c8 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.MapperService;
@@ -39,12 +40,14 @@ import static org.elasticsearch.test.ESTestCase.createTestAnalysis;
 
 public class MapperTestUtils {
 
-    public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
+    public static MapperService newMapperService(NamedXContentRegistry xContentRegistry, Path tempDir, Settings indexSettings)
+            throws IOException {
         IndicesModule indicesModule = new IndicesModule(Collections.emptyList());
-        return newMapperService(tempDir, indexSettings, indicesModule);
+        return newMapperService(xContentRegistry, tempDir, indexSettings, indicesModule);
     }
 
-    public static MapperService newMapperService(Path tempDir, Settings settings, IndicesModule indicesModule) throws IOException {
+    public static MapperService newMapperService(NamedXContentRegistry xContentRegistry, Path tempDir, Settings settings,
+            IndicesModule indicesModule) throws IOException {
         Settings.Builder settingsBuilder = Settings.builder()
             .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
             .put(settings);
@@ -58,6 +61,7 @@ public class MapperTestUtils {
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
         return new MapperService(indexSettings,
             indexAnalyzers,
+            xContentRegistry,
             similarityService,
             mapperRegistry,
             () -> null);
diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
index 8398430f4e9..20c869d38c6 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.index.shard;
 
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexNotFoundException;
 import org.apache.lucene.index.LeafReader;
@@ -30,7 +29,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -43,6 +41,7 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
@@ -56,6 +55,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
+import org.elasticsearch.index.seqno.SequenceNumbersService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.DirectoryService;
 import org.elasticsearch.index.store.Store;
@@ -67,6 +67,7 @@ import org.elasticsearch.indices.recovery.RecoverySourceHandler;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTarget;
 import org.elasticsearch.indices.recovery.StartRecoveryRequest;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -75,6 +76,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -262,10 +264,9 @@ public abstract class IndexShardTestCase extends ESTestCase {
         boolean success = false;
         try {
             IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
-            MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), indexSettings.getSettings());
-            for (ObjectObjectCursor<String, MappingMetaData> typeMapping : indexMetaData.getMappings()) {
-                mapperService.merge(typeMapping.key, typeMapping.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
-            }
+            MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
+                    indexSettings.getSettings());
+            mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
             final IndexEventListener indexEventListener = new IndexEventListener() {
             };
@@ -368,7 +369,8 @@ public abstract class IndexShardTestCase extends ESTestCase {
     }
 
     private DiscoveryNode getFakeDiscoNode(String id) {
-        return new DiscoveryNode(id, buildNewFakeTransportAddress(), Version.CURRENT);
+        return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class),
+            Version.CURRENT);
     }
 
     /** recovers a replica from the given primary **/
@@ -380,30 +382,45 @@ public abstract class IndexShardTestCase extends ESTestCase {
     }
 
     /**
-     * Recovers a replica from the give primary, allow the user to supply a custom recovery target.
-     * A typical usage of a custome recovery target is to assert things in the various stages of recovery
-     *
-     * @param markAsRecovering set to false if you have already marked the replica as recovering
+     * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
+     * target is to assert things in the various stages of recovery.
+     * @param replica                the recovery target shard
+     * @param primary                the recovery source shard
+     * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
+     * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
      */
-    protected void recoverReplica(IndexShard replica, IndexShard primary,
-                                  BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
-                                  boolean markAsRecovering)
-        throws IOException {
+    protected final void recoverReplica(final IndexShard replica,
+                                        final IndexShard primary,
+                                        final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
+                                        final boolean markAsRecovering) throws IOException {
         final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
         final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
         if (markAsRecovering) {
-            replica.markAsRecovering("remote",
-                new RecoveryState(replica.routingEntry(), pNode, rNode));
+            replica.markAsRecovering("remote", new RecoveryState(replica.routingEntry(), pNode, rNode));
         } else {
             assertEquals(replica.state(), IndexShardState.RECOVERING);
         }
         replica.prepareForIndexRecovery();
-        RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
-        StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), pNode, rNode,
-            getMetadataSnapshotOrEmpty(replica), false, 0);
-        RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, () -> 0L, e -> () -> {
-        },
-            (int) ByteSizeUnit.MB.toKB(1), logger);
+        final RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
+
+        final Store.MetadataSnapshot snapshot = getMetadataSnapshotOrEmpty(replica);
+        final long startingSeqNo;
+        if (snapshot.size() > 0) {
+            startingSeqNo = PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget);
+        } else {
+            startingSeqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
+        }
+
+        final StartRecoveryRequest request =
+            new StartRecoveryRequest(replica.shardId(), pNode, rNode, snapshot, false, 0, startingSeqNo);
+        final RecoverySourceHandler recovery = new RecoverySourceHandler(
+            primary,
+            recoveryTarget,
+            request,
+            () -> 0L,
+            e -> () -> {},
+            (int) ByteSizeUnit.MB.toBytes(1),
+            Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), pNode.getName()).build());
         recovery.recoverToTarget();
         recoveryTarget.markAsDone();
         replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
@@ -452,29 +469,35 @@ public abstract class IndexShardTestCase extends ESTestCase {
     }
 
 
-    protected Engine.Index indexDoc(IndexShard shard, String type, String id) {
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id) throws IOException {
         return indexDoc(shard, type, id, "{}");
     }
 
-    protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source) {
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source) throws IOException {
+        return indexDoc(shard, type, id, source, XContentType.JSON);
+    }
+
+    protected Engine.Index indexDoc(IndexShard shard, String type, String id, String source, XContentType xContentType) throws IOException {
         final Engine.Index index;
         if (shard.routingEntry().primary()) {
             index = shard.prepareIndexOnPrimary(
-                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)),
+                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source),
+                    xContentType),
                 Versions.MATCH_ANY,
                 VersionType.INTERNAL,
                 IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP,
                 false);
         } else {
             index = shard.prepareIndexOnReplica(
-                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source)),
+                SourceToParse.source(SourceToParse.Origin.PRIMARY, shard.shardId().getIndexName(), type, id, new BytesArray(source),
+                    xContentType),
                 randomInt(1 << 10), 1, VersionType.EXTERNAL, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
         }
         shard.index(index);
         return index;
     }
 
-    protected Engine.Delete deleteDoc(IndexShard shard, String type, String id) {
+    protected Engine.Delete deleteDoc(IndexShard shard, String type, String id) throws IOException {
         final Engine.Delete delete;
         if (shard.routingEntry().primary()) {
             delete = shard.prepareDeleteOnPrimary(type, id, Versions.MATCH_ANY, VersionType.INTERNAL);
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index c75d9bbcb6d..8774ba5836b 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -22,23 +22,21 @@ package org.elasticsearch.node;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.MockInternalClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
-import org.elasticsearch.discovery.zen.UnicastHostsProvider;
-import org.elasticsearch.discovery.zen.ZenPing;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.recovery.RecoverySettings;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.fetch.FetchPhase;
-import org.elasticsearch.test.discovery.MockZenPing;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
@@ -46,6 +44,7 @@ import org.elasticsearch.transport.TransportInterceptor;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Collection;
+import java.util.function.Function;
 
 /**
  * A node for testing which allows:
@@ -90,15 +89,17 @@ public class MockNode extends Node {
 
     @Override
     protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool,
-                                                   TransportInterceptor interceptor, ClusterSettings clusterSettings) {
+                                                   TransportInterceptor interceptor,
+                                                   Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                                   ClusterSettings clusterSettings) {
         // we use the MockTransportService.TestPlugin class as a marker to create a network
         // module with this MockNetworkService. NetworkService is such an integral part of the systme
         // we don't allow to plug it in from plugins or anything. this is a test-only override and
         // can't be done in a production env.
         if (getPluginsService().filterPlugins(MockTransportService.TestPlugin.class).isEmpty()) {
-            return super.newTransportService(settings, transport, threadPool, interceptor, clusterSettings);
+            return super.newTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
         } else {
-            return new MockTransportService(settings, transport, threadPool, interceptor, clusterSettings);
+            return new MockTransportService(settings, transport, threadPool, interceptor, localNodeFactory, clusterSettings);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java b/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java
index 9203b632cba..69291ccaba6 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/NodeTests.java
@@ -138,4 +138,41 @@ public class NodeTests extends ESTestCase {
 
     }
 
+    public void testNodeAttributes() throws IOException {
+        String attr = randomAsciiOfLength(5);
+        Settings.Builder settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            final Settings nodeSettings = randomBoolean() ? node.settings() : node.getEnvironment().settings();
+            assertEquals(attr, Node.NODE_ATTRIBUTES.get(nodeSettings).getAsMap().get("test_attr"));
+        }
+
+        // leading whitespace not allowed
+        attr = " leading";
+        settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            fail("should not allow a node attribute with leading whitespace");
+        } catch (IllegalArgumentException e) {
+            assertEquals("node.attr.test_attr cannot have leading or trailing whitespace [ leading]", e.getMessage());
+        }
+
+        // trailing whitespace not allowed
+        attr = "trailing ";
+        settings = baseSettings().put(Node.NODE_ATTRIBUTES.getKey() + "test_attr", attr);
+        try (Node node = new MockNode(settings.build(), Collections.singleton(MockTcpTransportPlugin.class))) {
+            fail("should not allow a node attribute with trailing whitespace");
+        } catch (IllegalArgumentException e) {
+            assertEquals("node.attr.test_attr cannot have leading or trailing whitespace [trailing ]", e.getMessage());
+        }
+    }
+
+    private static Settings.Builder baseSettings() {
+        final Path tempDir = createTempDir();
+        return Settings.builder()
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(NetworkModule.HTTP_ENABLED.getKey(), false)
+            .put("transport.type", "mock-socket-network")
+            .put(Node.NODE_DATA_SETTING.getKey(), true);
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index e25c16cb5c0..42029b053a4 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -46,7 +46,7 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
 
     public void testWriteRead() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer container = store.blobContainer(new BlobPath());
             byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
             writeBlob(container, "foobar", new BytesArray(data));
@@ -65,7 +65,7 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
     }
 
     public void testMoveAndList() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer container = store.blobContainer(new BlobPath());
             assertThat(container.listBlobs().size(), equalTo(0));
             int numberOfFooBlobs = randomIntBetween(0, 10);
@@ -113,7 +113,7 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
     }
 
     public void testDeleteBlob() throws IOException {
-        try (final BlobStore store = newBlobStore()) {
+        try (BlobStore store = newBlobStore()) {
             final String blobName = "foobar";
             final BlobContainer container = store.blobContainer(new BlobPath());
             expectThrows(IOException.class, () -> container.deleteBlob(blobName));
@@ -129,7 +129,7 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
     }
 
     public void testVerifyOverwriteFails() throws IOException {
-        try (final BlobStore store = newBlobStore()) {
+        try (BlobStore store = newBlobStore()) {
             final String blobName = "foobar";
             final BlobContainer container = store.blobContainer(new BlobPath());
             byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
index 0cec570dbe8..e7f8edb1fa2 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
@@ -36,7 +36,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 public abstract class ESBlobStoreTestCase extends ESTestCase {
 
     public void testContainerCreationAndDeletion() throws IOException {
-        try(final BlobStore store = newBlobStore()) {
+        try(BlobStore store = newBlobStore()) {
             final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo"));
             final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar"));
             byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
index 03f36bc1db2..f49cd174b7f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -32,6 +33,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
@@ -73,7 +75,7 @@ public class RandomSearchRequestGenerator {
      * Build a random search request.
      *
      * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use
-     *        {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier)}.
+     *        {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}.
      */
     public static SearchRequest randomSearchRequest(Supplier<SearchSourceBuilder> randomSearchSourceBuilder) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
@@ -111,7 +113,8 @@ public class RandomSearchRequestGenerator {
             Supplier<HighlightBuilder> randomHighlightBuilder,
             Supplier<SuggestBuilder> randomSuggestBuilder,
             Supplier<RescoreBuilder<?>> randomRescoreBuilder,
-            Supplier<List<SearchExtBuilder>> randomExtBuilders) {
+            Supplier<List<SearchExtBuilder>> randomExtBuilders,
+            Supplier<CollapseBuilder> randomCollapseBuilder) {
         SearchSourceBuilder builder = new SearchSourceBuilder();
         if (randomBoolean()) {
             builder.from(randomIntBetween(0, 10000));
@@ -296,11 +299,12 @@ public class RandomSearchRequestGenerator {
                 }
                 jsonBuilder.endArray();
                 jsonBuilder.endObject();
-                XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(jsonBuilder.bytes());
+                XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY,
+                        jsonBuilder.bytes());
                 parser.nextToken();
                 parser.nextToken();
                 parser.nextToken();
-                builder.searchAfter(SearchAfterBuilder.fromXContent(parser, null).getSortValues());
+                builder.searchAfter(SearchAfterBuilder.fromXContent(parser).getSortValues());
             } catch (IOException e) {
                 throw new RuntimeException("Error building search_from", e);
             }
@@ -333,6 +337,9 @@ public class RandomSearchRequestGenerator {
                 builder.slice(new SliceBuilder(field, id, max));
             }
         }
+        if (randomBoolean()) {
+            builder.collapse(randomCollapseBuilder.get());
+        }
         return builder;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java
new file mode 100644
index 00000000000..feabca586b8
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableSerializationTestCase.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+import org.elasticsearch.common.xcontent.ToXContent;
+
+import java.io.IOException;
+
+/**
+ * An abstract test case to ensure correct behavior of Diffable.
+ *
+ * This class can be used as a based class for tests of MetaData.Custom classes and other classes that support,
+ * Writable serialization, XContent-based serialization and is diffable.
+ */
+public abstract class AbstractDiffableSerializationTestCase<T extends Diffable<T> & ToXContent> extends AbstractSerializingTestCase<T> {
+
+    /**
+     *  Introduces random changes into the test object
+     */
+    protected abstract T makeTestChanges(T testInstance);
+
+    protected abstract Reader<Diff<T>> diffReader();
+
+    public void testDiffableSerialization() throws IOException {
+        DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(),
+            instanceReader(), diffReader());
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java
new file mode 100644
index 00000000000..aa801308825
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractDiffableWireSerializationTestCase.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+
+/**
+ * An abstract test case to ensure correct behavior of Diffable.
+ *
+ * This class can be used as a based class for tests of ClusterState.Custom classes and other classes that support,
+ * Writable serialization and is diffable.
+ */
+public abstract class AbstractDiffableWireSerializationTestCase<T extends Diffable<T>> extends AbstractWireSerializingTestCase<T> {
+    /**
+     *  Introduces random changes into the test object
+     */
+    protected abstract T makeTestChanges(T testInstance);
+
+    protected abstract Reader<Diff<T>> diffReader();
+
+    public void testDiffableSerialization() throws IOException {
+        DiffableTestUtils.testDiffableSerialization(this::createTestInstance, this::makeTestChanges, getNamedWriteableRegistry(),
+            instanceReader(), diffReader());
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 2055cde567e..2a230106994 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -37,7 +37,6 @@ import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -47,13 +46,12 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable.Reader;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -83,8 +81,7 @@ import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
@@ -117,15 +114,16 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.stream.Stream;
 
 import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
 import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
@@ -156,11 +154,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     private static String[] currentTypes;
     private static String[] randomTypes;
 
-    /**
-     * used to check warning headers of the deprecation logger
-     */
-    private ThreadContext threadContext;
-
     protected static Index getIndex() {
         return index;
     }
@@ -187,7 +180,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
         indexSettings = Settings.builder()
-                .put(ParseFieldMatcher.PARSE_STRICT, true)
                 .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated).build();
 
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
@@ -214,20 +206,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             serviceHolder = new ServiceHolder(nodeSettings, indexSettings, getPlugins(), this);
         }
         serviceHolder.clientInvocationHandler.delegate = this;
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(threadContext);
-    }
-
-    /**
-     * Check that there are no unaccounted warning headers. These should be checked with {@link #checkWarningHeaders(String...)} in the
-     * appropriate test
-     */
-    @After
-    public void teardown() throws IOException {
-        final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertNull("unexpected warning headers", warnings);
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        this.threadContext.close();
     }
 
     private static SearchContext getSearchContext(String[] types, QueryShardContext context) {
@@ -247,7 +225,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     }
 
     @After
-    public void afterTest() {
+    public void afterTest() throws IOException {
         serviceHolder.clientInvocationHandler.delegate = null;
     }
 
@@ -421,7 +399,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 BytesStreamOutput out = new BytesStreamOutput();
                 try (
                         XContentGenerator generator = XContentType.JSON.xContent().createGenerator(out);
-                        XContentParser parser = JsonXContent.jsonXContent.createParser(query);
+                        XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, query);
                 ) {
                     int objectIndex = -1;
                     Deque<String> levels = new LinkedList<>();
@@ -541,12 +519,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
     protected void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
-        assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT);
-    }
-
-    protected void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher)
-            throws IOException {
-        QueryBuilder newQuery = parseQuery(queryAsString, matcher);
+        QueryBuilder newQuery = parseQuery(queryAsString);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
@@ -555,37 +528,24 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    private void assertParsedQuery(XContentParser parser, QueryBuilder expectedQuery) throws IOException {
-        assertParsedQuery(parser, expectedQuery, ParseFieldMatcher.STRICT);
-    }
-
-    private void assertParsedQuery(XContentParser parser, QueryBuilder expectedQuery, ParseFieldMatcher matcher)
-            throws IOException {
-        QueryBuilder newQuery = parseQuery(parser, matcher);
+    private static void assertParsedQuery(XContentParser parser, QueryBuilder expectedQuery) throws IOException {
+        QueryBuilder newQuery = parseQuery(parser);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
     }
 
-    protected QueryBuilder parseQuery(String queryAsString) throws IOException {
-        return parseQuery(queryAsString, ParseFieldMatcher.STRICT);
-    }
-
-    protected QueryBuilder parseQuery(XContentParser parser) throws IOException {
-        return parseQuery(parser, ParseFieldMatcher.STRICT);
-    }
-
     protected QueryBuilder parseQuery(AbstractQueryBuilder<?> builder) throws IOException {
-        return parseQuery(createParser(JsonXContent.jsonXContent, builder.buildAsBytes(XContentType.JSON)), ParseFieldMatcher.STRICT);
+        return parseQuery(createParser(JsonXContent.jsonXContent, builder.buildAsBytes(XContentType.JSON)));
     }
 
-    protected QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
+    protected QueryBuilder parseQuery(String queryAsString) throws IOException {
         XContentParser parser = createParser(JsonXContent.jsonXContent, queryAsString);
-        return parseQuery(parser, matcher);
+        return parseQuery(parser);
     }
 
-    private QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
-        QueryParseContext context = createParseContext(parser, matcher);
+    protected static QueryBuilder parseQuery(XContentParser parser) throws IOException {
+        QueryParseContext context = createParseContext(parser);
         QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder();
         assertNull(parser.nextToken());
         return parseInnerQueryBuilder;
@@ -794,11 +754,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return serviceHolder.createShardContext();
     }
 
-    /**
-     * @return a new {@link QueryParseContext} based on the base test index and queryParserService
-     */
-    protected static QueryParseContext createParseContext(XContentParser parser, ParseFieldMatcher matcher) {
-        return new QueryParseContext(serviceHolder.indicesQueriesRegistry, parser, matcher);
+    private static QueryParseContext createParseContext(XContentParser parser) {
+        return new QueryParseContext(parser);
     }
 
     /**
@@ -1026,29 +983,16 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return query;
     }
 
-    protected void checkWarningHeaders(String... messages) {
-        final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.DEPRECATION_HEADER);
-        assertThat(warnings, hasSize(messages.length));
-        for (String msg : messages) {
-            assertThat(warnings, hasItem(equalTo(msg)));
-        }
-        // "clear" current warning headers by setting a new ThreadContext
-        DeprecationLogger.removeThreadContext(this.threadContext);
-        try {
-            this.threadContext.close();
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        this.threadContext = new ThreadContext(Settings.EMPTY);
-        DeprecationLogger.setThreadContext(this.threadContext);
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return serviceHolder.xContentRegistry;
     }
 
     private static class ServiceHolder implements Closeable {
-
-        private final IndicesQueriesRegistry indicesQueriesRegistry;
         private final IndexFieldDataService indexFieldDataService;
         private final SearchModule searchModule;
         private final NamedWriteableRegistry namedWriteableRegistry;
+        private final NamedXContentRegistry xContentRegistry;
         private final ClientInvocationHandler clientInvocationHandler = new ClientInvocationHandler();
         private final IndexSettings idxSettings;
         private final SimilarityService similarityService;
@@ -1056,7 +1000,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         private final BitsetFilterCache bitsetFilterCache;
         private final ScriptService scriptService;
         private final Client client;
-        private final long nowInMillis = randomPositiveLong();
+        private final long nowInMillis = randomNonNegativeLong();
 
         ServiceHolder(Settings nodeSettings, Settings indexSettings,
                       Collection<Class<? extends Plugin>> plugins, AbstractQueryTestCase<?> testCase) throws IOException {
@@ -1077,7 +1021,10 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
             entries.addAll(indicesModule.getNamedWriteables());
             entries.addAll(searchModule.getNamedWriteables());
-            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
+            namedWriteableRegistry = new NamedWriteableRegistry(entries);
+            xContentRegistry = new NamedXContentRegistry(Stream.of(
+                    searchModule.getNamedXContents().stream()
+                    ).flatMap(Function.identity()).collect(toList()));
             IndexScopedSettings indexScopedSettings = settingsModule.getIndexScopedSettings();
             idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
             AnalysisModule analysisModule = new AnalysisModule(new Environment(nodeSettings), emptyList());
@@ -1085,7 +1032,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             scriptService = scriptModule.getScriptService();
             similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
             MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
-            mapperService = new MapperService(idxSettings, indexAnalyzers, similarityService, mapperRegistry, this::createShardContext);
+            mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry,
+                    this::createShardContext);
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
             });
             indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,
@@ -1101,7 +1049,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
                 }
             });
-            indicesQueriesRegistry = searchModule.getQueryParserRegistry();
 
             for (String type : currentTypes) {
                 mapperService.merge(type, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(type,
@@ -1124,7 +1071,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                         MapperService.MergeReason.MAPPING_UPDATE, false);
             }
             testCase.initializeAdditionalMappings(mapperService);
-            this.namedWriteableRegistry = namedWriteableRegistry;
         }
 
         @Override
@@ -1133,7 +1079,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
         QueryShardContext createShardContext() {
             return new QueryShardContext(0, idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
-                    scriptService, indicesQueriesRegistry, this.client, null, () -> nowInMillis);
+                    scriptService, xContentRegistry, this.client, null, () -> nowInMillis);
         }
 
         ScriptModule createScriptModule(List<ScriptPlugin> scriptPlugins) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java
new file mode 100644
index 00000000000..392dcf1542a
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public abstract class AbstractSerializingTestCase<T extends ToXContent & Writeable> extends AbstractWireSerializingTestCase<T> {
+
+    /**
+     * Generic test that creates new instance from the test instance and checks
+     * both for equality and asserts equality on the two instances.
+     */
+    public void testFromXContent() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            XContentType xContentType = randomFrom(XContentType.values());
+            XContentBuilder builder = toXContent(testInstance, xContentType);
+            XContentBuilder shuffled = shuffleXContent(builder);
+            assertParsedInstance(xContentType, shuffled.bytes(), testInstance);
+            for (Map.Entry<String, T> alternateVersion : getAlternateVersions().entrySet()) {
+                String instanceAsString = alternateVersion.getKey();
+                assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue());
+            }
+        }
+    }
+
+    private void assertParsedInstance(XContentType xContentType, BytesReference instanceAsBytes, T expectedInstance)
+            throws IOException {
+
+        XContentParser parser = createParser(XContentFactory.xContent(xContentType), instanceAsBytes);
+        T newInstance = parseInstance(parser);
+        assertNotSame(newInstance, expectedInstance);
+        assertEquals(expectedInstance, newInstance);
+        assertEquals(expectedInstance.hashCode(), newInstance.hashCode());
+    }
+
+    private T parseInstance(XContentParser parser) throws IOException {
+        T parsedInstance = doParseInstance(parser);
+        assertNull(parser.nextToken());
+        return parsedInstance;
+    }
+
+    /**
+     * Parses to a new instance using the provided {@link XContentParser}
+     */
+    protected abstract T doParseInstance(XContentParser parser) throws IOException;
+
+    /**
+     * Renders the provided instance in XContent
+     *
+     * @param instance
+     *            the instance to render
+     * @param contentType
+     *            the content type to render to
+     */
+    protected XContentBuilder toXContent(T instance, XContentType contentType)
+            throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        if (instance.isFragment()) {
+            builder.startObject();
+        }
+        instance.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        if (instance.isFragment()) {
+            builder.endObject();
+        }
+        return builder;
+    }
+
+    /**
+     * Returns alternate string representation of the instance that need to be
+     * tested as they are never used as output of the test instance. By default
+     * there are no alternate versions.
+     *
+     * These alternatives must be JSON strings.
+     */
+    protected Map<String, T> getAlternateVersions() {
+        return Collections.emptyMap();
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java
new file mode 100644
index 00000000000..bdf9e4badd8
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Streamable;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class AbstractStreamableTestCase<T extends Streamable> extends ESTestCase {
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    /**
+     * Creates a random test instance to use in the tests. This method will be
+     * called multiple times during test execution and should return a different
+     * random instance each time it is called.
+     */
+    protected abstract T createTestInstance();
+
+    /**
+     * Creates an empty instance to use when deserialising the
+     * {@link Streamable}. This usually returns an instance created using the
+     * zer-arg constructor
+     */
+    protected abstract T createBlankInstance();
+
+    /**
+     * Tests that the equals and hashcode methods are consistent and copied
+     * versions of the instance have are equal.
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T firstInstance = createTestInstance();
+            assertFalse("instance is equal to null", firstInstance.equals(null));
+            assertFalse("instance is equal to incompatible type", firstInstance.equals(""));
+            assertEquals("instance is not equal to self", firstInstance, firstInstance);
+            assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T secondInstance = copyInstance(firstInstance, Version.CURRENT);
+            assertEquals("instance is not equal to self", secondInstance, secondInstance);
+            assertEquals("instance is not equal to its copy", firstInstance, secondInstance);
+            assertEquals("equals is not symmetric", secondInstance, firstInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T thirdInstance = copyInstance(secondInstance, Version.CURRENT);
+            assertEquals("instance is not equal to self", thirdInstance, thirdInstance);
+            assertEquals("instance is not equal to its copy", secondInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not transitive", firstInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not symmetric", thirdInstance, secondInstance);
+            assertEquals("equals is not symmetric", thirdInstance, firstInstance);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance.
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            assertSerialization(testInstance);
+        }
+    }
+
+    /**
+     * Serialize the given instance and asserts that both are equal
+     */
+    protected T assertSerialization(T testInstance) throws IOException {
+        T deserializedInstance = copyInstance(testInstance, Version.CURRENT);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    /**
+     * Round trip {@code instance} through binary serialization, setting the wire compatibility version to {@code version}.
+     */
+    protected T copyInstance(T instance, Version version) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            output.setVersion(version);
+            instance.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(),
+                    getNamedWriteableRegistry())) {
+                in.setVersion(version);
+                T newInstance = createBlankInstance();
+                newInstance.readFrom(in);
+                return newInstance;
+            }
+        }
+    }
+
+    /**
+     * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
+     * 
+     * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize.
+     * 
+     * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s
+     */
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.emptyList());
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java
new file mode 100644
index 00000000000..79bd9d09223
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+public abstract class AbstractStreamableXContentTestCase<T extends ToXContent & Streamable> extends AbstractStreamableTestCase<T> {
+
+    /**
+     * Generic test that creates new instance from the test instance and checks
+     * both for equality and asserts equality on the two queries.
+     */
+    public void testFromXContent() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            XContentType xContentType = randomFrom(XContentType.values());
+            XContentBuilder builder = toXContent(testInstance, xContentType);
+            XContentBuilder shuffled = shuffleXContent(builder);
+            assertParsedInstance(xContentType, shuffled.bytes(), testInstance);
+            for (Map.Entry<String, T> alternateVersion : getAlternateVersions().entrySet()) {
+                String instanceAsString = alternateVersion.getKey();
+                assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue());
+            }
+        }
+    }
+
+    private void assertParsedInstance(XContentType xContentType, BytesReference instanceAsBytes, T expectedInstance)
+            throws IOException {
+        XContentParser parser = createParser(XContentFactory.xContent(xContentType), instanceAsBytes);
+        T newInstance = parseInstance(parser);
+        assertNotSame(newInstance, expectedInstance);
+        assertEquals(expectedInstance, newInstance);
+        assertEquals(expectedInstance.hashCode(), newInstance.hashCode());
+    }
+
+    private T parseInstance(XContentParser parser) throws IOException {
+        T parsedInstance = doParseInstance(parser);
+        assertNull(parser.nextToken());
+        return parsedInstance;
+    }
+
+    /**
+     * Parses to a new instance using the provided {@link XContentParser}
+     */
+    protected abstract T doParseInstance(XContentParser parser);
+
+    /**
+     * Renders the provided instance in XContent
+     * 
+     * @param instance
+     *            the instance to render
+     * @param contentType
+     *            the content type to render to
+     */
+    protected static <T extends ToXContent> XContentBuilder toXContent(T instance, XContentType contentType)
+            throws IOException {
+        XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        instance.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        return builder;
+    }
+
+    /**
+     * Returns alternate string representation of the instance that need to be
+     * tested as they are never used as output of the test instance. By default
+     * there are no alternate versions.
+     * 
+     * These alternatives must be JSON strings.
+     */
+    protected Map<String, T> getAlternateVersions() {
+        return Collections.emptyMap();
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
new file mode 100644
index 00000000000..f260ab3975b
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public abstract class AbstractWireSerializingTestCase<T extends Writeable> extends ESTestCase {
+    protected static final int NUMBER_OF_TEST_RUNS = 20;
+
+    /**
+     * Creates a random test instance to use in the tests. This method will be
+     * called multiple times during test execution and should return a different
+     * random instance each time it is called.
+     */
+    protected abstract T createTestInstance();
+
+    /**
+     * Returns a {@link Reader} that can be used to de-serialize the instance
+     */
+    protected abstract Reader<T> instanceReader();
+
+    /**
+     * Tests that the equals and hashcode methods are consistent and copied
+     * versions of the instance have are equal.
+     */
+    public void testEqualsAndHashcode() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T firstInstance = createTestInstance();
+            assertFalse("instance is equal to null", firstInstance.equals(null));
+            assertFalse("instance is equal to incompatible type", firstInstance.equals(""));
+            assertEquals("instance is not equal to self", firstInstance, firstInstance);
+            assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T secondInstance = copyInstance(firstInstance);
+            assertEquals("instance is not equal to self", secondInstance, secondInstance);
+            if (false == firstInstance.equals(secondInstance)) {
+                firstInstance.equals(secondInstance);
+            assertEquals("instance is not equal to its copy", firstInstance, secondInstance);
+            }
+            assertEquals("equals is not symmetric", secondInstance, firstInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(firstInstance.hashCode()));
+
+            T thirdInstance = copyInstance(secondInstance);
+            assertEquals("instance is not equal to self", thirdInstance, thirdInstance);
+            assertEquals("instance is not equal to its copy", secondInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not transitive", firstInstance, thirdInstance);
+            assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(),
+                    equalTo(thirdInstance.hashCode()));
+            assertEquals("equals is not symmetric", thirdInstance, secondInstance);
+            assertEquals("equals is not symmetric", thirdInstance, firstInstance);
+        }
+    }
+
+    /**
+     * Test serialization and deserialization of the test instance.
+     */
+    public void testSerialization() throws IOException {
+        for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) {
+            T testInstance = createTestInstance();
+            assertSerialization(testInstance);
+        }
+    }
+
+    /**
+     * Serialize the given instance and asserts that both are equal
+     */
+    protected T assertSerialization(T testInstance) throws IOException {
+        T deserializedInstance = copyInstance(testInstance);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+    private T copyInstance(T instance) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            instance.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(),
+                    getNamedWriteableRegistry())) {
+                return instanceReader().read(in);
+            }
+        }
+    }
+
+    /**
+     * Get the {@link NamedWriteableRegistry} to use when de-serializing the object.
+     * 
+     * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize.
+     * 
+     * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s
+     */
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        return new NamedWriteableRegistry(Collections.emptyList());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index 3e3896dfc2c..b5c77a71aaf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -20,15 +20,15 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
@@ -50,21 +50,22 @@ public class ClusterServiceUtils {
     public static ClusterService createClusterService(ThreadPool threadPool, DiscoveryNode localNode) {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool);
-        clusterService.setLocalNode(localNode);
+                threadPool, () -> localNode);
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
-            public void connectToNodes(List<DiscoveryNode> addedNodes) {
+            public void connectToNodes(Iterable<DiscoveryNode> discoveryNodes) {
                 // skip
             }
 
             @Override
-            public void disconnectFromNodes(List<DiscoveryNode> removedNodes) {
+            public void disconnectFromNodesExcept(Iterable<DiscoveryNode> nodesToKeep) {
                 // skip
             }
         });
         clusterService.setClusterStatePublisher((event, ackListener) -> {
         });
+        clusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
         clusterService.start();
         final DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterService.state().nodes());
         nodes.masterNodeId(clusterService.localNode().getId());
@@ -84,16 +85,11 @@ public class ClusterServiceUtils {
 
     public static void setState(ClusterService clusterService, ClusterState clusterState) {
         CountDownLatch latch = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("test setting state", new ClusterStateUpdateTask() {
+        clusterService.submitStateUpdateTask("test setting state", new LocalClusterUpdateTask() {
             @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 // make sure we increment versions as listener may depend on it for change
-                return ClusterState.builder(clusterState).version(currentState.version() + 1).build();
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
+                return newState(ClusterState.builder(clusterState).version(currentState.version() + 1).build());
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java
new file mode 100644
index 00000000000..14d8ec02628
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/DiffableTestUtils.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.cluster.Diff;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
+
+import java.io.IOException;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import static org.elasticsearch.test.AbstractWireSerializingTestCase.NUMBER_OF_TEST_RUNS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * Utilities that simplify testing of diffable classes
+ */
+public final class DiffableTestUtils {
+    protected static final int NUMBER_OF_DIFF_TEST_RUNS = NUMBER_OF_TEST_RUNS;
+
+    private DiffableTestUtils() {
+
+    }
+
+    /**
+     * Asserts that changes are applied correctly, i.e. that applying diffs to localInstance produces that object
+     * equal but not the same as the remoteChanges instance.
+     */
+    public static <T extends Diffable<T>> T assertDiffApplication(T remoteChanges, T localInstance, Diff<T> diffs) {
+        T localChanges = diffs.apply(localInstance);
+        assertEquals(remoteChanges, localChanges);
+        assertEquals(remoteChanges.hashCode(), localChanges.hashCode());
+        assertNotSame(remoteChanges, localChanges);
+        return localChanges;
+    }
+
+    /**
+     * Simulates sending diffs over the wire
+     */
+    public static <T extends Writeable> T copyInstance(T diffs, NamedWriteableRegistry namedWriteableRegistry,
+                                                                  Reader<T> reader) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            diffs.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                return reader.read(in);
+            }
+        }
+    }
+
+    /**
+     * Tests making random changes to an object, calculating diffs for these changes, sending this
+     * diffs over the wire and appling these diffs on the other side.
+     */
+    public static <T extends Diffable<T>> void testDiffableSerialization(Supplier<T> testInstance,
+                                                                         Function<T, T> modifier,
+                                                                         NamedWriteableRegistry namedWriteableRegistry,
+                                                                         Reader<T> reader,
+                                                                         Reader<Diff<T>> diffReader) throws IOException {
+        T remoteInstance = testInstance.get();
+        T localInstance = assertSerialization(remoteInstance, namedWriteableRegistry, reader);
+        for (int runs = 0; runs < NUMBER_OF_DIFF_TEST_RUNS; runs++) {
+            T remoteChanges = modifier.apply(remoteInstance);
+            Diff<T> remoteDiffs = remoteChanges.diff(remoteInstance);
+            Diff<T> localDiffs = copyInstance(remoteDiffs, namedWriteableRegistry, diffReader);
+            localInstance = assertDiffApplication(remoteChanges, localInstance, localDiffs);
+            remoteInstance = remoteChanges;
+        }
+    }
+
+    /**
+     * Asserts that testInstance can be correctly.
+     */
+    public static  <T extends Writeable> T assertSerialization(T testInstance, NamedWriteableRegistry namedWriteableRegistry,
+                                                          Reader<T> reader) throws IOException {
+        T deserializedInstance = copyInstance(testInstance, namedWriteableRegistry, reader);
+        assertEquals(testInstance, deserializedInstance);
+        assertEquals(testInstance.hashCode(), deserializedInstance.hashCode());
+        assertNotSame(testInstance, deserializedInstance);
+        return deserializedInstance;
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index cb31ac6028b..92aafd4e8a3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -60,6 +61,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -77,6 +79,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.regex.Regex;
@@ -88,10 +91,12 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.discovery.Discovery;
@@ -339,6 +344,13 @@ public abstract class ESIntegTestCase extends ESTestCase {
         initializeSuiteScope();
     }
 
+    @Override
+    protected final boolean enableWarningsCheck() {
+        //In an integ test it doesn't make sense to keep track of warnings: if the cluster is external the warnings are in another jvm,
+        //if the cluster is internal the deprecation logger is shared across all nodes
+        return false;
+    }
+
     protected final void beforeInternal() throws Exception {
         final Scope currentClusterScope = getCurrentClusterScope();
         switch (currentClusterScope) {
@@ -865,7 +877,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             getExcludeSettings(index, n, builder);
         }
         Settings build = builder.build();
-        if (!build.getAsMap().isEmpty()) {
+        if (!build.isEmpty()) {
             logger.debug("allowNodes: updating [{}]'s setting to [{}]", index, build.toDelimitedString(';'));
             client().admin().indices().prepareUpdateSettings(index).setSettings(build).execute().actionGet();
         }
@@ -1076,10 +1088,18 @@ public abstract class ESIntegTestCase extends ESTestCase {
      */
     protected void ensureClusterStateConsistency() throws IOException {
         if (cluster() != null && cluster().size() > 0) {
+            final NamedWriteableRegistry namedWriteableRegistry;
+            if (isInternalCluster()) {
+                // If it's internal cluster - using existing registry in case plugin registered custom data
+                namedWriteableRegistry = internalCluster().getInstance(NamedWriteableRegistry.class);
+            } else {
+                // If it's external cluster - fall back to the standard set
+                namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
+            }
             ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState();
             byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState);
             // remove local node reference
-            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null);
+            masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry);
             Map<String, Object> masterStateMap = convertToMap(masterClusterState);
             int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length;
             String masterId = masterClusterState.nodes().getMasterNodeId();
@@ -1087,7 +1107,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState();
                 byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState);
                 // remove local node reference
-                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null);
+                localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry);
                 final Map<String, Object> localStateMap = convertToMap(localClusterState);
                 final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length;
                 // Check that the non-master node has the same version of the cluster state as the master and
@@ -1207,10 +1227,10 @@ public abstract class ESIntegTestCase extends ESTestCase {
      *   return client().prepareIndex(index, type, id).setSource(source).execute().actionGet();
      * </pre>
      * <p>
-     * where source is a String.
+     * where source is a JSON String.
      */
     protected final IndexResponse index(String index, String type, String id, String source) {
-        return client().prepareIndex(index, type, id).setSource(source).execute().actionGet();
+        return client().prepareIndex(index, type, id).setSource(source, XContentType.JSON).execute().actionGet();
     }
 
     /**
@@ -1368,7 +1388,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
                 String index = RandomPicks.randomFrom(random, indices);
                 bogusIds.add(new Tuple<>(index, id));
-                builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}"));
+                builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
             }
         }
         final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
@@ -1562,7 +1582,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
     private class LatchedActionListener<Response> implements ActionListener<Response> {
         private final CountDownLatch latch;
 
-        public LatchedActionListener(CountDownLatch latch) {
+        LatchedActionListener(CountDownLatch latch) {
             this.latch = latch;
         }
 
@@ -1590,7 +1610,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         private final CopyOnWriteArrayList<Tuple<T, Exception>> errors;
         private final T builder;
 
-        public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
+        PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
             super(latch);
             this.errors = errors;
             this.builder = builder;
@@ -2106,6 +2126,17 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return builder.build();
     }
 
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        if (isInternalCluster() && cluster().size() > 0) {
+            // If it's internal cluster - using existing registry in case plugin registered custom data
+            return internalCluster().getInstance(NamedXContentRegistry.class);
+        } else {
+            // If it's external cluster - fall back to the standard set
+            return new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
+        }
+    }
+
     /**
      * Returns an instance of {@link RestClient} pointing to the current test cluster.
      * Creates a new client if the method is invoked for the first time in the context of the current test scope.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 0b2adfa52e1..9d99cd6d9be 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -18,13 +18,7 @@
  */
 package org.elasticsearch.test;
 
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-
+import com.carrotsearch.randomizedtesting.RandomizedContext;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
@@ -41,8 +35,10 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
@@ -55,11 +51,16 @@ import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.MockTcpTransportPlugin;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -72,62 +73,56 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
 
     private static Node NODE = null;
 
-    private void reset() throws IOException {
-        assert NODE != null;
-        stopNode();
-        startNode();
-    }
-
-    protected void startNode() {
+    protected void startNode(long seed) throws Exception {
         assert NODE == null;
-        NODE = newNode();
-        // we must wait for the node to actually be up and running. otherwise the node might have started, elected itself master but might not yet have removed the
+        NODE = RandomizedContext.current().runWithPrivateRandomness(seed, this::newNode);
+        // we must wait for the node to actually be up and running. otherwise the node might have started,
+        // elected itself master but might not yet have removed the
         // SERVICE_UNAVAILABLE/1/state not recovered / initialized block
         ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForGreenStatus().get();
         assertFalse(clusterHealthResponse.isTimedOut());
         client().admin().indices()
-            .preparePutTemplate("random_index_template")
+            .preparePutTemplate("one_shard_index_template")
             .setPatterns(Collections.singletonList("*"))
             .setOrder(0)
             .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)).get();
     }
 
-    protected static void stopNode() throws IOException {
+    private static void stopNode() throws IOException {
         Node node = NODE;
         NODE = null;
         IOUtils.close(node);
     }
 
-    private void cleanup(boolean resetNode) throws IOException {
-        assertAcked(client().admin().indices().prepareDelete("*").get());
-        if (resetNode) {
-            reset();
-        }
-        MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
-        assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().getAsMap(),
-                metaData.persistentSettings().getAsMap().size(), equalTo(0));
-        assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(),
-                metaData.transientSettings().getAsMap().size(), equalTo(0));
-    }
-
-    @Before
     @Override
     public void setUp() throws Exception {
         super.setUp();
+        //the seed has to be created regardless of whether it will be used or not, for repeatability
+        long seed = random().nextLong();
         // Create the node lazily, on the first test. This is ok because we do not randomize any settings,
         // only the cluster name. This allows us to have overridden properties for plugins and the version to use.
         if (NODE == null) {
-            startNode();
+            startNode(seed);
         }
     }
 
-    @After
     @Override
     public void tearDown() throws Exception {
         logger.info("[{}#{}]: cleaning up after test", getTestClass().getSimpleName(), getTestName());
         super.tearDown();
-        cleanup(resetNodeAfterTest());
+        assertAcked(client().admin().indices().prepareDelete("*").get());
+        MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
+        assertThat("test leaves persistent cluster metadata behind: " + metaData.persistentSettings().getAsMap(),
+                metaData.persistentSettings().size(), equalTo(0));
+        assertThat("test leaves transient cluster metadata behind: " + metaData.transientSettings().getAsMap(),
+                metaData.transientSettings().size(), equalTo(0));
+        if (resetNodeAfterTest()) {
+            assert NODE != null;
+            stopNode();
+            //the seed can be created within this if as it will either be executed before every test method or will never be.
+            startNode(random().nextLong());
+        }
     }
 
     @BeforeClass
@@ -149,7 +144,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         return false;
     }
 
-
     /** The plugin classes that should be added to the node. */
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return Collections.emptyList();
@@ -170,13 +164,13 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
     private Node newNode() {
         final Path tempDir = createTempDir();
         Settings settings = Settings.builder()
-            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+            .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", random().nextLong()))
             .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
             .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             // TODO: use a consistent data path for custom paths
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
-            .put("node.name", nodeName())
+            .put("node.name", "node_s_0")
             .put("script.inline", "true")
             .put("script.stored", "true")
             .put(ScriptService.SCRIPT_MAX_COMPILATIONS_PER_MINUTE.getKey(), 1000)
@@ -184,6 +178,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put(NetworkModule.HTTP_ENABLED.getKey(), false)
             .put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)
             .put(Node.NODE_DATA_SETTING.getKey(), true)
+            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
         Collection<Class<? extends Plugin>> plugins = getPlugins();
@@ -211,13 +206,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         return NODE.client();
     }
 
-    /**
-     * Returns the single test nodes name.
-     */
-    public String nodeName() {
-        return "node_s_0";
-    }
-
     /**
      * Return a reference to the singleton node.
      */
@@ -273,7 +261,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         // Wait for the index to be allocated so that cluster state updates don't override
         // changes that would have been done locally
         ClusterHealthResponse health = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(index).waitForYellowStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
+                .health(Requests.clusterHealthRequest(index).waitForYellowStatus().waitForEvents(Priority.LANGUID)
+                        .waitForNoRelocatingShards(true)).actionGet();
         assertThat(health.getStatus(), lessThanOrEqualTo(ClusterHealthStatus.YELLOW));
         assertThat("Cluster must be a single node cluster", health.getNumberOfDataNodes(), equalTo(1));
         IndicesService instanceFromNode = getInstanceFromNode(IndicesService.class);
@@ -293,7 +282,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
     protected SearchContext createSearchContext(IndexService indexService) {
         BigArrays bigArrays = indexService.getBigArrays();
         ThreadPool threadPool = indexService.getThreadPool();
-        ScriptService scriptService = node().injector().getInstance(ScriptService.class);
         return new TestSearchContext(threadPool, bigArrays, indexService);
     }
 
@@ -316,7 +304,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
      */
     public ClusterHealthStatus ensureGreen(TimeValue timeout, String... indices) {
         ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID).waitForNoRelocatingShards(true)).actionGet();
+                .health(Requests.clusterHealthRequest(indices).timeout(timeout).waitForGreenStatus().waitForEvents(Priority.LANGUID)
+                        .waitForNoRelocatingShards(true)).actionGet();
         if (actionGet.isTimedOut()) {
             logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState(),
                 client().admin().cluster().preparePendingClusterTasks().get());
@@ -327,5 +316,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         return actionGet.getStatus();
     }
 
-
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return getInstanceFromNode(NamedXContentRegistry.class);
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 9a4085ff2e4..633a04f5f60 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -29,7 +29,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
-
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -47,6 +46,7 @@ import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.PathUtils;
@@ -57,16 +57,22 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
@@ -112,6 +118,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
@@ -127,8 +134,10 @@ import java.util.stream.Collectors;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -173,6 +182,8 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     protected final Logger logger = Loggers.getLogger(getClass());
+    protected final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+    private ThreadContext threadContext;
 
     // -----------------------------------------------------------------
     // Suite and test case setup/cleanup.
@@ -251,16 +262,79 @@ public abstract class ESTestCase extends LuceneTestCase {
     @Before
     public final void before()  {
         logger.info("[{}]: before test", getTestName());
+        assertNull("Thread context initialized twice", threadContext);
+        if (enableWarningsCheck()) {
+            this.threadContext = new ThreadContext(Settings.EMPTY);
+            DeprecationLogger.setThreadContext(threadContext);
+        }
+    }
+
+    /**
+     * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax
+     * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}.
+     */
+    protected boolean enableWarningsCheck() {
+        return true;
     }
 
     @After
     public final void after() throws Exception {
         checkStaticState();
+        // We check threadContext != null rather than enableWarningsCheck()
+        // because after methods are still called in the event that before
+        // methods failed, in which case threadContext might not have been
+        // initialized
+        if (threadContext != null) {
+            ensureNoWarnings();
+            threadContext = null;
+        }
         ensureAllSearchContextsReleased();
         ensureCheckIndexPassed();
         logger.info("[{}]: after test", getTestName());
     }
 
+    private void ensureNoWarnings() throws IOException {
+        //Check that there are no unaccounted warning headers. These should be checked with {@link #assertWarnings(String...)} in the
+        //appropriate test
+        try {
+            final List<String> warnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
+            assertNull("unexpected warning headers", warnings);
+        } finally {
+            resetDeprecationLogger();
+        }
+    }
+
+    protected final void assertWarnings(String... expectedWarnings) {
+        if (enableWarningsCheck() == false) {
+            throw new IllegalStateException("unable to check warning headers if the test is not set to do so");
+        }
+        try {
+            final List<String> actualWarnings = threadContext.getResponseHeaders().get(DeprecationLogger.WARNING_HEADER);
+            for (String msg : expectedWarnings) {
+                assertThat(actualWarnings, hasItem(containsString(msg)));
+            }
+            assertEquals("Expected " + expectedWarnings.length + " warnings but found " + actualWarnings.size() + "\nExpected: "
+                    + Arrays.asList(expectedWarnings) + "\nActual: " + actualWarnings,
+                expectedWarnings.length, actualWarnings.size());
+        } finally {
+            resetDeprecationLogger();
+        }
+    }
+
+    private void resetDeprecationLogger() {
+        // "clear" current warning headers by setting a new ThreadContext
+        DeprecationLogger.removeThreadContext(this.threadContext);
+        try {
+            this.threadContext.close();
+            // catch IOException to avoid that call sites have to deal with it. It is only declared because this class implements Closeable
+            // but it is impossible that this implementation will ever throw an IOException.
+        } catch (IOException ex) {
+            throw new AssertionError("IOException thrown while closing deprecation logger's thread context", ex);
+        }
+        this.threadContext = new ThreadContext(Settings.EMPTY);
+        DeprecationLogger.setThreadContext(this.threadContext);
+    }
+
     private static final List<StatusData> statusData = new ArrayList<>();
     static {
         // ensure that the status logger is set to the warn level so we do not miss any warnings with our Log4j usage
@@ -389,7 +463,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         return random().nextInt();
     }
 
-    public static long randomPositiveLong() {
+    public static long randomNonNegativeLong() {
         long randomLong;
         do {
             randomLong = randomLong();
@@ -743,7 +817,8 @@ public abstract class ESTestCase extends LuceneTestCase {
      */
     public static <T> List<T> randomSubsetOf(int size, Collection<T> collection) {
         if (size > collection.size()) {
-            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a collection of " + collection.size() + " objects");
+            throw new IllegalArgumentException("Can\'t pick " + size + " random objects from a collection of " +
+                    collection.size() + " objects");
         }
         List<T> tempList = new ArrayList<>(collection);
         Collections.shuffle(tempList, random());
@@ -787,13 +862,23 @@ public abstract class ESTestCase extends LuceneTestCase {
      * recursive shuffling behavior can be made by passing in the names of fields which
      * internally should stay untouched.
      */
-    public static XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
-        BytesReference bytes = builder.bytes();
-        XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes);
+    public XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
+        try (XContentParser parser = createParser(builder)) {
+            return shuffleXContent(parser, builder.isPrettyPrint(), exceptFieldNames);
+        }
+    }
+
+    /**
+     * Randomly shuffles the fields inside objects parsed using the {@link XContentParser} passed in.
+     * Recursively goes through inner objects and also shuffles them. Exceptions for this
+     * recursive shuffling behavior can be made by passing in the names of fields which
+     * internally should stay untouched.
+     */
+    public XContentBuilder shuffleXContent(XContentParser parser, boolean prettyPrint, String... exceptFieldNames) throws IOException {
         // use ordered maps for reproducibility
         Map<String, Object> shuffledMap = shuffleMap(parser.mapOrdered(), new HashSet<>(Arrays.asList(exceptFieldNames)));
-        XContentBuilder xContentBuilder = XContentFactory.contentBuilder(builder.contentType());
-        if (builder.isPrettyPrint()) {
+        XContentBuilder xContentBuilder = XContentFactory.contentBuilder(parser.contentType());
+        if (prettyPrint) {
             xContentBuilder.prettyPrint();
         }
         return xContentBuilder.map(shuffledMap);
@@ -881,39 +966,78 @@ public abstract class ESTestCase extends LuceneTestCase {
         assertThat(count + " files exist that should have been cleaned:\n" + sb.toString(), count, equalTo(0));
     }
 
+    /**
+     * Assert that two objects are equals, calling {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)} to print out their
+     * differences if they aren't equal.
+     */
+    public static <T extends ToXContent> void assertEqualsWithErrorMessageFromXContent(T expected, T actual) {
+        if (Objects.equals(expected, actual)) {
+            return;
+        }
+        if (expected == null) {
+            throw new AssertionError("Expected null be actual was [" + actual.toString() + "]");
+        }
+        if (actual == null) {
+            throw new AssertionError("Didn't expect null but actual was [null]");
+        }
+        try (XContentBuilder actualJson = JsonXContent.contentBuilder();
+                XContentBuilder expectedJson = JsonXContent.contentBuilder()) {
+            actualJson.startObject();
+            actual.toXContent(actualJson, ToXContent.EMPTY_PARAMS);
+            actualJson.endObject();
+            expectedJson.startObject();
+            expected.toXContent(expectedJson, ToXContent.EMPTY_PARAMS);
+            expectedJson.endObject();
+            NotEqualMessageBuilder message = new NotEqualMessageBuilder();
+            message.compareMaps(
+                    XContentHelper.convertToMap(actualJson.bytes(), false).v2(),
+                    XContentHelper.convertToMap(expectedJson.bytes(), false).v2());
+            throw new AssertionError("Didn't match expected value:\n" + message);
+        } catch (IOException e) {
+            throw new AssertionError("IOException while building failure message", e);
+        }
+    }
+
     /**
      * Create a new {@link XContentParser}.
      */
     protected final XContentParser createParser(XContentBuilder builder) throws IOException {
-        return builder.generator().contentType().xContent().createParser(builder.bytes());
+        return builder.generator().contentType().xContent().createParser(xContentRegistry(), builder.bytes());
     }
 
     /**
      * Create a new {@link XContentParser}.
      */
     protected final XContentParser createParser(XContent xContent, String data) throws IOException {
-        return xContent.createParser(data);
+        return xContent.createParser(xContentRegistry(), data);
     }
 
     /**
      * Create a new {@link XContentParser}.
      */
     protected final XContentParser createParser(XContent xContent, InputStream data) throws IOException {
-        return xContent.createParser(data);
+        return xContent.createParser(xContentRegistry(), data);
     }
 
     /**
      * Create a new {@link XContentParser}.
      */
     protected final XContentParser createParser(XContent xContent, byte[] data) throws IOException {
-        return xContent.createParser(data);
+        return xContent.createParser(xContentRegistry(), data);
     }
 
     /**
      * Create a new {@link XContentParser}.
      */
     protected final XContentParser createParser(XContent xContent, BytesReference data) throws IOException {
-        return xContent.createParser(data);
+        return xContent.createParser(xContentRegistry(), data);
+    }
+
+    /**
+     * The {@link NamedXContentRegistry} to use for this test. Subclasses should override and use liberally.
+     */
+    protected NamedXContentRegistry xContentRegistry() {
+        return new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
     }
 
     /** Returns the suite failure marker: internal use only! */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java
index bf1cd8132da..76cfcce033b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/EqualsHashCodeTestUtils.java
@@ -78,7 +78,8 @@ public class EqualsHashCodeTestUtils {
             assertThat(objectName + " hashcode returns different values if called multiple times", original.hashCode(),
                     equalTo(original.hashCode()));
             if (mutationFunction != null) {
-                assertThat(objectName + " mutation should not be equal to original", mutationFunction.mutate(original),
+                T mutation = mutationFunction.mutate(original);
+                assertThat(objectName + " mutation should not be equal to original", mutation,
                         not(equalTo(original)));
             }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 38c4f6003be..85278358bec 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -27,7 +27,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
@@ -85,7 +84,7 @@ import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
-import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.node.NodeService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
@@ -132,6 +131,7 @@ import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
 import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
+import static org.elasticsearch.test.ESTestCase.awaitBusy;
 import static org.elasticsearch.test.ESTestCase.randomFrom;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -1052,21 +1052,38 @@ public final class InternalTestCluster extends TestCluster {
         logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
     }
 
-    /** ensure a cluster is form with {@link #nodes}.size() nodes. */
+    /** ensure a cluster is formed with all published nodes. */
     private void validateClusterFormed() {
         String name = randomFrom(random, getNodeNames());
         validateClusterFormed(name);
     }
 
-    /** ensure a cluster is form with {@link #nodes}.size() nodes, but do so by using the client of the specified node */
+    /** ensure a cluster is formed with all published nodes, but do so by using the client of the specified node */
     private void validateClusterFormed(String viaNode) {
-        final int size = nodes.size();
-        logger.trace("validating cluster formed via [{}], expecting [{}]", viaNode, size);
+        Set<DiscoveryNode> expectedNodes = new HashSet<>();
+        for (NodeAndClient nodeAndClient : nodes.values()) {
+            expectedNodes.add(getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode());
+        }
+        logger.trace("validating cluster formed via [{}], expecting {}", viaNode, expectedNodes);
         final Client client = client(viaNode);
-        ClusterHealthResponse response = client.admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(size)).get();
-        if (response.isTimedOut()) {
-            logger.warn("failed to wait for a cluster of size [{}], got [{}]", size, response);
-            throw new IllegalStateException("cluster failed to reach the expected size of [" + size + "]");
+        try {
+            if (awaitBusy(() -> {
+                DiscoveryNodes discoveryNodes = client.admin().cluster().prepareState().get().getState().nodes();
+                if (discoveryNodes.getSize() != expectedNodes.size()) {
+                    return false;
+                }
+                for (DiscoveryNode expectedNode : expectedNodes) {
+                    if (discoveryNodes.nodeExists(expectedNode) == false) {
+                        return false;
+                    }
+                }
+                return true;
+            }, 30, TimeUnit.SECONDS) == false) {
+                throw new IllegalStateException("cluster failed to from with expected nodes " + expectedNodes + " and actual nodes " +
+                    client.admin().cluster().prepareState().get().getState().nodes());
+            }
+        } catch (InterruptedException e) {
+            throw new IllegalStateException(e);
         }
     }
 
@@ -1128,13 +1145,8 @@ public final class InternalTestCluster extends TestCluster {
                                     .map(task -> task.taskInfo(localNode.getId(), true))
                                     .collect(Collectors.toList());
                             ListTasksResponse response = new ListTasksResponse(taskInfos, Collections.emptyList(), Collections.emptyList());
-                            XContentBuilder builder = null;
                             try {
-                                builder = XContentFactory.jsonBuilder()
-                                        .prettyPrint()
-                                        .startObject()
-                                        .value(response)
-                                        .endObject();
+                                XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint().value(response);
                                 throw new AssertionError("expected index shard counter on shard " + indexShard.shardId() + " on node " +
                                         nodeAndClient.name + " to be 0 but was " + activeOperationsCount + ". Current replication tasks on node:\n" +
                                         builder.string());
@@ -1834,7 +1846,7 @@ public final class InternalTestCluster extends TestCluster {
     private static final class MasterNodePredicate implements Predicate<NodeAndClient> {
         private final String masterNodeName;
 
-        public MasterNodePredicate(String masterNodeName) {
+        MasterNodePredicate(String masterNodeName) {
             this.masterNodeName = masterNodeName;
         }
 
@@ -1925,8 +1937,7 @@ public final class InternalTestCluster extends TestCluster {
     private static final class NodeNamePredicate implements Predicate<Settings> {
         private final HashSet<String> nodeNames;
 
-
-        public NodeNamePredicate(HashSet<String> nodeNames) {
+        NodeNamePredicate(HashSet<String> nodeNames) {
             this.nodeNames = nodeNames;
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java b/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java
new file mode 100644
index 00000000000..e187be8c7d0
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/NotEqualMessageBuilder.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import org.elasticsearch.common.Nullable;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+
+/**
+ * Builds a message describing how two sets of values are unequal. 
+ */
+public class NotEqualMessageBuilder {
+    private final StringBuilder message;
+    private int indent = 0;
+
+    /**
+     * The name of the field being compared.
+     */
+    public NotEqualMessageBuilder() {
+        this.message = new StringBuilder();
+    }
+
+    /**
+     * The failure message.
+     */
+    @Override
+    public String toString() {
+        return message.toString();
+    }
+
+    /**
+     * Compare two maps.
+     */
+    public void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
+        actual = new TreeMap<>(actual);
+        expected = new TreeMap<>(expected);
+        for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+            compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
+        }
+        for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
+            field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
+        }
+    }
+
+    /**
+     * Compare two lists.
+     */
+    public void compareLists(List<Object> actual, List<Object> expected) {
+        int i = 0;
+        while (i < actual.size() && i < expected.size()) {
+            compare(Integer.toString(i), actual.get(i), expected.get(i));
+            i++;
+        }
+        if (actual.size() == expected.size()) {
+            return;
+        }
+        indent();
+        if (actual.size() < expected.size()) {
+            message.append("expected [").append(expected.size() - i).append("] more entries\n");
+            return;
+        }
+        message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
+    }
+
+    /**
+     * Compare two values.
+     * @param field the name of the field being compared.
+     */
+    public void compare(String field, @Nullable Object actual, Object expected) {
+        if (expected instanceof Map) {
+            if (actual == null) {
+                field(field, "expected map but not found");
+                return;
+            }
+            if (false == actual instanceof Map) {
+                field(field, "expected map but found [" + actual + "]");
+                return;
+            }
+            @SuppressWarnings("unchecked")
+            Map<String, Object> expectedMap = (Map<String, Object>) expected;
+            @SuppressWarnings("unchecked")
+            Map<String, Object> actualMap = (Map<String, Object>) actual;
+            if (expectedMap.isEmpty() && actualMap.isEmpty()) {
+                field(field, "same [empty map]");
+                return;
+            }
+            field(field, null);
+            indent += 1;
+            compareMaps(actualMap, expectedMap);
+            indent -= 1;
+            return;
+        }
+        if (expected instanceof List) {
+            if (actual == null) {
+                field(field, "expected list but not found");
+                return;
+            }
+            if (false == actual instanceof List) {
+                field(field, "expected list but found [" + actual + "]");
+                return;
+            }
+            @SuppressWarnings("unchecked")
+            List<Object> expectedList = (List<Object>) expected;
+            @SuppressWarnings("unchecked")
+            List<Object> actualList = (List<Object>) actual;
+            if (expectedList.isEmpty() && actualList.isEmpty()) {
+                field(field, "same [empty list]");
+                return;
+            }
+            field(field, null);
+            indent += 1;
+            compareLists(actualList, expectedList);
+            indent -= 1;
+            return;
+        }
+        if (actual == null) {
+            field(field, "expected [" + expected + "] but not found");
+            return;
+        }
+        if (Objects.equals(expected, actual)) {
+            if (expected instanceof String) {
+                String expectedString = (String) expected;
+                if (expectedString.length() > 50) {
+                    expectedString = expectedString.substring(0, 50) + "...";
+                }
+                field(field, "same [" + expectedString + "]");
+                return;
+            }
+            field(field, "same [" + expected + "]");
+            return;
+        }
+        field(field, "expected [" + expected + "] but was [" + actual + "]");
+    }
+
+    private void indent() {
+        for (int i = 0; i < indent; i++) {
+            message.append("  ");
+        }
+    }
+
+    private void field(Object name, String info) {
+        indent();
+        message.append(String.format(Locale.ROOT, "%30s: ", name));
+        if (info != null) {
+            message.append(info);
+        }
+        message.append('\n');
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java b/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
index 7f43c9de61b..81d8fa84a19 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/OldIndexUtils.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.IndexFolderUpgrader;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.MergePolicyConfig;
 import org.elasticsearch.index.engine.Segment;
@@ -131,7 +132,8 @@ public class OldIndexUtils {
                 }
             }
             assertThat(indexFolders.toString(), indexFolders.size(), equalTo(1));
-            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, indexFolders.get(0));
+            final IndexMetaData indexMetaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY,
+                indexFolders.get(0));
             assertNotNull(indexMetaData);
             assertThat(indexFolders.get(0).getFileName().toString(), equalTo(indexMetaData.getIndexUUID()));
             assertThat(indexMetaData.getCreationVersion(), equalTo(version));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
new file mode 100644
index 00000000000..7611c17492e
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/RandomObjects.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.RoutingMissingException;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.IndexShardRecoveringException;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Random;
+
+import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength;
+import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomUnicodeOfLengthBetween;
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+
+public final class RandomObjects {
+
+    private RandomObjects() {
+
+    }
+
+    /**
+     * Returns a tuple containing random stored field values and their corresponding expected values once printed out
+     * via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
+     * {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
+     * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
+     * {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
+     *
+     * @param random Random generator
+     * @param xContentType the content type, used to determine what the expected values are for float numbers.
+     */
+    public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
+        int numValues = randomIntBetween(random, 1, 5);
+        List<Object> originalValues = new ArrayList<>();
+        List<Object> expectedParsedValues = new ArrayList<>();
+        int dataType = randomIntBetween(random, 0, 8);
+        for (int i = 0; i < numValues; i++) {
+            switch(dataType) {
+                case 0:
+                    long randomLong = random.nextLong();
+                    originalValues.add(randomLong);
+                    expectedParsedValues.add(randomLong);
+                    break;
+                case 1:
+                    int randomInt = random.nextInt();
+                    originalValues.add(randomInt);
+                    expectedParsedValues.add(randomInt);
+                    break;
+                case 2:
+                    Short randomShort = (short) random.nextInt();
+                    originalValues.add(randomShort);
+                    expectedParsedValues.add(randomShort.intValue());
+                    break;
+                case 3:
+                    Byte randomByte = (byte)random.nextInt();
+                    originalValues.add(randomByte);
+                    expectedParsedValues.add(randomByte.intValue());
+                    break;
+                case 4:
+                    double randomDouble = random.nextDouble();
+                    originalValues.add(randomDouble);
+                    expectedParsedValues.add(randomDouble);
+                    break;
+                case 5:
+                    Float randomFloat = random.nextFloat();
+                    originalValues.add(randomFloat);
+                    if (xContentType == XContentType.CBOR) {
+                        //with CBOR we get back a float
+                        expectedParsedValues.add(randomFloat);
+                    } else if (xContentType == XContentType.SMILE) {
+                        //with SMILE we get back a double
+                        expectedParsedValues.add(randomFloat.doubleValue());
+                    } else {
+                        //with JSON AND YAML we get back a double, but with float precision.
+                        expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
+                    }
+                    break;
+                case 6:
+                    boolean randomBoolean = random.nextBoolean();
+                    originalValues.add(randomBoolean);
+                    expectedParsedValues.add(randomBoolean);
+                    break;
+                case 7:
+                    String randomString = random.nextBoolean() ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10 ) :
+                            randomUnicodeOfLengthBetween(random, 3, 10);
+                    originalValues.add(randomString);
+                    expectedParsedValues.add(randomString);
+                    break;
+                case 8:
+                    byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
+                    BytesArray randomBytesArray = new BytesArray(randomBytes);
+                    originalValues.add(randomBytesArray);
+                    if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
+                        //JSON and YAML write the base64 format
+                        expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
+                    } else {
+                        //SMILE and CBOR write the original bytes as they support binary format
+                        expectedParsedValues.add(randomBytesArray);
+                    }
+                    break;
+                default:
+                    throw new UnsupportedOperationException();
+            }
+        }
+        return Tuple.tuple(originalValues, expectedParsedValues);
+    }
+
+    /**
+     * Returns a random source containing a random number of fields, objects and array, with maximum depth 5.
+     *
+     * @param random Random generator
+     */
+    public static BytesReference randomSource(Random random) {
+        //the source can be stored in any format and eventually converted when retrieved depending on the format of the response
+        XContentType xContentType = RandomPicks.randomFrom(random, XContentType.values());
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
+            builder.startObject();
+            addFields(random, builder, 0);
+            builder.endObject();
+            return builder.bytes();
+        } catch(IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5.
+     */
+    private static void addFields(Random random, XContentBuilder builder, int currentDepth) throws IOException {
+        int numFields = randomIntBetween(random, 1, 5);
+        for (int i = 0; i < numFields; i++) {
+            if (currentDepth < 5 && random.nextBoolean()) {
+                if (random.nextBoolean()) {
+                    builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
+                    addFields(random, builder, currentDepth + 1);
+                    builder.endObject();
+                } else {
+                    builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
+                    int numElements = randomIntBetween(random, 1, 5);
+                    boolean object = random.nextBoolean();
+                    int dataType = -1;
+                    if (object == false) {
+                        dataType = randomDataType(random);
+                    }
+                    for (int j = 0; j < numElements; j++) {
+                        if (object) {
+                            builder.startObject();
+                            addFields(random, builder, 5);
+                            builder.endObject();
+                        } else {
+                            builder.value(randomFieldValue(random, dataType));
+                        }
+                    }
+                    builder.endArray();
+                }
+            } else {
+                builder.field(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10),
+                        randomFieldValue(random, randomDataType(random)));
+            }
+        }
+    }
+
+    private static int randomDataType(Random random) {
+        return randomIntBetween(random, 0, 3);
+    }
+
+    private static Object randomFieldValue(Random random, int dataType) {
+        switch(dataType) {
+            case 0:
+                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+            case 1:
+                return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
+            case 2:
+                return random.nextLong();
+            case 3:
+                return random.nextDouble();
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Returns a random {@link ShardInfo} object with on or more {@link ShardInfo.Failure} if requested.
+     *
+     * @param random   Random generator
+     * @param failures If true, the {@link ShardInfo} will have random failures
+     * @return a random {@link ShardInfo}
+     */
+    public static ShardInfo randomShardInfo(Random random, boolean failures) {
+        int total = randomIntBetween(random, 1, 10);
+        if (failures == false) {
+            return new ShardInfo(total, total);
+        }
+
+        int successful = randomIntBetween(random, 1, total);
+        return new ShardInfo(total, successful, randomShardInfoFailures(random, Math.max(1, (total - successful))));
+    }
+
+    public static ShardInfo.Failure[] randomShardInfoFailures(Random random, int nbFailures) {
+        List<ShardInfo.Failure> randomFailures = new ArrayList<>(nbFailures);
+        for (int i = 0; i < nbFailures; i++) {
+            randomFailures.add(randomShardInfoFailure(random));
+        }
+        return randomFailures.toArray(new ShardInfo.Failure[nbFailures]);
+    }
+
+    public static ShardInfo.Failure randomShardInfoFailure(Random random) {
+        String index = randomAsciiOfLength(random, 5);
+        String indexUuid = randomAsciiOfLength(random, 5);
+        int shardId = randomIntBetween(random, 1, 10);
+        ShardId shard = new ShardId(index, indexUuid, shardId);
+        RestStatus restStatus = randomFrom(RestStatus.values());
+        Exception exception = RandomPicks.randomFrom(random, Arrays.asList(new IndexShardRecoveringException(shard),
+                new ElasticsearchException(new IllegalArgumentException("Argument is wrong")),
+                new RoutingMissingException(index, randomAsciiOfLength(random, 5), randomAsciiOfLength(random, 5))));
+        return new ShardInfo.Failure(shard, randomAsciiOfLength(random, 3), exception, restStatus, random.nextBoolean());
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
index a655f17faca..fb709a7fd06 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetaData.java
@@ -21,15 +21,21 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.AbstractDiffable;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.Diffable;
+import org.elasticsearch.cluster.NamedDiff;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
-public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custom> implements MetaData.Custom {
+public abstract class TestCustomMetaData extends AbstractNamedDiffable<MetaData.Custom> implements MetaData.Custom {
     private final String data;
 
     protected TestCustomMetaData(String data) {
@@ -57,11 +63,12 @@ public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custo
         return data.hashCode();
     }
 
-    protected abstract TestCustomMetaData newTestCustomMetaData(String data);
+    protected static <T extends TestCustomMetaData> T readFrom(Function<String, T> supplier, StreamInput in) throws IOException {
+        return supplier.apply(in.readString());
+    }
 
-    @Override
-    public MetaData.Custom readFrom(StreamInput in) throws IOException {
-        return newTestCustomMetaData(in.readString());
+    public static NamedDiff<MetaData.Custom> readDiffFrom(String name, StreamInput in)  throws IOException {
+        return readDiffFrom(MetaData.Custom.class, name, in);
     }
 
     @Override
@@ -69,8 +76,9 @@ public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custo
         out.writeString(getData());
     }
 
-    @Override
-    public MetaData.Custom fromXContent(XContentParser parser) throws IOException {
+    @SuppressWarnings("unchecked")
+    public static <T extends MetaData.Custom> T fromXContent(Function<String, MetaData.Custom> supplier, XContentParser parser)
+        throws IOException {
         XContentParser.Token token;
         String data = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -91,7 +99,7 @@ public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custo
         if (data == null) {
             throw new ElasticsearchParseException("failed to parse snapshottable metadata, data not found");
         }
-        return newTestCustomMetaData(data);
+        return (T) supplier.apply(data);
     }
 
     @Override
@@ -102,6 +110,6 @@ public abstract class TestCustomMetaData extends AbstractDiffable<MetaData.Custo
 
     @Override
     public String toString() {
-        return "[" + type() + "][" + data +"]";
+        return "[" + getWriteableName() + "][" + data +"]";
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index b21142bd407..bec80036230 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchTask;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
@@ -41,6 +40,7 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
+import org.elasticsearch.search.collapse.CollapseContext;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
@@ -91,7 +91,6 @@ public class TestSearchContext extends SearchContext {
     private final Map<String, SearchExtBuilder> searchExtBuilders = new HashMap<>();
 
     public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, IndexService indexService) {
-        super(ParseFieldMatcher.STRICT);
         this.bigArrays = bigArrays.withCircuitBreaking();
         this.indexService = indexService;
         this.indexFieldDataService = indexService.fieldData();
@@ -102,7 +101,6 @@ public class TestSearchContext extends SearchContext {
     }
 
     public TestSearchContext(QueryShardContext queryShardContext) {
-        super(ParseFieldMatcher.STRICT);
         this.bigArrays = null;
         this.indexService = null;
         this.indexFieldDataService = null;
@@ -366,6 +364,16 @@ public class TestSearchContext extends SearchContext {
         return null;
     }
 
+    @Override
+    public SearchContext collapse(CollapseContext collapse) {
+        return null;
+    }
+
+    @Override
+    public CollapseContext collapse() {
+        return null;
+    }
+
     @Override
     public SearchContext parsedPostFilter(ParsedQuery postFilter) {
         this.postFilter = postFilter;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
index 31380e25eb3..dcad7187fbe 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/XContentTestUtils.java
@@ -41,7 +41,7 @@ public final class XContentTestUtils {
         builder.startObject();
         part.toXContent(builder, EMPTY_PARAMS);
         builder.endObject();
-        return XContentHelper.convertToMap(builder.bytes(), false).v2();
+        return XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
     }
 
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
index 3fd2b024a1d..f4be0d0d529 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.mocksocket.MockServerSocket;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeConfigurationSource;
 import org.elasticsearch.transport.TransportSettings;
@@ -136,7 +137,7 @@ public class ClusterDiscoveryConfiguration extends NodeConfigurationSource {
             for (int i = 0; i < unicastHostPorts.length; i++) {
                 boolean foundPortInRange = false;
                 while (tries < InternalTestCluster.PORTS_PER_JVM && !foundPortInRange) {
-                    try (ServerSocket serverSocket = new ServerSocket()) {
+                    try (ServerSocket serverSocket = new MockServerSocket()) {
                         // Set SO_REUSEADDR as we may bind here and not be able to reuse the address immediately without it.
                         serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress());
                         serverSocket.bind(new InetSocketAddress(IP_ADDR, nextPort));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
index fe16f034116..b03e157b01c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/MockZenPing.java
@@ -28,10 +28,9 @@ import org.elasticsearch.discovery.zen.PingContextProvider;
 import org.elasticsearch.discovery.zen.ZenPing;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.function.Consumer;
 
 /**
  * A {@link ZenPing} implementation which returns results based on an static in-memory map. This allows pinging
@@ -62,7 +61,7 @@ public final class MockZenPing extends AbstractComponent implements ZenPing {
     }
 
     @Override
-    public void ping(PingListener listener, TimeValue timeout) {
+    public void ping(Consumer<PingCollection> resultsConsumer, TimeValue timeout) {
         logger.info("pinging using mock zen ping");
         synchronized (activeNodesPerCluster) {
             Set<MockZenPing> activeNodes = getActiveNodesForCurrentCluster();
@@ -76,11 +75,12 @@ public final class MockZenPing extends AbstractComponent implements ZenPing {
                 activeNodes = getActiveNodesForCurrentCluster();
             }
             lastDiscoveredPings = activeNodes;
-            List<PingResponse> responseList = activeNodes.stream()
+            PingCollection pingCollection = new PingCollection();
+            activeNodes.stream()
                 .filter(p -> p != this) // remove this as pings are not expected to return the local node
                 .map(MockZenPing::getPingResponse)
-                .collect(Collectors.toList());
-            listener.onPing(responseList);
+                .forEach(pingCollection::addPing);
+            resultsConsumer.accept(pingCollection);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
index 3ca66c11bf7..c2a237ef337 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import java.util.function.Supplier;
 
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
@@ -54,9 +55,10 @@ public class TestZenDiscovery extends ZenDiscovery {
         }
         @Override
         public Map<String, Supplier<Discovery>> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService,
+                                                                  NamedWriteableRegistry namedWriteableRegistry,
                                                                   ClusterService clusterService, UnicastHostsProvider hostsProvider) {
             return Collections.singletonMap("test-zen",
-                () -> new TestZenDiscovery(settings, threadPool, transportService, clusterService, hostsProvider));
+                () -> new TestZenDiscovery(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider));
         }
 
         @Override
@@ -71,8 +73,9 @@ public class TestZenDiscovery extends ZenDiscovery {
     }
 
     private TestZenDiscovery(Settings settings, ThreadPool threadPool, TransportService transportService,
-                             ClusterService clusterService, UnicastHostsProvider hostsProvider) {
-        super(settings, threadPool, transportService, clusterService, hostsProvider);
+                             NamedWriteableRegistry namedWriteableRegistry, ClusterService clusterService,
+                             UnicastHostsProvider hostsProvider) {
+        super(settings, threadPool, transportService, namedWriteableRegistry, clusterService, hostsProvider);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
index 956088f0fd1..8a6be290502 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.disruption;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
@@ -58,21 +58,16 @@ public class BlockClusterStateProcessing extends SingleNodeDisruption {
         boolean success = disruptionLatch.compareAndSet(null, new CountDownLatch(1));
         assert success : "startDisrupting called without waiting on stopDisrupting to complete";
         final CountDownLatch started = new CountDownLatch(1);
-        clusterService.submitStateUpdateTask("service_disruption_block", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("service_disruption_block", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 started.countDown();
                 CountDownLatch latch = disruptionLatch.get();
                 if (latch != null) {
                     latch.await();
                 }
-                return currentState;
+                return unchanged();
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
index f69c0a3085d..61afa4f77f3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.disruption;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.LocalClusterUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.unit.TimeValue;
@@ -102,15 +102,10 @@ public class SlowClusterStateProcessing extends SingleNodeDisruption {
             return false;
         }
         final AtomicBoolean stopped = new AtomicBoolean(false);
-        clusterService.submitStateUpdateTask("service_disruption_delay", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("service_disruption_delay", new LocalClusterUpdateTask(Priority.IMMEDIATE) {
 
             @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
+            public ClusterTasksResult<LocalClusterUpdateTask> execute(ClusterState currentState) throws Exception {
                 long count = duration.millis() / 200;
                 // wait while checking for a stopped
                 for (; count > 0 && !stopped.get(); count--) {
@@ -120,7 +115,7 @@ public class SlowClusterStateProcessing extends SingleNodeDisruption {
                     Thread.sleep(duration.millis() % 200);
                 }
                 countDownLatch.countDown();
-                return currentState;
+                return unchanged();
             }
 
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
index 37ed43b9450..9da5d7515c7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
@@ -147,7 +147,7 @@ public class ThrowingLeafReaderWrapper extends FilterLeafReader {
     static class ThrowingTermsEnum extends FilterTermsEnum {
         private final Thrower thrower;
 
-        public ThrowingTermsEnum(TermsEnum in, Thrower thrower) {
+        ThrowingTermsEnum(TermsEnum in, Thrower thrower) {
             super(in);
             this.thrower = thrower;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index bc560c9b0f0..6280de426ce 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -55,6 +55,11 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchModule;
@@ -73,8 +78,10 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.emptyList;
@@ -766,4 +773,76 @@ public class ElasticsearchAssertions {
         assertFileExists(dir);
         assertThat("file [" + dir + "] should be a directory.", Files.isDirectory(dir), is(true));
     }
+
+    /**
+     * Asserts that the provided {@link BytesReference}s created through
+     * {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} hold the same content.
+     * The comparison is done by parsing both into a map and comparing those two, so that keys ordering doesn't matter.
+     * Also binary values (byte[]) are properly compared through arrays comparisons.
+     */
+    public static void assertToXContentEquivalent(BytesReference expected, BytesReference actual, XContentType xContentType)
+            throws IOException {
+        //we tried comparing byte per byte, but that didn't fly for a couple of reasons:
+        //1) whenever anything goes through a map while parsing, ordering is not preserved, which is perfectly ok
+        //2) Jackson SMILE parser parses floats as double, which then get printed out as double (with double precision)
+        //Note that byte[] holding binary values need special treatment as they need to be properly compared item per item.
+        try (XContentParser actualParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, actual)) {
+            Map<String, Object> actualMap = actualParser.map();
+            try (XContentParser expectedParser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, expected)) {
+                Map<String, Object> expectedMap = expectedParser.map();
+                assertMapEquals(expectedMap, actualMap);
+            }
+        }
+    }
+
+    /**
+     * Compares two maps recursively, using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertMapEquals(Map<String, Object> expected, Map<String, Object> actual) {
+        assertEquals(expected.size(), actual.size());
+        for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+            String expectedKey = expectedEntry.getKey();
+            Object expectedValue = expectedEntry.getValue();
+            if (expectedValue == null) {
+                assertTrue(actual.get(expectedKey) == null && actual.containsKey(expectedKey));
+            } else {
+                Object actualValue = actual.get(expectedKey);
+                assertObjectEquals(expectedValue, actualValue);
+            }
+        }
+    }
+
+    /**
+     * Compares two lists recursively, but using arrays comparisons for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertListEquals(List<Object> expected, List<Object> actual) {
+        assertEquals(expected.size(), actual.size());
+        Iterator<Object> actualIterator = actual.iterator();
+        for (Object expectedValue : expected) {
+            Object actualValue = actualIterator.next();
+            assertObjectEquals(expectedValue, actualValue);
+        }
+    }
+
+    /**
+     * Compares two objects, recursively walking eventual maps and lists encountered, and using arrays comparisons
+     * for byte[] through Arrays.equals(byte[], byte[])
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertObjectEquals(Object expected, Object actual) {
+        if (expected instanceof Map) {
+            assertThat(actual, instanceOf(Map.class));
+            assertMapEquals((Map<String, Object>) expected, (Map<String, Object>) actual);
+        } else if (expected instanceof List) {
+            assertListEquals((List<Object>) expected, (List<Object>) actual);
+        } else if (expected instanceof byte[]) {
+            //byte[] is really a special case for binary values when comparing SMILE and CBOR, arrays of other types
+            //don't need to be handled. Ordinary arrays get parsed as lists.
+            assertArrayEquals((byte[]) expected, (byte[]) actual);
+        } else {
+            assertEquals(expected, actual);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
index 0009c21d6aa..e021df52c60 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 package org.elasticsearch.test.junit.listeners;
 
 import org.apache.logging.log4j.Logger;
@@ -26,18 +27,18 @@ import org.junit.runner.Description;
 import org.junit.runner.Result;
 import org.junit.runner.notification.RunListener;
 
-import java.util.HashMap;
+import java.util.Collections;
 import java.util.Map;
+import java.util.TreeMap;
 
 /**
- * A {@link RunListener} that allows to change the log level for a specific test method.
- * When a test method is annotated with the {@link org.elasticsearch.test.junit.annotations.TestLogging} annotation, the level for the specified loggers
- * will be internally saved before the test method execution and overridden with the specified ones.
- * At the end of the test method execution the original loggers levels will be restored.
+ * A {@link RunListener} that allows changing the log level for a specific test method. When a test method is annotated with the
+ * {@link TestLogging} annotation, the level for the specified loggers will be internally saved before the test method execution and
+ * overridden with the specified ones. At the end of the test method execution the original loggers levels will be restored.
  *
- * Note: This class is not thread-safe. Given the static nature of the logging api, it assumes that tests
- * are never run concurrently in the same jvm. For the very same reason no synchronization has been implemented
- * regarding the save/restore process of the original loggers levels.
+ * This class is not thread-safe. Given the static nature of the logging API, it assumes that tests are never run concurrently in the same
+ * JVM. For the very same reason no synchronization has been implemented regarding the save/restore process of the original loggers
+ * levels.
  */
 public class LoggingListener extends RunListener {
 
@@ -46,29 +47,35 @@ public class LoggingListener extends RunListener {
     private Map<String, String> previousPackageLoggingMap;
 
     @Override
-    public void testRunStarted(Description description) throws Exception {
+    public void testRunStarted(final Description description) throws Exception {
         Package testClassPackage = description.getTestClass().getPackage();
         previousPackageLoggingMap = processTestLogging(testClassPackage != null ? testClassPackage.getAnnotation(TestLogging.class) : null);
         previousClassLoggingMap = processTestLogging(description.getAnnotation(TestLogging.class));
     }
 
     @Override
-    public void testRunFinished(Result result) throws Exception {
+    public void testRunFinished(final Result result) throws Exception {
         previousClassLoggingMap = reset(previousClassLoggingMap);
         previousPackageLoggingMap = reset(previousPackageLoggingMap);
     }
 
     @Override
-    public void testStarted(Description description) throws Exception {
+    public void testStarted(final Description description) throws Exception {
         final TestLogging testLogging = description.getAnnotation(TestLogging.class);
         previousLoggingMap = processTestLogging(testLogging);
     }
 
     @Override
-    public void testFinished(Description description) throws Exception {
+    public void testFinished(final Description description) throws Exception {
         previousLoggingMap = reset(previousLoggingMap);
     }
 
+    /**
+     * Obtain the logger with the given name.
+     *
+     * @param loggerName the logger to obtain
+     * @return the logger
+     */
     private static Logger resolveLogger(String loggerName) {
         if (loggerName.equalsIgnoreCase("_root")) {
             return ESLoggerFactory.getRootLogger();
@@ -76,44 +83,71 @@ public class LoggingListener extends RunListener {
         return Loggers.getLogger(loggerName);
     }
 
-    private Map<String, String> processTestLogging(TestLogging testLogging) {
-        Map<String, String> map = getLoggersAndLevelsFromAnnotation(testLogging);
+    /**
+     * Applies the test logging annotation and returns the existing logging levels.
+     *
+     * @param testLogging the test logging annotation to apply
+     * @return the existing logging levels
+     */
+    private Map<String, String> processTestLogging(final TestLogging testLogging) {
+        final Map<String, String> map = getLoggersAndLevelsFromAnnotation(testLogging);
+
         if (map == null) {
-            return null;
+            return Collections.emptyMap();
         }
-        Map<String, String> previousValues = new HashMap<>();
-        for (Map.Entry<String, String> entry : map.entrySet()) {
-            Logger logger = resolveLogger(entry.getKey());
-            previousValues.put(entry.getKey(), logger.getLevel().toString());
+
+        // obtain the existing logging levels so that we can restore them at the end of the test; we have to do this separately from setting
+        // the logging levels so that setting foo does not impact the logging level for foo.bar when we check the existing logging level for
+        // for.bar
+        final Map<String, String> existing = new TreeMap<>();
+        for (final Map.Entry<String, String> entry : map.entrySet()) {
+            final Logger logger = resolveLogger(entry.getKey());
+            existing.put(entry.getKey(), logger.getLevel().toString());
+        }
+        for (final Map.Entry<String, String> entry : map.entrySet()) {
+            final Logger logger = resolveLogger(entry.getKey());
             Loggers.setLevel(logger, entry.getValue());
         }
-        return previousValues;
+        return existing;
     }
 
-    public static Map<String, String> getLoggersAndLevelsFromAnnotation(TestLogging testLogging) {
+    /**
+     * Obtain the logging levels from the test logging annotation.
+     *
+     * @param testLogging the test logging annotation
+     * @return a map from logger name to logging level
+     */
+    private static Map<String, String> getLoggersAndLevelsFromAnnotation(final TestLogging testLogging) {
         if (testLogging == null) {
-            return null;
+            return Collections.emptyMap();
         }
-        Map<String, String> map = new HashMap<>();
+        // use a sorted set so that we apply a parent logger before its children thus not overwriting the child setting when processing the
+        // parent setting
+        final Map<String, String> map = new TreeMap<>();
         final String[] loggersAndLevels = testLogging.value().split(",");
-        for (String loggerAndLevel : loggersAndLevels) {
-            String[] loggerAndLevelArray = loggerAndLevel.split(":");
-            if (loggerAndLevelArray.length >=2) {
-                String loggerName = loggerAndLevelArray[0];
-                String level = loggerAndLevelArray[1];
-                map.put(loggerName, level);
+        for (final String loggerAndLevel : loggersAndLevels) {
+            final String[] loggerAndLevelArray = loggerAndLevel.split(":");
+            if (loggerAndLevelArray.length == 2) {
+                map.put(loggerAndLevelArray[0], loggerAndLevelArray[1]);
+            } else {
+                throw new IllegalArgumentException("invalid test logging annotation [" + loggerAndLevel + "]");
             }
         }
         return map;
     }
 
-    private Map<String, String> reset(Map<String, String> map) {
-        if (map != null) {
-            for (Map.Entry<String, String> previousLogger : map.entrySet()) {
-                Logger logger = resolveLogger(previousLogger.getKey());
-                Loggers.setLevel(logger, previousLogger.getValue());
-            }
+    /**
+     * Reset the logging levels to the state provided by the map.
+     *
+     * @param map the logging levels to apply
+     * @return an empty map
+     */
+    private Map<String, String> reset(final Map<String, String> map) {
+        for (final Map.Entry<String, String> previousLogger : map.entrySet()) {
+            final Logger logger = resolveLogger(previousLogger.getKey());
+            Loggers.setLevel(logger, previousLogger.getValue());
         }
-        return null;
+
+        return Collections.emptyMap();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 0fc8cb4506b..1fc4cec9f80 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -111,8 +111,8 @@ public abstract class ESRestTestCase extends ESTestCase {
             }
             clusterHosts = unmodifiableList(hosts);
             logger.info("initializing REST clients against {}", clusterHosts);
-            client = buildClient(restClientSettings());
-            adminClient = buildClient(restAdminSettings());
+            client = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
+            adminClient = buildClient(restAdminSettings(), clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
         }
         assert client != null;
         assert adminClient != null;
@@ -174,6 +174,13 @@ public abstract class ESRestTestCase extends ESTestCase {
         return false;
     }
 
+    /**
+     * Returns whether to preserve the repositories on completion of this test.
+     */
+    protected boolean preserveReposUponCompletion() {
+        return false;
+    }
+
     private void wipeCluster() throws IOException {
         if (preserveIndicesUponCompletion() == false) {
             // wipe indices
@@ -217,8 +224,10 @@ public abstract class ESRestTestCase extends ESTestCase {
                     adminClient().performRequest("DELETE", "_snapshot/" + repoName + "/" + name);
                 }
             }
-            logger.debug("wiping snapshot repository [{}]", repoName);
-            adminClient().performRequest("DELETE", "_snapshot/" + repoName);
+            if (preserveReposUponCompletion() == false) {
+                logger.debug("wiping snapshot repository [{}]", repoName);
+                adminClient().performRequest("DELETE", "_snapshot/" + repoName);
+            }
         }
     }
 
@@ -272,8 +281,8 @@ public abstract class ESRestTestCase extends ESTestCase {
         return "http";
     }
 
-    private RestClient buildClient(Settings settings) throws IOException {
-        RestClientBuilder builder = RestClient.builder(clusterHosts.toArray(new HttpHost[clusterHosts.size()]));
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
         String keystorePath = settings.get(TRUSTSTORE_PATH);
         if (keystorePath != null) {
             final String keystorePass = settings.get(TRUSTSTORE_PASSWORD);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
index 3d1ce291432..bca4af7bd60 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestChannel.java
@@ -19,9 +19,9 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -53,8 +53,8 @@ public final class FakeRestChannel extends AbstractRestChannel {
     }
 
     @Override
-    public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
-        return super.newBuilder(autoDetectSource, useFiltering);
+    public XContentBuilder newBuilder(@Nullable XContentType requestContentType, boolean useFiltering) throws IOException {
+        return super.newBuilder(requestContentType, useFiltering);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
index f291e0bbeec..83caf0293e0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
@@ -20,27 +20,32 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.rest.RestRequest;
 
+import java.net.SocketAddress;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class FakeRestRequest extends RestRequest {
 
-    private final Map<String, String> headers;
     private final BytesReference content;
     private final Method method;
-
+    private final SocketAddress remoteAddress;
 
     public FakeRestRequest() {
-        this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/");
+        this(NamedXContentRegistry.EMPTY, new HashMap<>(), new HashMap<>(), null, Method.GET, "/", null);
     }
 
-    private FakeRestRequest(Map<String, String> headers, Map<String, String> params, BytesReference content, Method method, String path) {
-        super(params, path);
-        this.headers = headers;
+    private FakeRestRequest(NamedXContentRegistry xContentRegistry, Map<String, List<String>> headers, Map<String, String> params,
+                            BytesReference content, Method method, String path, SocketAddress remoteAddress) {
+        super(xContentRegistry, params, path, headers);
         this.content = content;
         this.method = method;
+        this.remoteAddress = remoteAddress;
     }
 
     @Override
@@ -64,18 +69,14 @@ public class FakeRestRequest extends RestRequest {
     }
 
     @Override
-    public String header(String name) {
-        return headers.get(name);
-    }
-
-    @Override
-    public Iterable<Map.Entry<String, String>> headers() {
-        return headers.entrySet();
+    public SocketAddress getRemoteAddress() {
+        return remoteAddress;
     }
 
     public static class Builder {
+        private final NamedXContentRegistry xContentRegistry;
 
-        private Map<String, String> headers = new HashMap<>();
+        private Map<String, List<String>> headers = new HashMap<>();
 
         private Map<String, String> params = new HashMap<>();
 
@@ -85,7 +86,13 @@ public class FakeRestRequest extends RestRequest {
 
         private Method method = Method.GET;
 
-        public Builder withHeaders(Map<String, String> headers) {
+        private SocketAddress address = null;
+
+        public Builder(NamedXContentRegistry xContentRegistry) {
+            this.xContentRegistry = xContentRegistry;
+        }
+
+        public Builder withHeaders(Map<String, List<String>> headers) {
             this.headers = headers;
             return this;
         }
@@ -95,8 +102,11 @@ public class FakeRestRequest extends RestRequest {
             return this;
         }
 
-        public Builder withContent(BytesReference content) {
+        public Builder withContent(BytesReference content, XContentType xContentType) {
             this.content = content;
+            if (xContentType != null) {
+                headers.put("Content-Type", Collections.singletonList(xContentType.mediaType()));
+            }
             return this;
         }
 
@@ -110,8 +120,13 @@ public class FakeRestRequest extends RestRequest {
             return this;
         }
 
+        public Builder withRemoteAddress(SocketAddress address) {
+            this.address = address;
+            return this;
+        }
+
         public FakeRestRequest build() {
-            return new FakeRestRequest(headers, params, content, method, path);
+            return new FakeRestRequest(xContentRegistry, headers, params, content, method, path, address);
         }
 
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
index 7b216a148be..aa51efaa333 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java
@@ -32,7 +32,6 @@ import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestPath;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
@@ -40,13 +39,10 @@ import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 /**
  * Used by {@link ESClientYamlSuiteTestCase} to execute REST requests according to the tests written in yaml suite files. Wraps a
@@ -55,22 +51,19 @@ import java.util.Set;
  */
 public class ClientYamlTestClient {
     private static final Logger logger = Loggers.getLogger(ClientYamlTestClient.class);
-    /**
-     * Query params that don't need to be declared in the spec, they are supported by default.
-     */
-    private static final Set<String> ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet(
-            "error_trace", "filter_path", "pretty", "source");
 
     private final ClientYamlSuiteRestSpec restSpec;
     private final RestClient restClient;
     private final Version esVersion;
+    private final Map<HttpHost, Version> hostVersionMap;
 
     public ClientYamlTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List<HttpHost> hosts,
-                                Version esVersion) throws IOException {
+                                Version esVersion, Map<HttpHost, Version> hostVersionMap) throws IOException {
         assert hosts.size() > 0;
         this.restSpec = restSpec;
         this.restClient = restClient;
         this.esVersion = esVersion;
+        this.hostVersionMap = hostVersionMap;
     }
 
     public Version getEsVersion() {
@@ -95,28 +88,9 @@ public class ClientYamlTestClient {
             // And everything else is a url parameter!
             try {
                 Response response = restClient.performRequest(method, path, queryStringParams, entity);
-                return new ClientYamlTestResponse(response);
+                return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost()));
             } catch(ResponseException e) {
-                throw new ClientYamlTestResponseException(e);
-            }
-        }
-
-        List<Integer> ignores = new ArrayList<>();
-        Map<String, String> requestParams;
-        if (params == null) {
-            requestParams = Collections.emptyMap();
-        } else {
-            requestParams = new HashMap<>(params);
-            if (params.isEmpty() == false) {
-                //ignore is a special parameter supported by the clients, shouldn't be sent to es
-                String ignoreString = requestParams.remove("ignore");
-                if (ignoreString != null) {
-                    try {
-                        ignores.add(Integer.valueOf(ignoreString));
-                    } catch (NumberFormatException e) {
-                        throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead");
-                    }
-                }
+                throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost()));
             }
         }
 
@@ -125,11 +99,12 @@ public class ClientYamlTestClient {
         //divide params between ones that go within query string and ones that go within path
         Map<String, String> pathParts = new HashMap<>();
         Map<String, String> queryStringParams = new HashMap<>();
-        for (Map.Entry<String, String> entry : requestParams.entrySet()) {
+        for (Map.Entry<String, String> entry : params.entrySet()) {
             if (restApi.getPathParts().contains(entry.getKey())) {
                 pathParts.put(entry.getKey(), entry.getValue());
             } else {
-                if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
+                if (restApi.getParams().contains(entry.getKey()) || restSpec.isGlobalParameter(entry.getKey())
+                        || restSpec.isClientParameter(entry.getKey())) {
                     queryStringParams.put(entry.getKey(), entry.getValue());
                 } else {
                     throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in ["
@@ -149,6 +124,9 @@ public class ClientYamlTestClient {
             if (supportedMethods.contains("GET") && RandomizedTest.rarely()) {
                 logger.debug("sending the request body as source param with GET method");
                 queryStringParams.put("source", body);
+                if (esVersion.after(Version.V_5_3_0_UNRELEASED)) { // TODO make onOrAfter with backport
+                    queryStringParams.put("source_content_type", ContentType.APPLICATION_JSON.toString());
+                }
                 requestMethod = "GET";
             } else {
                 requestMethod = RandomizedTest.randomFrom(supportedMethods);
@@ -195,12 +173,9 @@ public class ClientYamlTestClient {
         logger.debug("calling api [{}]", apiName);
         try {
             Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, requestBody, requestHeaders);
-            return new ClientYamlTestResponse(response);
+            return new ClientYamlTestResponse(response, hostVersionMap.get(response.getHost()));
         } catch(ResponseException e) {
-            if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
-                return new ClientYamlTestResponse(e.getResponse());
-            }
-            throw new ClientYamlTestResponseException(e);
+            throw new ClientYamlTestResponseException(e, hostVersionMap.get(e.getResponse().getHost()));
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
index 8215105887d..27653a13f29 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java
@@ -71,9 +71,9 @@ public class ClientYamlTestExecutionContext {
             throw e;
         } finally {
             // if we hit a bad exception the response is null
-            Object repsponseBody = response != null ? response.getBody() : null;
+            Object responseBody = response != null ? response.getBody() : null;
             //we always stash the last response body
-            stash.stashValue("body", repsponseBody);
+            stash.stashValue("body", responseBody);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
index 481ae752d05..f47cfd68d4e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponse.java
@@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.yaml;
 import org.apache.http.Header;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.xcontent.XContentType;
 
@@ -37,10 +38,12 @@ public class ClientYamlTestResponse {
 
     private final Response response;
     private final String body;
+    private final Version nodeVersion;
     private ObjectPath parsedResponse;
 
-    ClientYamlTestResponse(Response response) throws IOException {
+    ClientYamlTestResponse(Response response, Version version) throws IOException {
         this.response = response;
+        this.nodeVersion = version;
         if (response.getEntity() != null) {
             try {
                 this.body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
@@ -80,7 +83,18 @@ public class ClientYamlTestResponse {
         List<String> warningHeaders = new ArrayList<>();
         for (Header header : response.getHeaders()) {
             if (header.getName().equals("Warning")) {
-                warningHeaders.add(header.getValue());
+                if (nodeVersion.after(Version.V_5_3_0_UNRELEASED) && response.getRequestLine().getMethod().equals("GET")
+                    && response.getRequestLine().getUri().contains("source")
+                    && response.getRequestLine().getUri().contains("source_content_type") == false && header.getValue().startsWith(
+                        "Deprecated use of the [source] parameter without the [source_content_type] parameter.")) {
+                    // this is because we do not send the source content type header when the node is 5.3.0 or below and the request
+                    // might have been sent to a node with a version > 5.3.0 when running backwards 5.0 tests. The Java RestClient
+                    // has control of the node the request is sent to so we can only detect this after the fact right now
+                    // TODO remove this when we bump versions
+                } else {
+                    warningHeaders.add(header.getValue());
+                }
+
             }
         }
         return warningHeaders;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
index 7d983d48029..48a7ee578d5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestResponseException.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test.rest.yaml;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.client.ResponseException;
 
 import java.io.IOException;
@@ -32,10 +33,10 @@ public class ClientYamlTestResponseException extends IOException {
     private final ClientYamlTestResponse restTestResponse;
     private final ResponseException responseException;
 
-    ClientYamlTestResponseException(ResponseException responseException) throws IOException {
+    ClientYamlTestResponseException(ResponseException responseException, Version version) throws IOException {
         super(responseException);
         this.responseException = responseException;
-        this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse());
+        this.restTestResponse = new ClientYamlTestResponse(responseException.getResponse(), version);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
index 24f977c2740..411f657329d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.test.rest.yaml;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
+
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
@@ -31,8 +32,6 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi;
 import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
 import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
@@ -54,6 +53,7 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -113,11 +113,6 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         if (restTestExecutionContext == null) {
             assert adminExecutionContext == null;
             assert blacklistPathMatchers == null;
-            String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
-            blacklistPathMatchers = new ArrayList<>();
-            for (String entry : blacklist) {
-                blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry));
-            }
             String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH);
             ClientYamlSuiteRestSpec restSpec = null;
             FileSystem fileSystem = getFileSystem();
@@ -131,6 +126,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
             validateSpec(restSpec);
             List<HttpHost> hosts = getClusterHosts();
             RestClient restClient = client();
+            Tuple<Version, Map<HttpHost, Version>> versionMapTuple = readVersionsFromInfo(restClient, hosts.size());
             Version esVersion;
             try {
                 Tuple<Version, Version> versionVersionTuple = readVersionsFromCatNodes(restClient);
@@ -141,15 +137,21 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
             } catch (ResponseException ex) {
                 if (ex.getResponse().getStatusLine().getStatusCode() == 403) {
                     logger.warn("Fallback to simple info '/' request, _cat/nodes is not authorized");
-                    esVersion = readVersionsFromInfo(restClient, hosts.size());
+                    esVersion = versionMapTuple.v1();
                     logger.info("initializing yaml client, minimum es version: [{}] hosts: {}", esVersion, hosts);
                 } else {
                     throw ex;
                 }
             }
-            ClientYamlTestClient clientYamlTestClient = new ClientYamlTestClient(restSpec, restClient, hosts, esVersion);
+            ClientYamlTestClient clientYamlTestClient =
+                new ClientYamlTestClient(restSpec, restClient, hosts, esVersion, versionMapTuple.v2());
             restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient);
             adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient);
+            String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null);
+            blacklistPathMatchers = new ArrayList<>();
+            for (String entry : blacklist) {
+                blacklistPathMatchers.add(new BlacklistedPathPatternMatcher(entry));
+            }
         }
         assert restTestExecutionContext != null;
         assert adminExecutionContext != null;
@@ -182,11 +184,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
     protected void afterIfFailed(List<Throwable> errors) {
         // Dump the stash on failure. Instead of dumping it in true json we escape `\n`s so stack traces are easier to read
         logger.info("Stash dump on failure [{}]",
-                XContentHelper.toString(restTestExecutionContext.stash()).replace("\\n", "\n").replace("\\t", "\t"));
+                XContentHelper.toString(restTestExecutionContext.stash()).replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t"));
         super.afterIfFailed(errors);
     }
 
-    public static Iterable<Object[]> createParameters() throws IOException, ClientYamlTestParseException {
+    public static Iterable<Object[]> createParameters() throws IOException {
         List<ClientYamlTestCandidate> restTestCandidates = collectTestCandidates();
         List<Object[]> objects = new ArrayList<>();
         for (ClientYamlTestCandidate restTestCandidate : restTestCandidates) {
@@ -195,7 +197,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         return objects;
     }
 
-    private static List<ClientYamlTestCandidate> collectTestCandidates() throws ClientYamlTestParseException, IOException {
+    private static List<ClientYamlTestCandidate> collectTestCandidates() throws IOException {
         List<ClientYamlTestCandidate> testCandidates = new ArrayList<>();
         FileSystem fileSystem = getFileSystem();
         // don't make a try-with, getFileSystem returns null
@@ -203,12 +205,11 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         try {
             String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH);
             Map<String, Set<Path>> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths);
-            ClientYamlTestSuiteParser restTestSuiteParser = new ClientYamlTestSuiteParser();
             //yaml suites are grouped by directory (effectively by api)
             for (String api : yamlSuites.keySet()) {
                 List<Path> yamlFiles = new ArrayList<>(yamlSuites.get(api));
                 for (Path yamlFile : yamlFiles) {
-                    ClientYamlTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile);
+                    ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile);
                     for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) {
                         testCandidates.add(new ClientYamlTestCandidate(restTestSuite, testSection));
                     }
@@ -292,7 +293,7 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
     private static Tuple<Version, Version> readVersionsFromCatNodes(RestClient restClient) throws IOException {
         // we simply go to the _cat/nodes API and parse all versions in the cluster
         Response response = restClient.performRequest("GET", "/_cat/nodes", Collections.singletonMap("h", "version,master"));
-        ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response);
+        ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT);
         String nodesCatResponse = restTestResponse.getBodyAsString();
         String[] split = nodesCatResponse.split("\n");
         Version version = null;
@@ -315,12 +316,13 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
         return new Tuple<>(version, masterVersion);
     }
 
-    private static Version readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException {
+    private static Tuple<Version, Map<HttpHost, Version>> readVersionsFromInfo(RestClient restClient, int numHosts) throws IOException {
         Version version = null;
+        Map<HttpHost, Version> hostVersionMap = new HashMap<>();
         for (int i = 0; i < numHosts; i++) {
             //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
             Response response = restClient.performRequest("GET", "/");
-            ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response);
+            ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response, Version.CURRENT);
             Object latestVersion = restTestResponse.evaluate("version.number");
             if (latestVersion == null) {
                 throw new RuntimeException("elasticsearch version not found in the response");
@@ -331,8 +333,9 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
             } else if (version.onOrAfter(currentVersion)) {
                 version = currentVersion;
             }
+            hostVersionMap.put(response.getHost(), currentVersion);
         }
-        return version;
+        return new Tuple<>(version, Collections.unmodifiableMap(hostVersionMap));
     }
 
     public void test() throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
index 6311944fdcb..e2a41365032 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ObjectPath.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test.rest.yaml;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 
@@ -34,7 +35,7 @@ public class ObjectPath {
     private final Object object;
 
     public static ObjectPath createFromXContent(XContent xContent, String input) throws IOException {
-        try (XContentParser parser = xContent.createParser(input)) {
+        try (XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, input)) {
             if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                 return new ObjectPath(parser.listOrderedMap());
             }
@@ -46,17 +47,28 @@ public class ObjectPath {
         this.object = object;
     }
 
+
+    /**
+     * A utility method that creates an {@link ObjectPath} via {@link #ObjectPath(Object)} returns
+     * the result of calling {@link #evaluate(String)} on it.
+     */
+    public static <T> T evaluate(Object object, String path) throws IOException {
+        return new ObjectPath(object).evaluate(path, Stash.EMPTY);
+    }
+
+
     /**
      * Returns the object corresponding to the provided path if present, null otherwise
      */
-    public Object evaluate(String path) throws IOException {
+    public <T> T evaluate(String path) throws IOException {
         return evaluate(path, Stash.EMPTY);
     }
 
     /**
      * Returns the object corresponding to the provided path if present, null otherwise
      */
-    public Object evaluate(String path, Stash stash) throws IOException {
+    @SuppressWarnings("unchecked")
+    public <T> T evaluate(String path, Stash stash) throws IOException {
         String[] parts = parsePath(path);
         Object object = this.object;
         for (String part : parts) {
@@ -65,7 +77,7 @@ public class ObjectPath {
                 return null;
             }
         }
-        return object;
+        return (T)object;
     }
 
     @SuppressWarnings("unchecked")
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
deleted file mode 100644
index b6b6adfd037..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-
-/**
- * Parser for a complete test section
- */
-public class ClientYamlTestSectionParser implements ClientYamlTestFragmentParser<ClientYamlTestSection> {
-
-    @Override
-    public ClientYamlTestSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-        parseContext.advanceToFieldName();
-        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.currentName());
-        try {
-            parser.nextToken();
-            testSection.setSkipSection(parseContext.parseSkipSection());
-
-            while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
-                parseContext.advanceToFieldName();
-                testSection.addExecutableSection(parseContext.parseExecutableSection());
-            }
-
-            parser.nextToken();
-            assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected "
-                + XContentParser.Token.END_OBJECT  + " but was " + parser.currentToken();
-            parser.nextToken();
-
-            return testSection;
-        } catch (Exception e) {
-            throw new ClientYamlTestParseException("Error parsing test named [" + testSection.getName() + "]", e);
-        }
-    }
-
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
deleted file mode 100644
index e466c1010f6..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Context shared across the whole tests parse phase.
- * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
- */
-public class ClientYamlTestSuiteParseContext implements ParseFieldMatcherSupplier {
-
-    private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser();
-    private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser();
-    private static final ClientYamlTestSectionParser TEST_SECTION_PARSER = new ClientYamlTestSectionParser();
-    private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser();
-    private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser();
-    private static final Map<String, ClientYamlTestFragmentParser<? extends ExecutableSection>> EXECUTABLE_SECTIONS_PARSERS =
-            new HashMap<>();
-    static {
-        EXECUTABLE_SECTIONS_PARSERS.put("do", DO_SECTION_PARSER);
-        EXECUTABLE_SECTIONS_PARSERS.put("set", new SetSectionParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("match", new MatchParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_true", new IsTrueParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("is_false", new IsFalseParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gt", new GreaterThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("gte", new GreaterThanEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
-    }
-
-    private final String api;
-    private final String suiteName;
-    private final XContentParser parser;
-
-    public ClientYamlTestSuiteParseContext(String api, String suiteName, XContentParser parser) {
-        this.api = api;
-        this.suiteName = suiteName;
-        this.parser = parser;
-    }
-
-    public String getApi() {
-        return api;
-    }
-
-    public String getSuiteName() {
-        return suiteName;
-    }
-
-    public XContentParser parser() {
-        return parser;
-    }
-
-    public SetupSection parseSetupSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("setup".equals(parser.currentName())) {
-            parser.nextToken();
-            SetupSection setupSection = SETUP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return setupSection;
-        }
-
-        return SetupSection.EMPTY;
-    }
-
-    public TeardownSection parseTeardownSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-
-        if ("teardown".equals(parser.currentName())) {
-            parser.nextToken();
-            TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return teardownSection;
-        }
-
-        return TeardownSection.EMPTY;
-    }
-
-    public ClientYamlTestSection parseTestSection() throws IOException, ClientYamlTestParseException {
-        return TEST_SECTION_PARSER.parse(this);
-    }
-
-    public SkipSection parseSkipSection() throws IOException, ClientYamlTestParseException {
-
-        advanceToFieldName();
-
-        if ("skip".equals(parser.currentName())) {
-            SkipSection skipSection = SKIP_SECTION_PARSER.parse(this);
-            parser.nextToken();
-            return skipSection;
-        }
-
-        return SkipSection.EMPTY;
-    }
-
-    public ExecutableSection parseExecutableSection() throws IOException, ClientYamlTestParseException {
-        advanceToFieldName();
-        String section = parser.currentName();
-        ClientYamlTestFragmentParser<? extends ExecutableSection> execSectionParser = EXECUTABLE_SECTIONS_PARSERS.get(section);
-        if (execSectionParser == null) {
-            throw new ClientYamlTestParseException("no parser found for executable section [" + section + "]");
-        }
-        XContentLocation location = parser.getTokenLocation();
-        try {
-            ExecutableSection executableSection = execSectionParser.parse(this);
-            parser.nextToken();
-            return executableSection;
-        } catch (Exception e) {
-            throw new IOException("Error parsing section starting at ["+ location + "]", e);
-        }
-    }
-
-    public DoSection parseDoSection() throws IOException, ClientYamlTestParseException {
-        return DO_SECTION_PARSER.parse(this);
-    }
-
-    public void advanceToFieldName() throws IOException, ClientYamlTestParseException {
-        XContentParser.Token token = parser.currentToken();
-        //we are in the beginning, haven't called nextToken yet
-        if (token == null) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_ARRAY) {
-            token = parser.nextToken();
-        }
-        if (token == XContentParser.Token.START_OBJECT) {
-            token = parser.nextToken();
-        }
-        if (token != XContentParser.Token.FIELD_NAME) {
-            throw new ClientYamlTestParseException("malformed test section: field name expected but found " + token + " at "
-                    + parser.getTokenLocation());
-        }
-    }
-
-    public String parseField() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        assert parser.currentToken().isValue();
-        String field = parser.text();
-        parser.nextToken();
-        return field;
-    }
-
-    public Tuple<String, Object> parseTuple() throws IOException, ClientYamlTestParseException {
-        parser.nextToken();
-        advanceToFieldName();
-        Map<String,Object> map = parser.map();
-        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
-        parser.nextToken();
-
-        if (map.size() != 1) {
-            throw new ClientYamlTestParseException("expected key value pair but found " + map.size() + " ");
-        }
-
-        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
-        return Tuple.tuple(entry.getKey(), entry.getValue());
-    }
-
-    @Override
-    public ParseFieldMatcher getParseFieldMatcher() {
-        return ParseFieldMatcher.STRICT;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
deleted file mode 100644
index 65277888471..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-
-/**
- * Parser for a complete test suite (yaml file)
- */
-public class ClientYamlTestSuiteParser implements ClientYamlTestFragmentParser<ClientYamlTestSuite> {
-
-    public ClientYamlTestSuite parse(String api, Path file) throws IOException, ClientYamlTestParseException {
-
-        if (!Files.isRegularFile(file)) {
-            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
-        }
-
-        String filename = file.getFileName().toString();
-        //remove the file extension
-        int i = filename.lastIndexOf('.');
-        if (i > 0) {
-            filename = filename.substring(0, i);
-        }
-
-        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
-        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
-            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
-            channel.read(bb, channel.size() - 1);
-            if (bb.get(0) != 10) {
-                throw new ClientYamlTestParseException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
-            }
-        }
-
-        try (XContentParser parser = YamlXContent.yamlXContent.createParser(Files.newInputStream(file))) {
-            ClientYamlTestSuiteParseContext testParseContext = new ClientYamlTestSuiteParseContext(api, filename, parser);
-            return parse(testParseContext);
-        } catch(Exception e) {
-            throw new ClientYamlTestParseException("Error parsing " + api + "/" + filename, e);
-        }
-    }
-
-    @Override
-    public ClientYamlTestSuite parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
-                + parser.currentToken();
-
-        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(parseContext.getApi(), parseContext.getSuiteName());
-
-        restTestSuite.setSetupSection(parseContext.parseSetupSection());
-        restTestSuite.setTeardownSection(parseContext.parseTeardownSection());
-
-        while(true) {
-            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
-            //we need to somehow distinguish between a null in the middle of a test ("---")
-            // and a null at the end of the file (at least two consecutive null tokens)
-            if(parser.currentToken() == null) {
-                if (parser.nextToken() == null) {
-                    break;
-                }
-            }
-
-            ClientYamlTestSection testSection = parseContext.parseTestSection();
-            if (!restTestSuite.addTestSection(testSection)) {
-                throw new ClientYamlTestParseException("duplicate test section [" + testSection.getName() + "] found in ["
-                        + restTestSuite.getPath() + "]");
-            }
-        }
-
-        return restTestSuite;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
deleted file mode 100644
index b89aa821ec9..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableList;
-
-/**
- * Parser for do sections
- */
-public class DoSectionParser implements ClientYamlTestFragmentParser<DoSection> {
-
-    @Override
-    public DoSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        DoSection doSection = new DoSection(parseContext.parser().getTokenLocation());
-        ApiCallSection apiCallSection = null;
-        Map<String, String> headers = new HashMap<>();
-        List<String> expectedWarnings = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("catch".equals(currentFieldName)) {
-                    doSection.setCatch(parser.text());
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("warnings".equals(currentFieldName)) {
-                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
-                        expectedWarnings.add(parser.text());
-                    }
-                    if (token != XContentParser.Token.END_ARRAY) {
-                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
-                    }
-                } else {
-                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
-                }
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                if ("headers".equals(currentFieldName)) {
-                    String headerName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            headerName = parser.currentName();
-                        } else if (token.isValue()) {
-                            headers.put(headerName, parser.text());
-                        }
-                    }
-                } else if (currentFieldName != null) { // must be part of API call then
-                    apiCallSection = new ApiCallSection(currentFieldName);
-                    String paramName = null;
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                        if (token == XContentParser.Token.FIELD_NAME) {
-                            paramName = parser.currentName();
-                        } else if (token.isValue()) {
-                            if ("body".equals(paramName)) {
-                                String body = parser.text();
-                                XContentType bodyContentType = XContentFactory.xContentType(body);
-                                XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser(body);
-                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
-                                while(bodyParser.nextToken() != null) {
-                                    apiCallSection.addBody(bodyParser.mapOrdered());
-                                }
-                            } else {
-                                apiCallSection.addParam(paramName, parser.text());
-                            }
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            if ("body".equals(paramName)) {
-                                apiCallSection.addBody(parser.mapOrdered());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try {
-            if (apiCallSection == null) {
-                throw new ClientYamlTestParseException("client call section is mandatory within a do section");
-            }
-            if (headers.isEmpty() == false) {
-                apiCallSection.addHeaders(headers);
-            }
-            doSection.setApiCallSection(apiCallSection);
-            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
-        } finally {
-            parser.nextToken();
-        }
-        return doSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
deleted file mode 100644
index 65a46d139bd..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gte assert sections
- */
-public class GreaterThanEqualToParser implements ClientYamlTestFragmentParser<GreaterThanEqualToAssertion> {
-    @Override
-    public GreaterThanEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext)
-            throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
deleted file mode 100644
index f6c53272067..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for gt assert sections
- */
-public class GreaterThanParser implements ClientYamlTestFragmentParser<GreaterThanAssertion> {
-
-    @Override
-    public GreaterThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("gt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
deleted file mode 100644
index 98c030cd6a1..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for length assert sections
- */
-public class LengthParser implements ClientYamlTestFragmentParser<LengthAssertion> {
-
-    @Override
-    public LengthAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        assert stringObjectTuple.v2() != null;
-        int value;
-        if (stringObjectTuple.v2() instanceof  Number) {
-            value = ((Number) stringObjectTuple.v2()).intValue();
-        } else {
-            try {
-                value = Integer.valueOf(stringObjectTuple.v2().toString());
-            } catch(NumberFormatException e) {
-                throw new ClientYamlTestParseException("length is not a valid number", e);
-            }
-
-        }
-        return new LengthAssertion(location, stringObjectTuple.v1(), value);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
deleted file mode 100644
index 5dfe4c742f5..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanOrEqualToAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lte assert section
- */
-public class LessThanOrEqualToParser implements ClientYamlTestFragmentParser<LessThanOrEqualToAssertion> {
-
-    @Override
-    public LessThanOrEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lte section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
deleted file mode 100644
index 0caf12ea91a..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-
-import java.io.IOException;
-
-/**
- * Parser for lt assert sections
- */
-public class LessThanParser implements ClientYamlTestFragmentParser<LessThanAssertion> {
-
-    @Override
-    public LessThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentLocation location = parseContext.parser().getTokenLocation();
-        Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
-        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new ClientYamlTestParseException("lt section can only be used with objects that support natural ordering, found "
-                    + stringObjectTuple.v2().getClass().getSimpleName());
-        }
-        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
deleted file mode 100644
index c7797e42d57..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-
-import java.io.IOException;
-
-/**
- * Parser for set sections
- */
-public class SetSectionParser implements ClientYamlTestFragmentParser<SetSection> {
-
-    @Override
-    public SetSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-
-        SetSection setSection = new SetSection(parser.getTokenLocation());
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                setSection.addSet(currentFieldName, parser.text());
-            }
-        }
-
-        parser.nextToken();
-
-        if (setSection.getStash().isEmpty()) {
-            throw new ClientYamlTestParseException("set section must set at least a value");
-        }
-
-        return setSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
deleted file mode 100644
index e62b3af5251..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SetupSection;
-
-import java.io.IOException;
-
-/**
- * Parser for setup sections
- */
-public class SetupSectionParser implements ClientYamlTestFragmentParser<SetupSection> {
-
-    @Override
-    public SetupSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        SetupSection setupSection = new SetupSection();
-        setupSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within setup section");
-            }
-
-            parser.nextToken();
-            setupSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-
-        return setupSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
deleted file mode 100644
index b73edf7d2c6..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Parser for skip sections
- */
-public class SkipSectionParser implements ClientYamlTestFragmentParser<SkipSection> {
-
-    @Override
-    public SkipSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-
-        XContentParser parser = parseContext.parser();
-
-        String currentFieldName = null;
-        XContentParser.Token token;
-        String version = null;
-        String reason = null;
-        List<String> features = new ArrayList<>();
-
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token.isValue()) {
-                if ("version".equals(currentFieldName)) {
-                    version = parser.text();
-                } else if ("reason".equals(currentFieldName)) {
-                    reason = parser.text();
-                } else if ("features".equals(currentFieldName)) {
-                    features.add(parser.text());
-                }
-                else {
-                    throw new ClientYamlTestParseException("field " + currentFieldName + " not supported within skip section");
-                }
-            } else if (token == XContentParser.Token.START_ARRAY) {
-                if ("features".equals(currentFieldName)) {
-                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                        features.add(parser.text());
-                    }
-                }
-            }
-        }
-
-        parser.nextToken();
-
-        if (!Strings.hasLength(version) && features.isEmpty()) {
-            throw new ClientYamlTestParseException("version or features is mandatory within skip section");
-        }
-        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
-            throw new ClientYamlTestParseException("reason is mandatory within skip version section");
-        }
-
-        return new SkipSection(version, features, reason);
-
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
deleted file mode 100644
index ed1b42c3a9c..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
-
-import java.io.IOException;
-
-/**
- * Parser for teardown section
- */
-public class TeardownSectionParser implements ClientYamlTestFragmentParser<TeardownSection> {
-
-    @Override
-    public TeardownSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException {
-        XContentParser parser = parseContext.parser();
-
-        TeardownSection teardownSection = new TeardownSection();
-        teardownSection.setSkipSection(parseContext.parseSkipSection());
-
-        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            if (!"do".equals(parser.currentName())) {
-                throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within teardown section");
-            }
-
-            parser.nextToken();
-            teardownSection.addDoSection(parseContext.parseDoSection());
-            parser.nextToken();
-        }
-
-        parser.nextToken();
-        return teardownSection;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
index 9839bb089ae..15f2f7e3016 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.test.rest.yaml.restspec;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.rest.yaml.FileUtils;
@@ -29,18 +30,21 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Holds the specification used to turn {@code do} actions in the YAML suite into REST api calls.
  */
 public class ClientYamlSuiteRestSpec {
-    Map<String, ClientYamlSuiteRestApi> restApiMap = new HashMap<>();
+    private final Set<String> globalParameters = new HashSet<>();
+    private final Map<String, ClientYamlSuiteRestApi> restApiMap = new HashMap<>();
 
     private ClientYamlSuiteRestSpec() {
     }
 
-    void addApi(ClientYamlSuiteRestApi restApi) {
+    private void addApi(ClientYamlSuiteRestApi restApi) {
         ClientYamlSuiteRestApi previous = restApiMap.putIfAbsent(restApi.getName(), restApi);
         if (previous != null) {
             throw new IllegalArgumentException("cannot register api [" + restApi.getName() + "] found in [" + restApi.getLocation() + "]. "
@@ -56,6 +60,21 @@ public class ClientYamlSuiteRestSpec {
         return restApiMap.values();
     }
 
+    /**
+     * Returns whether the provided parameter is one of those parameters that are supported by all Elasticsearch api
+     */
+    public boolean isGlobalParameter(String param) {
+        return globalParameters.contains(param);
+    }
+
+    /**
+     * Returns whether the provided parameter is one of those parameters that are supported by the Elasticsearch language clients, meaning
+     * that they influence the client behaviour and don't get sent to Elasticsearch
+     */
+    public boolean isClientParameter(String name) {
+        return "ignore".equals(name);
+    }
+
     /**
      * Parses the complete set of REST spec available under the provided directories
      */
@@ -65,15 +84,39 @@ public class ClientYamlSuiteRestSpec {
         for (String path : paths) {
             for (Path jsonFile : FileUtils.findJsonSpec(fileSystem, optionalPathPrefix, path)) {
                 try (InputStream stream = Files.newInputStream(jsonFile)) {
-                    try (XContentParser parser = JsonXContent.jsonXContent.createParser(stream)) {
-                        ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
-                        String filename = jsonFile.getFileName().toString();
-                        String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
-                        if (restApi.getName().equals(expectedApiName) == false) {
-                            throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() + "]. " +
-                                    "Each api is expected to have the same name as the file that defines it.");
+                    String filename = jsonFile.getFileName().toString();
+                    try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, stream)) {
+                        if (filename.equals("_common.json")) {
+                            String currentFieldName = null;
+                            while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                                if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                                    currentFieldName = parser.currentName();
+                                } else if (parser.currentToken() == XContentParser.Token.START_OBJECT
+                                        && "params".equals(currentFieldName)) {
+                                    while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                        String param = parser.currentName();
+                                        if (restSpec.globalParameters.contains(param)) {
+                                            throw new IllegalArgumentException("Found duplicate global param [" + param + "]");
+                                        }
+                                        restSpec.globalParameters.add(param);
+                                        parser.nextToken();
+                                        if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                                            throw new IllegalArgumentException("Expected params field in rest api definition to " +
+                                                    "contain an object");
+                                        }
+                                        parser.skipChildren();
+                                    }
+                                }
+                            }
+                        } else {
+                            ClientYamlSuiteRestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
+                            String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
+                            if (restApi.getName().equals(expectedApiName) == false) {
+                                throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() +
+                                        "]. " + "Each api is expected to have the same name as the file that defines it.");
+                            }
+                            restSpec.addApi(restApi);
                         }
-                        restSpec.addApi(restApi);
                     }
                 } catch (Exception ex) {
                     throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
index cd9cc1fbe48..cbef40fbcb7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,15 +30,44 @@ import java.util.List;
  * Represents a test section, which is composed of a skip section and multiple executable sections.
  */
 public class ClientYamlTestSection implements Comparable<ClientYamlTestSection> {
+    public static ClientYamlTestSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName());
+        try {
+            parser.nextToken();
+            testSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+            while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+                ParserUtils.advanceToFieldName(parser);
+                testSection.addExecutableSection(ExecutableSection.parse(parser));
+            }
+
+            parser.nextToken();
+            assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected "
+                + XContentParser.Token.END_OBJECT  + " but was " + parser.currentToken();
+            parser.nextToken();
+
+            return testSection;
+        } catch (Exception e) {
+            throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e);
+        }
+    }
+
+    private final XContentLocation location;
     private final String name;
     private SkipSection skipSection;
     private final List<ExecutableSection> executableSections;
 
-    public ClientYamlTestSection(String name) {
+    public ClientYamlTestSection(XContentLocation location, String name) {
+        this.location = location;
         this.name = name;
         this.executableSections = new ArrayList<>();
     }
 
+    public XContentLocation getLocation() {
+        return location;
+    }
+
     public String getName() {
         return name;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
index af3e1be0528..5f97eb4af6a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
@@ -18,6 +18,16 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -28,6 +38,63 @@ import java.util.TreeSet;
  * Supports a setup section and multiple test sections.
  */
 public class ClientYamlTestSuite {
+    public static ClientYamlTestSuite parse(String api, Path file) throws IOException {
+        if (!Files.isRegularFile(file)) {
+            throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file");
+        }
+
+        String filename = file.getFileName().toString();
+        //remove the file extension
+        int i = filename.lastIndexOf('.');
+        if (i > 0) {
+            filename = filename.substring(0, i);
+        }
+
+        //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break.
+        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) {
+            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
+            channel.read(bb, channel.size() - 1);
+            if (bb.get(0) != 10) {
+                throw new IOException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)");
+            }
+        }
+
+        try (XContentParser parser = YamlXContent.yamlXContent.createParser(ExecutableSection.XCONTENT_REGISTRY,
+                Files.newInputStream(file))) {
+            return parse(api, filename, parser);
+        } catch(Exception e) {
+            throw new IOException("Error parsing " + api + "/" + filename, e);
+        }
+    }
+
+    public static ClientYamlTestSuite parse(String api, String suiteName, XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
+                + parser.currentToken();
+
+        ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName);
+
+        restTestSuite.setSetupSection(SetupSection.parseIfNext(parser));
+        restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser));
+
+        while(true) {
+            //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
+            //we need to somehow distinguish between a null in the middle of a test ("---")
+            // and a null at the end of the file (at least two consecutive null tokens)
+            if(parser.currentToken() == null) {
+                if (parser.nextToken() == null) {
+                    break;
+                }
+            }
+
+            ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+            if (!restTestSuite.addTestSection(testSection)) {
+                throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]");
+            }
+        }
+
+        return restTestSuite;
+    }
 
     private final String api;
     private final String name;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index e233e9fab80..17292380404 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -19,15 +19,21 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponseException;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -35,6 +41,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.common.collect.Tuple.tuple;
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.allOf;
@@ -65,6 +72,86 @@ import static org.junit.Assert.fail;
  *
  */
 public class DoSection implements ExecutableSection {
+    public static DoSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        DoSection doSection = new DoSection(parser.getTokenLocation());
+        ApiCallSection apiCallSection = null;
+        Map<String, String> headers = new HashMap<>();
+        List<String> expectedWarnings = new ArrayList<>();
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("catch".equals(currentFieldName)) {
+                    doSection.setCatch(parser.text());
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("warnings".equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) {
+                        expectedWarnings.add(parser.text());
+                    }
+                    if (token != XContentParser.Token.END_ARRAY) {
+                        throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]");
+                    }
+                } else {
+                    throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if ("headers".equals(currentFieldName)) {
+                    String headerName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            headerName = parser.currentName();
+                        } else if (token.isValue()) {
+                            headers.put(headerName, parser.text());
+                        }
+                    }
+                } else if (currentFieldName != null) { // must be part of API call then
+                    apiCallSection = new ApiCallSection(currentFieldName);
+                    String paramName = null;
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            paramName = parser.currentName();
+                        } else if (token.isValue()) {
+                            if ("body".equals(paramName)) {
+                                String body = parser.text();
+                                XContentType bodyContentType = XContentFactory.xContentType(body);
+                                XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser(
+                                        NamedXContentRegistry.EMPTY, body);
+                                //multiple bodies are supported e.g. in case of bulk provided as a whole string
+                                while(bodyParser.nextToken() != null) {
+                                    apiCallSection.addBody(bodyParser.mapOrdered());
+                                }
+                            } else {
+                                apiCallSection.addParam(paramName, parser.text());
+                            }
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            if ("body".equals(paramName)) {
+                                apiCallSection.addBody(parser.mapOrdered());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        try {
+            if (apiCallSection == null) {
+                throw new IllegalArgumentException("client call section is mandatory within a do section");
+            }
+            if (headers.isEmpty() == false) {
+                apiCallSection.addHeaders(headers);
+            }
+            doSection.setApiCallSection(apiCallSection);
+            doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings));
+        } finally {
+            parser.nextToken();
+        }
+        return doSection;
+    }
+
 
     private static final Logger logger = Loggers.getLogger(DoSection.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
index 11b8f3e7a0f..827457f4c2a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java
@@ -18,15 +18,47 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Represents a test fragment that can be executed (e.g. api call, assertion)
  */
 public interface ExecutableSection {
+    /**
+     * {@link NamedXContentRegistry} needed in the {@link XContentParser} before calling {@link ExecutableSection#parse(XContentParser)}.
+     */
+    NamedXContentRegistry XCONTENT_REGISTRY = new NamedXContentRegistry(Arrays.asList(
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gt"), GreaterThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse),
+            new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse)));
+
+    static ExecutableSection parse(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+        String section = parser.currentName();
+        XContentLocation location = parser.getTokenLocation();
+        try {
+            ExecutableSection executableSection = parser.namedObject(ExecutableSection.class, section, null);
+            parser.nextToken();
+            return executableSection;
+        } catch (Exception e) {
+            throw new IOException("Error parsing section starting at [" + location + "]", e);
+        }
+    }
+
     /**
      * Get the location in the test that this was defined. 
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
index b531f180fd6..39a1f1d3780 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
@@ -33,6 +37,15 @@ import static org.junit.Assert.fail;
  * - gt:    { fields._ttl: 0}
  */
 public class GreaterThanAssertion extends Assertion {
+    public static GreaterThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
index 14b1a08a879..3fd9bf7adfd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *   - gte:     { fields._ttl: 0 }
  */
 public class GreaterThanEqualToAssertion extends Assertion {
+    public static GreaterThanEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (! (stringObjectTuple.v2() instanceof Comparable) ) {
+            throw new IllegalArgumentException("gte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(GreaterThanEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
index a356182ab46..56ee603c70f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java
@@ -21,6 +21,9 @@ package org.elasticsearch.test.rest.yaml.section;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -34,6 +37,9 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class IsFalseAssertion extends Assertion {
+    public static IsFalseAssertion parse(XContentParser parser) throws IOException {
+        return new IsFalseAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsFalseAssertion.class);
 
@@ -42,7 +48,6 @@ public class IsFalseAssertion extends Assertion {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] doesn't have a true value (field: [{}])", actualValue, getField());
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
index 76ca0de70d9..9b3f37e1f52 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java
@@ -21,6 +21,9 @@ package org.elasticsearch.test.rest.yaml.section;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.equalToIgnoringCase;
@@ -35,6 +38,9 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class IsTrueAssertion extends Assertion {
+    public static IsTrueAssertion parse(XContentParser parser) throws IOException {
+        return new IsTrueAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser));
+    }
 
     private static final Logger logger = Loggers.getLogger(IsTrueAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
index 062b9ecd87f..aeecc50b90d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java
@@ -19,9 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -35,6 +38,22 @@ import static org.junit.Assert.assertThat;
  * - length:   { hits.hits: 1  }
  */
 public class LengthAssertion extends Assertion {
+    public static LengthAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        assert stringObjectTuple.v2() != null;
+        int value;
+        if (stringObjectTuple.v2() instanceof Number) {
+            value = ((Number) stringObjectTuple.v2()).intValue();
+        } else {
+            try {
+                value = Integer.valueOf(stringObjectTuple.v2().toString());
+            } catch(NumberFormatException e) {
+                throw new IllegalArgumentException("length is not a valid number", e);
+            }
+        }
+        return new LengthAssertion(location, stringObjectTuple.v1(), value);
+    }
 
     private static final Logger logger = Loggers.getLogger(LengthAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
index 591bd83fa62..75a1edcf81c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java
@@ -19,8 +19,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThan;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *
  */
 public class LessThanAssertion extends Assertion {
+    public static LessThanAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanAssertion.class);
 
@@ -42,7 +55,6 @@ public class LessThanAssertion extends Assertion {
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField());
         assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
index 7c5710f689d..23b6a1e4efc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java
@@ -20,8 +20,12 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -34,6 +38,15 @@ import static org.junit.Assert.fail;
  *   - lte:     { fields._ttl: 0 }
  */
 public class LessThanOrEqualToAssertion  extends Assertion {
+    public static LessThanOrEqualToAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        if (false == stringObjectTuple.v2() instanceof Comparable) {
+            throw new IllegalArgumentException("lte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
+        }
+        return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(LessThanOrEqualToAssertion.class);
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
index 60e34247044..82d8dbeebe6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java
@@ -19,15 +19,13 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.NotEqualMessageBuilder;
 
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
+import java.io.IOException;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
@@ -43,6 +41,11 @@ import static org.junit.Assert.assertThat;
  *
  */
 public class MatchAssertion extends Assertion {
+    public static MatchAssertion parse(XContentParser parser) throws IOException {
+        XContentLocation location = parser.getTokenLocation();
+        Tuple<String,Object> stringObjectTuple = ParserUtils.parseTuple(parser);
+        return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2());
+    }
 
     private static final Logger logger = Loggers.getLogger(MatchAssertion.class);
 
@@ -79,127 +82,9 @@ public class MatchAssertion extends Assertion {
         }
 
         if (expectedValue.equals(actualValue) == false) {
-            FailureMessage message = new FailureMessage(getField());
+            NotEqualMessageBuilder message = new NotEqualMessageBuilder();
             message.compare(getField(), actualValue, expectedValue);
-            throw new AssertionError(message.message);
-        }
-    }
-
-    private static class FailureMessage {
-        private final StringBuilder message;
-        private int indent = 0;
-
-        private FailureMessage(String field) {
-            this.message = new StringBuilder(field + " didn't match the expected value:\n");
-        }
-
-        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
-            actual = new TreeMap<>(actual);
-            expected = new TreeMap<>(expected);
-            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
-                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
-            }
-            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
-                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
-            }
-        }
-
-        private void compareLists(List<Object> actual, List<Object> expected) {
-            int i = 0;
-            while (i < actual.size() && i < expected.size()) {
-                compare(Integer.toString(i), actual.get(i), expected.get(i));
-                i++;
-            }
-            if (actual.size() == expected.size()) {
-                return;
-            }
-            indent();
-            if (actual.size() < expected.size()) {
-                message.append("expected [").append(expected.size() - i).append("] more entries\n");
-                return;
-            }
-            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
-        }
-
-        private void compare(String field, @Nullable Object actual, Object expected) {
-            if (expected instanceof Map) {
-                if (actual == null) {
-                    field(field, "expected map but not found");
-                    return;
-                }
-                if (false == actual instanceof Map) {
-                    field(field, "expected map but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                Map<String, Object> expectedMap = (Map<String, Object>) expected;
-                @SuppressWarnings("unchecked")
-                Map<String, Object> actualMap = (Map<String, Object>) actual;
-                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
-                    field(field, "same [empty map]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareMaps(actualMap, expectedMap);
-                indent -= 1;
-                return;
-            }
-            if (expected instanceof List) {
-                if (actual == null) {
-                    field(field, "expected list but not found");
-                    return;
-                }
-                if (false == actual instanceof List) {
-                    field(field, "expected list but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                List<Object> expectedList = (List<Object>) expected;
-                @SuppressWarnings("unchecked")
-                List<Object> actualList = (List<Object>) actual;
-                if (expectedList.isEmpty() && actualList.isEmpty()) {
-                    field(field, "same [empty list]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareLists(actualList, expectedList);
-                indent -= 1;
-                return;
-            }
-            if (actual == null) {
-                field(field, "expected [" + expected + "] but not found");
-                return;
-            }
-            if (Objects.equals(expected, actual)) {
-                if (expected instanceof String) {
-                    String expectedString = (String) expected;
-                    if (expectedString.length() > 50) {
-                        expectedString = expectedString.substring(0, 50) + "...";
-                    }
-                    field(field, "same [" + expectedString + "]");
-                    return;
-                }
-                field(field, "same [" + expected + "]");
-                return;
-            }
-            field(field, "expected [" + expected + "] but was [" + actual + "]");
-        }
-
-        private void indent() {
-            for (int i = 0; i < indent; i++) {
-                message.append("  ");
-            }
-        }
-
-        private void field(Object name, String info) {
-            indent();
-            message.append(String.format(Locale.ROOT, "%30s: ", name));
-            if (info != null) {
-                message.append(info);
-            }
-            message.append('\n');
+            throw new AssertionError(getField() + " didn't match expected value:\n" + message);
         }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
new file mode 100644
index 00000000000..b35450c66e6
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.yaml.section;
+
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Utility methods used for parsing test sections.
+ */
+class ParserUtils {
+    private ParserUtils() {
+        // Do not build.
+    }
+
+    public static String parseField(XContentParser parser) throws IOException {
+        parser.nextToken();
+        assert parser.currentToken().isValue();
+        String field = parser.text();
+        parser.nextToken();
+        return field;
+    }
+
+    public static Tuple<String, Object> parseTuple(XContentParser parser) throws IOException {
+        parser.nextToken();
+        advanceToFieldName(parser);
+        Map<String,Object> map = parser.map();
+        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
+        parser.nextToken();
+
+        if (map.size() != 1) {
+            throw new IllegalArgumentException("expected key value pair but found an object with " + map.size() + " fields");
+        }
+
+        Map.Entry<String, Object> entry = map.entrySet().iterator().next();
+        return Tuple.tuple(entry.getKey(), entry.getValue());
+    }
+
+    public static void advanceToFieldName(XContentParser parser) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        //we are in the beginning, haven't called nextToken yet
+        if (token == null) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_ARRAY) {
+            token = parser.nextToken();
+        }
+        if (token == XContentParser.Token.START_OBJECT) {
+            token = parser.nextToken();
+        }
+        if (token != XContentParser.Token.FIELD_NAME) {
+            throw new IllegalArgumentException("malformed test section: field name expected but found " + token + " at "
+                    + parser.getTokenLocation());
+        }
+    } 
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
index aa39c4d30c8..087f667ac9b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java
@@ -18,7 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 
 import java.io.IOException;
@@ -32,6 +34,28 @@ import java.util.Map;
  *
  */
 public class SetSection implements ExecutableSection {
+    public static SetSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+
+        SetSection setSection = new SetSection(parser.getTokenLocation());
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                setSection.addSet(currentFieldName, parser.text());
+            }
+        }
+
+        parser.nextToken();
+
+        if (setSection.getStash().isEmpty()) {
+            throw new ParsingException(setSection.location, "set section must set at least a value");
+        }
+
+        return setSection;
+    }
 
     private final Map<String, String> stash = new HashMap<>();
     private final XContentLocation location;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
index c2bcaa3ecd1..f0783cc1ccf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java
@@ -18,6 +18,9 @@
  */
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -25,6 +28,41 @@ import java.util.List;
  * Represents a setup section. Holds a skip section and multiple do sections.
  */
 public class SetupSection {
+    /**
+     * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}.
+     */
+    public static SetupSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("setup".equals(parser.currentName())) {
+            parser.nextToken();
+            SetupSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SetupSection parse(XContentParser parser) throws IOException {
+        SetupSection setupSection = new SetupSection();
+        setupSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section");
+            }
+
+            parser.nextToken();
+            setupSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+
+        return setupSection;
+    }
 
     public static final SetupSection EMPTY;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
index e508c3c1015..62588baf46d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java
@@ -19,9 +19,13 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.rest.yaml.Features;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -32,6 +36,63 @@ import java.util.List;
  * - a specific test feature required that might not be implemented yet by the runner
  */
 public class SkipSection {
+    /**
+     * Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}.
+     */
+    public static SkipSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("skip".equals(parser.currentName())) {
+            SkipSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static SkipSection parse(XContentParser parser) throws IOException {
+        String currentFieldName = null;
+        XContentParser.Token token;
+        String version = null;
+        String reason = null;
+        List<String> features = new ArrayList<>();
+
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("version".equals(currentFieldName)) {
+                    version = parser.text();
+                } else if ("reason".equals(currentFieldName)) {
+                    reason = parser.text();
+                } else if ("features".equals(currentFieldName)) {
+                    features.add(parser.text());
+                }
+                else {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "field " + currentFieldName + " not supported within skip section");
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if ("features".equals(currentFieldName)) {
+                    while(parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                        features.add(parser.text());
+                    }
+                }
+            }
+        }
+
+        parser.nextToken();
+
+        if (!Strings.hasLength(version) && features.isEmpty()) {
+            throw new ParsingException(parser.getTokenLocation(), "version or features is mandatory within skip section");
+        }
+        if (Strings.hasLength(version) && !Strings.hasLength(reason)) {
+            throw new ParsingException(parser.getTokenLocation(), "reason is mandatory within skip version section");
+        }
+
+        return new SkipSection(version, features, reason);
+    }
 
     public static final SkipSection EMPTY = new SkipSection();
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
index 1e49f8b5037..1ae4d41a039 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java
@@ -19,10 +19,49 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 public class TeardownSection {
+    /**
+     * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}.
+     */
+    public static TeardownSection parseIfNext(XContentParser parser) throws IOException {
+        ParserUtils.advanceToFieldName(parser);
+
+        if ("teardown".equals(parser.currentName())) {
+            parser.nextToken();
+            TeardownSection section = parse(parser);
+            parser.nextToken();
+            return section;
+        }
+
+        return EMPTY;
+    }
+
+    public static TeardownSection parse(XContentParser parser) throws IOException {
+        TeardownSection teardownSection = new TeardownSection();
+        teardownSection.setSkipSection(SkipSection.parseIfNext(parser));
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            ParserUtils.advanceToFieldName(parser);
+            if (!"do".equals(parser.currentName())) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "section [" + parser.currentName() + "] not supported within teardown section");
+            }
+
+            parser.nextToken();
+            teardownSection.addDoSection(DoSection.parse(parser));
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+        return teardownSection;
+    }
 
     public static final TeardownSection EMPTY;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
index 2a653ce5871..281dbb5115f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
@@ -190,7 +190,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
         private final BaseDirectoryWrapper dir;
         private final TestRuleMarkFailure failureMarker;
 
-        public CloseableDirectory(BaseDirectoryWrapper dir) {
+        CloseableDirectory(BaseDirectoryWrapper dir) {
             this.dir = dir;
             this.failureMarker = ESTestCase.getSuiteFailureMarker();
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index a35a1919cb7..5a03035e457 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -20,8 +20,10 @@
 package org.elasticsearch.test.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -37,6 +39,7 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.tasks.MockTaskManager;
@@ -63,11 +66,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Function;
 
 /**
  * A mock transport service that allows to simulate different network topology failures.
@@ -93,7 +96,7 @@ public final class MockTransportService extends TransportService {
 
     public static MockTransportService createNewService(Settings settings, Version version, ThreadPool threadPool,
             @Nullable ClusterSettings clusterSettings) {
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(ClusterModule.getNamedWriteables());
         final Transport transport = new MockTcpTransport(settings, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
                 new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(settings, Collections.emptyList()), version);
         return new MockTransportService(settings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, clusterSettings);
@@ -109,7 +112,21 @@ public final class MockTransportService extends TransportService {
      */
     public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor,
             @Nullable ClusterSettings clusterSettings) {
-        super(settings, new LookupTestTransport(transport), threadPool, interceptor, clusterSettings);
+        this(settings, transport, threadPool, interceptor, (boundAddress) ->
+            DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), settings.get(Node.NODE_NAME_SETTING.getKey(),
+                UUIDs.randomBase64UUID())), clusterSettings);
+    }
+
+    /**
+     * Build the service.
+     *
+     * @param clusterSettings if non null the the {@linkplain TransportService} will register with the {@link ClusterSettings} for settings
+     *        updates for {@link #TRACE_LOG_EXCLUDE_SETTING} and {@link #TRACE_LOG_INCLUDE_SETTING}.
+     */
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor,
+                                Function<BoundTransportAddress, DiscoveryNode> localNodeFactory,
+                                @Nullable ClusterSettings clusterSettings) {
+        super(settings, new LookupTestTransport(transport), threadPool, interceptor, localNodeFactory, clusterSettings);
         this.original = transport;
     }
 
@@ -683,6 +700,11 @@ public final class MockTransportService extends TransportService {
             return connection.getNode();
         }
 
+        @Override
+        public Version getVersion() {
+            return connection.getVersion();
+        }
+
         @Override
         public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
             throws IOException, TransportException {
@@ -737,6 +759,12 @@ public final class MockTransportService extends TransportService {
     @Override
     protected void doClose() {
         super.doClose();
-        assert openConnections.size() == 0 : "still open connections: " + openConnections;
+        synchronized (openConnections) {
+            assert openConnections.size() == 0 : "still open connections: " + openConnections;
+        }
+    }
+
+    public DiscoveryNode getLocalDiscoNode() {
+        return this.getLocalNode();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 540420f35e7..4ea5ec500af 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.transport;
 
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ExceptionsHelper;
@@ -29,12 +29,9 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListenerResponseHandler;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -44,9 +41,10 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.mocksocket.MockServerSocket;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -54,16 +52,16 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -110,15 +108,12 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         threadPool = new TestThreadPool(getClass().getName());
         clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         serviceA = buildService("TS_A", version0, clusterSettings); // this one supports dynamic tracer updates
-        nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
-        // serviceA.setLocalNode(nodeA);
+        nodeA = serviceA.getLocalNode();
         serviceB = buildService("TS_B", version1, null); // this one doesn't support dynamic tracer updates
-        nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
-        //serviceB.setLocalNode(nodeB);
+        nodeB = serviceB.getLocalNode();
         // wait till all nodes are properly connected and the event has been sent, so tests in this class
         // will not get this callback called on the connections done in this setup
-        final boolean useLocalNode = randomBoolean();
-        final CountDownLatch latch = new CountDownLatch(useLocalNode ? 2 : 4);
+        final CountDownLatch latch = new CountDownLatch(2);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -133,18 +128,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceA.addConnectionListener(waitForConnection);
         serviceB.addConnectionListener(waitForConnection);
         int numHandshakes = 1;
-        if (useLocalNode) {
-            logger.info("--> using local node optimization");
-            serviceA.setLocalNode(nodeA);
-            serviceB.setLocalNode(nodeB);
-        } else {
-            logger.info("--> actively connecting to local node");
-            serviceA.connectToNode(nodeA);
-            serviceB.connectToNode(nodeB);
-            assertNumHandshakes(numHandshakes, serviceA.getOriginalTransport());
-            assertNumHandshakes(numHandshakes, serviceB.getOriginalTransport());
-            numHandshakes++;
-        }
         serviceA.connectToNode(nodeB);
         serviceB.connectToNode(nodeA);
         assertNumHandshakes(numHandshakes, serviceA.getOriginalTransport());
@@ -208,16 +191,13 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     public void testHelloWorld() {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
-                    assertThat("moshe", equalTo(request.message));
-                    try {
-                        channel.sendResponse(new StringMessageResponse("hello " + request.message));
-                    } catch (IOException e) {
-                        logger.error("Unexpected failure", e);
-                        fail(e.getMessage());
-                    }
+            (request, channel) -> {
+                assertThat("moshe", equalTo(request.message));
+                try {
+                    channel.sendResponse(new StringMessageResponse("hello " + request.message));
+                } catch (IOException e) {
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             });
 
@@ -341,20 +321,15 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     public void testLocalNodeConnection() throws InterruptedException {
         assertTrue("serviceA is not connected to nodeA", serviceA.nodeConnected(nodeA));
-        if (((TransportService) serviceA).getLocalNode() != null) {
-            // this should be a noop
-            serviceA.disconnectFromNode(nodeA);
-        }
+        // this should be a noop
+        serviceA.disconnectFromNode(nodeA);
         final AtomicReference<Exception> exception = new AtomicReference<>();
         serviceA.registerRequestHandler("localNode", StringMessageRequest::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<StringMessageRequest>() {
-                @Override
-                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
-                    try {
-                        channel.sendResponse(new StringMessageResponse(request.message));
-                    } catch (IOException e) {
-                        exception.set(e);
-                    }
+            (request, channel) -> {
+                try {
+                    channel.sendResponse(new StringMessageResponse(request.message));
+                } catch (IOException e) {
+                    exception.set(e);
                 }
             });
         final AtomicReference<String> responseString = new AtomicReference<>();
@@ -840,7 +815,6 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         assertTrue(inFlight.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS));
     }
 
-    @TestLogging("org.elasticsearch.transport:DEBUG")
     public void testTracerLog() throws InterruptedException {
         TransportRequestHandler handler = (request, channel) -> channel.sendResponse(new StringMessageResponse(""));
         TransportRequestHandler handlerWithError = new TransportRequestHandler<StringMessageRequest>() {
@@ -883,7 +857,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceB.registerRequestHandler("test", StringMessageRequest::new, ThreadPool.Names.SAME, handler);
         serviceB.registerRequestHandler("testError", StringMessageRequest::new, ThreadPool.Names.SAME, handlerWithError);
 
-        final Tracer tracer = new Tracer();
+        final Tracer tracer = new Tracer(new HashSet<>(Arrays.asList("test", "testError")));
+        // the tracer is invoked concurrently after the actual action is executed. that means a Tracer#requestSent can still be in-flight
+        // from a handshake executed on connect in the setup method. this might confuse this test since it expects exact number of
+        // invocations. To prevent any unrelated events messing with this test we filter on the actions we execute in this test.
         serviceA.addTracer(tracer);
         serviceB.addTracer(tracer);
 
@@ -947,6 +924,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     }
 
     private static class Tracer extends MockTransportService.Tracer {
+        private final Set<String> actions;
         public volatile boolean sawRequestSent;
         public volatile boolean sawRequestReceived;
         public volatile boolean sawResponseSent;
@@ -954,42 +932,52 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         public volatile boolean sawResponseReceived;
 
         public AtomicReference<CountDownLatch> expectedEvents = new AtomicReference<>();
-        private final Logger logger = Loggers.getLogger(getClass());
-
+        Tracer(Set<String> actions) {
+            this.actions = actions;
+        }
         @Override
         public void receivedRequest(long requestId, String action) {
             super.receivedRequest(requestId, action);
-            sawRequestReceived = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawRequestReceived = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
             super.requestSent(node, requestId, action, options);
-            sawRequestSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawRequestSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void responseSent(long requestId, String action) {
             super.responseSent(requestId, action);
-            logger.debug("#### responseSent for action: {}", action);
-            sawResponseSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawResponseSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void responseSent(long requestId, String action, Throwable t) {
             super.responseSent(requestId, action, t);
-            sawErrorSent = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawErrorSent = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         @Override
         public void receivedResponse(long requestId, DiscoveryNode sourceNode, String action) {
             super.receivedResponse(requestId, sourceNode, action);
-            sawResponseReceived = true;
-            expectedEvents.get().countDown();
+            if (actions.contains(action)) {
+                sawResponseReceived = true;
+                expectedEvents.get().countDown();
+            }
         }
 
         public void reset(int expectedCount) {
@@ -1358,7 +1346,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             // all is well
         }
 
-        try (Transport.Connection connection = serviceB.openConnection(nodeA, ConnectionProfile.LIGHT_PROFILE)){
+        try (Transport.Connection connection = serviceB.openConnection(nodeA, MockTcpTransport.LIGHT_PROFILE)) {
             serviceB.handshake(connection, 100);
             fail("exception should be thrown");
         } catch (IllegalStateException e) {
@@ -1416,7 +1404,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             // all is well
         }
 
-        try (Transport.Connection connection = serviceB.openConnection(nodeA, ConnectionProfile.LIGHT_PROFILE)){
+        try (Transport.Connection connection = serviceB.openConnection(nodeA, MockTcpTransport.LIGHT_PROFILE)) {
             serviceB.handshake(connection, 100);
             fail("exception should be thrown");
         } catch (IllegalStateException e) {
@@ -1555,10 +1543,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         String info;
 
-        public TestResponse() {
+        TestResponse() {
         }
 
-        public TestResponse(String info) {
+        TestResponse(String info) {
             this.info = info;
         }
 
@@ -1591,11 +1579,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 .build(),
             version0,
             null, true);
-        DiscoveryNode nodeC =
-            new DiscoveryNode("TS_C", "TS_C", serviceC.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+        DiscoveryNode nodeC = serviceC.getLocalNode();
         serviceC.acceptIncomingRequests();
 
-        final CountDownLatch latch = new CountDownLatch(5);
+        final CountDownLatch latch = new CountDownLatch(4);
         TransportConnectionListener waitForConnection = new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
@@ -1615,12 +1602,11 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceC.connectToNode(nodeB);
         serviceA.connectToNode(nodeC);
         serviceB.connectToNode(nodeC);
-        serviceC.connectToNode(nodeC);
 
         latch.await();
         serviceA.removeConnectionListener(waitForConnection);
         serviceB.removeConnectionListener(waitForConnection);
-        serviceB.removeConnectionListener(waitForConnection);
+        serviceC.removeConnectionListener(waitForConnection);
 
 
         Map<TransportService, DiscoveryNode> toNodeMap = new HashMap<>();
@@ -1704,7 +1690,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
             private final int id;
 
-            public TestResponseHandler(int id) {
+            TestResponseHandler(int id) {
                 this.id = id;
             }
 
@@ -1770,7 +1756,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     public void testTimeoutPerConnection() throws IOException {
         assumeTrue("Works only on BSD network stacks and apparently windows",
             Constants.MAC_OS_X || Constants.FREE_BSD || Constants.WINDOWS);
-        try (ServerSocket socket = new ServerSocket()) {
+        try (ServerSocket socket = new MockServerSocket()) {
             // note - this test uses backlog=1 which is implementation specific ie. it might not work on some TCP/IP stacks
             // on linux (at least newer ones) the listen(addr, backlog=1) should just ignore new connections if the queue is full which
             // means that once we received an ACK from the client we just drop the packet on the floor (which is what we want) and we run
@@ -1809,19 +1795,65 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
     }
 
+    public void testHandshakeWithIncompatVersion() {
+        assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()),
+            Version.fromString("2.0.0"))) {
+            transport.transportServiceAdapter(serviceA.new Adapter());
+            transport.start();
+            DiscoveryNode node =
+                new DiscoveryNode("TS_TPC", "TS_TPC", transport.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            expectThrows(ConnectTransportException.class, () -> serviceA.openConnection(node, builder.build()));
+        }
+    }
+
+    public void testHandshakeUpdatesVersion() throws IOException {
+        assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
+        Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
+        try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList()),version)) {
+            transport.transportServiceAdapter(serviceA.new Adapter());
+            transport.start();
+            DiscoveryNode node =
+                new DiscoveryNode("TS_TPC", "TS_TPC", transport.boundAddress().publishAddress(), emptyMap(), emptySet(),
+                    Version.fromString("2.0.0"));
+            ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+            builder.addConnections(1,
+                TransportRequestOptions.Type.BULK,
+                TransportRequestOptions.Type.PING,
+                TransportRequestOptions.Type.RECOVERY,
+                TransportRequestOptions.Type.REG,
+                TransportRequestOptions.Type.STATE);
+            try (Transport.Connection connection = serviceA.openConnection(node, builder.build())) {
+                assertEquals(connection.getVersion(), version);
+            }
+        }
+    }
+
+
     public void testTcpHandshake() throws IOException, InterruptedException {
         assumeTrue("only tcp transport has a handshake method", serviceA.getOriginalTransport() instanceof TcpTransport);
         TcpTransport originalTransport = (TcpTransport) serviceA.getOriginalTransport();
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
 
         try (MockTcpTransport transport = new MockTcpTransport(Settings.EMPTY, threadPool, BigArrays.NON_RECYCLING_INSTANCE,
-            new NoneCircuitBreakerService(), namedWriteableRegistry,  new NetworkService(Settings.EMPTY, Collections.emptyList())){
+            new NoneCircuitBreakerService(), namedWriteableRegistry, new NetworkService(Settings.EMPTY, Collections.emptyList())) {
             @Override
             protected String handleRequest(MockChannel mockChannel, String profileName, StreamInput stream, long requestId,
                                            int messageLengthBytes, Version version, InetSocketAddress remoteAddress, byte status)
                 throws IOException {
                 return super.handleRequest(mockChannel, profileName, stream, requestId, messageLengthBytes, version, remoteAddress,
-                    (byte)(status & ~(1<<3))); // we flip the isHanshake bit back and ackt like the handler is not found
+                    (byte)(status & ~(1<<3))); // we flip the isHandshake bit back and act like the handler is not found
             }
         }) {
             transport.transportServiceAdapter(serviceA.new Adapter());
@@ -1846,7 +1878,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     }
 
     public void testTcpHandshakeTimeout() throws IOException {
-        try (ServerSocket socket = new ServerSocket()) {
+        try (ServerSocket socket = new MockServerSocket()) {
             socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
             socket.setReuseAddress(true);
             DiscoveryNode dummy = new DiscoveryNode("TEST", new TransportAddress(socket.getInetAddress(),
@@ -1862,12 +1894,12 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             builder.setHandshakeTimeout(TimeValue.timeValueMillis(1));
             ConnectTransportException ex = expectThrows(ConnectTransportException.class,
                 () -> serviceA.connectToNode(dummy, builder.build()));
-            assertEquals("[][" + dummy.getAddress() +"] handshake_timeout[1ms]", ex.getMessage());
+            assertEquals("[][" + dummy.getAddress() + "] handshake_timeout[1ms]", ex.getMessage());
         }
     }
 
     public void testTcpHandshakeConnectionReset() throws IOException, InterruptedException {
-        try (ServerSocket socket = new ServerSocket()) {
+        try (ServerSocket socket = new MockServerSocket()) {
             socket.bind(new InetSocketAddress(InetAddress.getLocalHost(), 0), 1);
             socket.setReuseAddress(true);
             DiscoveryNode dummy = new DiscoveryNode("TEST", new TransportAddress(socket.getInetAddress(),
@@ -1896,9 +1928,73 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             builder.setHandshakeTimeout(TimeValue.timeValueHours(1));
             ConnectTransportException ex = expectThrows(ConnectTransportException.class,
                 () -> serviceA.connectToNode(dummy, builder.build()));
-            assertEquals(ex.getMessage(), "[][" + dummy.getAddress() +"] general node connection failure");
+            assertEquals(ex.getMessage(), "[][" + dummy.getAddress() + "] general node connection failure");
             assertThat(ex.getCause().getMessage(), startsWith("handshake failed"));
             t.join();
         }
     }
+
+    public void testResponseHeadersArePreserved() throws InterruptedException {
+        List<String> executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet());
+        CollectionUtil.timSort(executors); // makes sure it's reproducible
+        serviceA.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
+            (request, channel) -> {
+
+                threadPool.getThreadContext().putTransient("boom", new Object());
+                threadPool.getThreadContext().addResponseHeader("foo.bar", "baz");
+                if ("fail".equals(request.info)) {
+                    throw new RuntimeException("boom");
+                } else {
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
+                }
+            });
+
+        CountDownLatch latch = new CountDownLatch(2);
+
+        TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() {
+            @Override
+            public TransportResponse newInstance() {
+                return TransportResponse.Empty.INSTANCE;
+            }
+
+            @Override
+            public void handleResponse(TransportResponse response) {
+                try {
+                    assertSame(response, TransportResponse.Empty.INSTANCE);
+                    assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar"));
+                    assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size());
+                    assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0));
+                    assertNull(threadPool.getThreadContext().getTransient("boom"));
+                } finally {
+                    latch.countDown();
+                }
+
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                try {
+                    assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar"));
+                    assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size());
+                    assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0));
+                    assertNull(threadPool.getThreadContext().getTransient("boom"));
+                } finally {
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public String executor() {
+                if (1 == 1)
+                    return "same";
+
+                return randomFrom(executors);
+            }
+        };
+
+        serviceB.sendRequest(nodeA, "action",  new TestRequest(randomFrom("fail", "pass")), transportResponseHandler);
+        serviceA.sendRequest(nodeA, "action",  new TestRequest(randomFrom("fail", "pass")), transportResponseHandler);
+        latch.await();
+    }
+
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AssertingTransportInterceptor.java b/test/framework/src/main/java/org/elasticsearch/transport/AssertingTransportInterceptor.java
index 0c6c18e1a20..cbe2006bf74 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/AssertingTransportInterceptor.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/AssertingTransportInterceptor.java
@@ -22,6 +22,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.Task;
@@ -51,7 +52,8 @@ public final class AssertingTransportInterceptor implements TransportInterceptor
         }
 
         @Override
-        public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry) {
+        public List<TransportInterceptor> getTransportInterceptors(NamedWriteableRegistry namedWriteableRegistry,
+                                                                   ThreadContext threadContext) {
             return Collections.singletonList(new AssertingTransportInterceptor(settings, namedWriteableRegistry));
         }
     }
@@ -64,6 +66,7 @@ public final class AssertingTransportInterceptor implements TransportInterceptor
 
     @Override
     public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(String action, String executor,
+                                                                                    boolean forceExecution,
                                                                                     TransportRequestHandler<T> actualHandler) {
         return new TransportRequestHandler<T>() {
 
diff --git a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
index acda061a704..2bb6b87fe15 100644
--- a/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/transport/MockTcpTransport.java
@@ -35,6 +35,8 @@ import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.mocksocket.MockServerSocket;
+import org.elasticsearch.mocksocket.MockSocket;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.BufferedInputStream;
@@ -47,8 +49,11 @@ import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketException;
 import java.net.SocketTimeoutException;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
@@ -66,6 +71,25 @@ import java.util.function.Consumer;
  */
 public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel> {
 
+    /**
+     * A pre-built light connection profile that shares a single connection across all
+     * types.
+     */
+    public static final ConnectionProfile LIGHT_PROFILE;
+
+    private final Set<MockChannel> openChannels = new HashSet<>();
+
+    static  {
+        ConnectionProfile.Builder builder = new ConnectionProfile.Builder();
+        builder.addConnections(1,
+            TransportRequestOptions.Type.BULK,
+            TransportRequestOptions.Type.PING,
+            TransportRequestOptions.Type.RECOVERY,
+            TransportRequestOptions.Type.REG,
+            TransportRequestOptions.Type.STATE);
+        LIGHT_PROFILE = builder.build();
+    }
+
     private final ExecutorService executor;
     private final Version mockVersion;
 
@@ -93,7 +117,7 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
 
     @Override
     protected MockChannel bind(final String name, InetSocketAddress address) throws IOException {
-        ServerSocket socket = new ServerSocket();
+        MockServerSocket socket = new MockServerSocket();
         socket.bind(address);
         socket.setReuseAddress(TCP_REUSE_ADDRESS.get(settings));
         ByteSizeValue tcpReceiveBufferSize = TCP_RECEIVE_BUFFER_SIZE.get(settings);
@@ -159,9 +183,9 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
     @Override
     protected NodeChannels connectToChannels(DiscoveryNode node, ConnectionProfile profile) throws IOException {
         final MockChannel[] mockChannels = new MockChannel[1];
-        final NodeChannels nodeChannels = new NodeChannels(node, mockChannels, ConnectionProfile.LIGHT_PROFILE); // we always use light here
+        final NodeChannels nodeChannels = new NodeChannels(node, mockChannels, LIGHT_PROFILE); // we always use light here
         boolean success = false;
-        final Socket socket = new Socket();
+        final MockSocket socket = new MockSocket();
         try {
             Consumer<MockChannel> onClose = (channel) -> {
                 final NodeChannels connected = connectedNodes.get(node);
@@ -245,7 +269,7 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
         private final AtomicBoolean isOpen = new AtomicBoolean(true);
         private final InetSocketAddress localAddress;
         private final ServerSocket serverSocket;
-        private final ConcurrentHashMap<MockChannel, Boolean> workerChannels = new ConcurrentHashMap<>();
+        private final Set<MockChannel> workerChannels = Collections.newSetFromMap(new ConcurrentHashMap<>());
         private final Socket activeChannel;
         private final String profile;
         private final CancellableThreads cancellableThreads = new CancellableThreads();
@@ -265,6 +289,9 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
             this.serverSocket = null;
             this.profile = profile;
             this.onClose = () -> onClose.accept(this);
+            synchronized (openChannels) {
+                openChannels.add(this);
+            }
         }
 
         /**
@@ -279,6 +306,9 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
             this.profile = profile;
             this.activeChannel = null;
             this.onClose = null;
+            synchronized (openChannels) {
+                openChannels.add(this);
+            }
         }
 
         public void accept(Executor executor) throws IOException {
@@ -287,11 +317,12 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
                 MockChannel incomingChannel = null;
                 try {
                     configureSocket(incomingSocket);
-                    incomingChannel = new MockChannel(incomingSocket, localAddress, profile, workerChannels::remove);
-                    //establish a happens-before edge between closing and accepting a new connection
                     synchronized (this) {
                         if (isOpen.get()) {
-                            workerChannels.put(incomingChannel, Boolean.TRUE);
+                            incomingChannel = new MockChannel(incomingSocket, localAddress, profile, workerChannels::remove);
+                            //establish a happens-before edge between closing and accepting a new connection
+                            workerChannels.add(incomingChannel);
+
                             // this spawns a new thread immediately, so OK under lock
                             incomingChannel.loopRead(executor);
                             // the channel is properly registered and will be cleared by the close code.
@@ -314,8 +345,9 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
                     if (isOpen.get()) {
                         try {
                             onException(MockChannel.this, e);
-                        } catch (IOException ex) {
+                        } catch (Exception ex) {
                             logger.warn("failed on handling exception", ex);
+                            IOUtils.closeWhileHandlingException(MockChannel.this); // pure paranoia
                         }
                     }
                 }
@@ -332,16 +364,34 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
         }
 
         @Override
-        public void close() throws IOException {
+        public synchronized void close() throws IOException {
+            // establish a happens-before edge between closing and accepting a new connection
+            // we have to sync this entire block to ensure that our openChannels checks work correctly.
+            // The close block below will close all worker channels but if one of the worker channels runs into an exception
+            // for instance due to a disconnect the handling of this exception might be executed concurrently.
+            // now if we are in-turn concurrently call close we might not wait for the actual close to happen and that will, down the road
+            // make the assertion trip that not all channels are closed.
             if (isOpen.compareAndSet(true, false)) {
-                //establish a happens-before edge between closing and accepting a new connection
-                synchronized (this) {
-                    onChannelClosed(this);
-                    IOUtils.close(serverSocket, activeChannel, () -> IOUtils.close(workerChannels.keySet()),
-                        () -> cancellableThreads.cancel("channel closed"), onClose);
+                final boolean removedChannel;
+                synchronized (openChannels) {
+                    removedChannel = openChannels.remove(this);
                 }
+                onChannelClosed(this);
+                IOUtils.close(serverSocket, activeChannel, () -> IOUtils.close(workerChannels),
+                    () -> cancellableThreads.cancel("channel closed"), onClose);
+                assert removedChannel: "Channel was not removed or removed twice?";
             }
         }
+
+        @Override
+        public String toString() {
+            return "MockChannel{" +
+                "profile='" + profile + '\'' +
+                ", isOpen=" + isOpen +
+                ", localAddress=" + localAddress +
+                ", isServerSocket=" + (serverSocket != null) +
+                '}';
+        }
     }
 
 
@@ -369,12 +419,14 @@ public class MockTcpTransport extends TcpTransport<MockTcpTransport.MockChannel>
     @Override
     protected void stopInternal() {
         ThreadPool.terminate(executor, 10, TimeUnit.SECONDS);
+        synchronized (openChannels) {
+            assert openChannels.isEmpty() : "there are still open channels: " + openChannels;
+        }
     }
 
     @Override
     protected Version getCurrentVersion() {
         return mockVersion;
     }
-
 }
 
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index 0776fa1edb2..5f7e38b7ec1 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -32,10 +33,16 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 
 public class MockSearchServiceTests extends ESTestCase {
+    public static final IndexMetaData EMPTY_INDEX_METADATA = IndexMetaData.builder("")
+        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
+        .numberOfShards(1).numberOfReplicas(0).build();
+
     public void testAssertNoInFlightContext() {
-        final long nowInMillis = randomPositiveLong();
-        SearchContext s = new TestSearchContext(new QueryShardContext(0, new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null,
-                null, null, null, null, null, null, () -> nowInMillis)) {
+        final long nowInMillis = randomNonNegativeLong();
+        SearchContext s = new TestSearchContext(new QueryShardContext(0,
+            new IndexSettings(EMPTY_INDEX_METADATA, Settings.EMPTY), null, null, null, null, null, xContentRegistry(),
+            null, null, () -> nowInMillis)) {
+
             @Override
             public SearchShardTarget shardTarget() {
                 return new SearchShardTarget("node", new Index("idx", "ignored"), 0);
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
deleted file mode 100644
index 6057de6c702..00000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java
+++ /dev/null
@@ -1,455 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ApiCallSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.hamcrest.MatcherAssert;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-import static java.util.Collections.singletonList;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class DoSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseDoSectionNoBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "get:\n" +
-                "    index:    test_index\n" +
-                "    type:    test_type\n" +
-                "    id:        1"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("get"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionNoParamsNoBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "cluster.node_info: {}"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithJsonBody() throws Exception {
-        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   " + body
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
-        String bodies[] = new String[]{
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
-                "{ \"f1\":\"v1\", \"f2\":42 }\n",
-                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
-                "{ \"f1\":\"v2\", \"f2\":47 }\n"
-        };
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: |\n" +
-                "        " + bodies[0] +
-                "        " + bodies[1] +
-                "        " + bodies[2] +
-                "        " + bodies[3]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(4));
-    }
-
-    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
-        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
-            XContent.isStrictDuplicateDetectionEnabled());
-
-        String[] bodies = new String[] {
-                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
-                "{ \"f1\":\"v1\", \"f2\":42 }",
-        };
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body: \n" +
-                "        " + bodies[0] + "\n" +
-                "    body: \n" +
-                "        " + bodies[1]
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBody() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "search:\n" +
-                "    body:\n" +
-                "        \"_source\": [ include.field1, include.field2 ]\n" +
-                "        \"query\": { \"match_all\": {} }"
-        );
-        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("search"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        - index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "        - f1: v1\n" +
-                "          f2: 42\n" +
-                "        - index:\n" +
-                "            _index: test_index2\n" +
-                "            _type:  test_type2\n" +
-                "            _id:    test_id2\n" +
-                "        - f1: v2\n" +
-                "          f2: 47"
-        );
-        String[] bodies = new String[4];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
-        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
-        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
-            XContent.isStrictDuplicateDetectionEnabled());
-
-        parser = createParser(YamlXContent.yamlXContent,
-                "bulk:\n" +
-                "    refresh: true\n" +
-                "    body:\n" +
-                "        index:\n" +
-                "            _index: test_index\n" +
-                "            _type:  test_type\n" +
-                "            _id:    test_id\n" +
-                "    body:\n" +
-                "        f1: v1\n" +
-                "        f2: 42\n"
-        );
-        String[] bodies = new String[2];
-        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
-        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("bulk"));
-        assertThat(apiCallSection.getParams().size(), equalTo(1));
-        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
-
-        for (int i = 0; i < bodies.length; i++) {
-            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
-        }
-    }
-
-    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "mget:\n" +
-                "    body:\n" +
-                "        docs:\n" +
-                "            - { _index: test_2, _type: test, _id: 1}\n" +
-                "            - { _index: test_1, _type: none, _id: 1}"
-        );
-        String body = "{ \"docs\": [ " +
-                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
-                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
-                "]}";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("mget"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        assertJsonEquals(apiCallSection.getBodies().get(0), body);
-    }
-
-    public void testParseDoSectionWithBodyStringified() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    index:  test_1\n" +
-                "    type:   test\n" +
-                "    id:     1\n" +
-                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection, notNullValue());
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(3));
-        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
-        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
-        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(1));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-    }
-
-    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "index:\n" +
-                "    body:\n" +
-                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
-                "        - { size: 100, query: { match_all: {} } }"
-        );
-
-        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        ApiCallSection apiCallSection = doSection.getApiCallSection();
-
-        assertThat(apiCallSection.getApi(), equalTo("index"));
-        assertThat(apiCallSection.getParams().size(), equalTo(0));
-        assertThat(apiCallSection.hasBody(), equalTo(true));
-        assertThat(apiCallSection.getBodies().size(), equalTo(2));
-        //stringified body is taken as is
-        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
-        assertJsonEquals(apiCallSection.getBodies().get(1), body);
-    }
-
-    public void testParseDoSectionWithCatch() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "catch: missing\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseDoSectionWithHeaders() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "headers:\n" +
-                "    Authorization: \"thing one\"\n" +
-                "    Content-Type: \"application/json\"\n" +
-                "indices.get_warmer:\n" +
-                "    index: test_index\n" +
-                "    name: test_warmer"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
-        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
-        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
-    }
-
-    public void testParseDoSectionWithoutClientCallSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "catch: missing\n"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        try {
-            doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
-        }
-    }
-
-    public void testParseDoSectionMultivaluedField() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "        field: [ text , text1 ]"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-    }
-
-    public void testParseDoSectionExpectedWarnings() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "        type: test_type\n" +
-                "warnings:\n" +
-                "    - some test warning they are typically pretty long\n" +
-                "    - some other test warning somtimes they have [in] them"
-        );
-
-        DoSectionParser doSectionParser = new DoSectionParser();
-        DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
-        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
-                "some test warning they are typically pretty long",
-                "some other test warning somtimes they have [in] them")));
-
-        parser = createParser(YamlXContent.yamlXContent,
-                "indices.get_field_mapping:\n" +
-                "        index: test_index\n" +
-                "warnings:\n" +
-                "    - just one entry this time"
-        );
-
-        doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
-                "just one entry this time")));
-
-    }
-
-    private void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
-        Map<String,Object> expectedMap;
-        try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) {
-            expectedMap = parser.mapOrdered();
-        }
-        MatcherAssert.assertThat(actual, equalTo(expectedMap));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
deleted file mode 100644
index 9f488fd9d5c..00000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.util.Arrays;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class SkipSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseSkipSectionVersionNoFeature() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 2.1.0\"\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionAllVersions() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-            "version:     \" all \"\n" +
-            "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
-        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
-        assertThat(skipSection.getFeatures().size(), equalTo(0));
-        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
-    }
-
-    public void testParseSkipSectionFeatureNoVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "features:     regex"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(1));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "features:     [regex1,regex2,regex3]"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-
-        SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(skipSection, notNullValue());
-        assertThat(skipSection.isVersionCheck(), equalTo(false));
-        assertThat(skipSection.getFeatures().size(), equalTo(3));
-        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
-        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
-        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
-        assertThat(skipSection.getReason(), nullValue());
-    }
-
-    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 0.90.2\"\n" +
-                "features:     regex\n" +
-                "reason:      Delete ignores the parent param"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        SkipSection parse = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertEquals(VersionUtils.getFirstVersion(), parse.getLowerVersion());
-        assertEquals(Version.fromString("0.90.2"), parse.getUpperVersion());
-        assertEquals(Arrays.asList("regex"), parse.getFeatures());
-        assertEquals("Delete ignores the parent param", parse.getReason());
-    }
-
-    public void testParseSkipSectionNoReason() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "version:     \" - 0.90.2\"\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
-        }
-    }
-
-    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "reason:      Delete ignores the parent param\n"
-        );
-
-        SkipSectionParser skipSectionParser = new SkipSectionParser();
-        try {
-            skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
-        }
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
deleted file mode 100644
index 5673f20f065..00000000000
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.yaml.parser;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.LengthAssertion;
-import org.elasticsearch.test.rest.yaml.section.LessThanAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
-import org.elasticsearch.test.rest.yaml.section.SkipSection;
-
-import java.util.Map;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-public class TestSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-    public void testParseTestSectionWithDoSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"First test section\": \n" +
-                " - do :\n" +
-                "     catch: missing\n" +
-                "     indices.get_warmer:\n" +
-                "         index: test_index\n" +
-                "         name: test_warmer"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(1));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
-        String yaml =
-                "\"First test section\": \n" +
-                        "  - skip:\n" +
-                        "      version:  \"2.0.0 - 2.2.0\"\n" +
-                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
-                        "  - do :\n" +
-                        "      catch: missing\n" +
-                        "      indices.get_warmer:\n" +
-                        "          index: test_index\n" +
-                        "          name: test_warmer\n" +
-                        "  - set: {_scroll_id: scroll_id}";
-
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        parser = createParser(YamlXContent.yamlXContent,yaml);
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("First test section"));
-        assertThat(testSection.getSkipSection(), notNullValue());
-        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
-        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
-        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), equalTo("missing"));
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
-        assertThat(setSection.getStash().size(), equalTo(1));
-        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
-    }
-
-    public void testParseTestSectionWithMultipleDoSections() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(2));
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-    }
-
-    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"Basic\":\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      index:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        type:  test\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - match: { _index:   test_1 }\n" +
-                        "  - is_true: _source\n" +
-                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
-                        "\n" +
-                        "  - do:\n" +
-                        "      get:\n" +
-                        "        index: test_1\n" +
-                        "        id:    中文\n" +
-                        "\n" +
-                        "  - length: { _index:   6 }\n" +
-                        "  - is_false: whatever\n" +
-                        "  - gt: { size: 5      }\n" +
-                        "  - lt: { size: 10      }"
-        );
-
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("Basic"));
-        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
-        assertThat(testSection.getExecutableSections().size(), equalTo(10));
-
-        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(1);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
-        assertThat(matchAssertion.getField(), equalTo("_index"));
-        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
-
-        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
-        assertThat(trueAssertion.getField(), equalTo("_source"));
-
-        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
-        assertThat(matchAssertion.getField(), equalTo("_source"));
-        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
-        Map map = (Map) matchAssertion.getExpectedValue();
-        assertThat(map.size(), equalTo(1));
-        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
-
-        doSection = (DoSection)testSection.getExecutableSections().get(5);
-        assertThat(doSection.getCatch(), nullValue());
-        assertThat(doSection.getApiCallSection(), notNullValue());
-        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
-        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
-        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
-
-        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
-        assertThat(lengthAssertion.getField(), equalTo("_index"));
-        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
-
-        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
-        assertThat(falseAssertion.getField(), equalTo("whatever"));
-
-        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
-        assertThat(greaterThanAssertion.getField(), equalTo("size"));
-        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
-
-        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
-        assertThat(lessThanAssertion.getField(), equalTo("size"));
-        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
-        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
-    }
-
-    public void testSmallSection() throws Exception {
-        parser = createParser(YamlXContent.yamlXContent,
-                "\"node_info test\":\n" +
-                "  - do:\n" +
-                "      cluster.node_info: {}\n" +
-                "  \n" +
-                "  - is_true: nodes\n" +
-                "  - is_true: cluster_name\n");
-        ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser();
-        ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-        assertThat(testSection, notNullValue());
-        assertThat(testSection.getName(), equalTo("node_info test"));
-        assertThat(testSection.getExecutableSections().size(), equalTo(3));
-    }
-}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
index caca95b1315..6acdd935400 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.rest.yaml.restspec;
 
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.AbstractClientYamlTestFragmentParserTestCase;
+import org.elasticsearch.test.rest.yaml.section.AbstractClientYamlTestFragmentParserTestCase;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
similarity index 69%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
index 3b0859d2d8b..b7038a7d7a3 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java
@@ -17,19 +17,20 @@
  * under the License.
  */
 
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
 import org.junit.After;
 
 import static org.hamcrest.Matchers.nullValue;
 
 /**
- * Superclass for tests of subclasses of {@link ClientYamlTestFragmentParser}.
+ * Superclass for tests that parse parts of the test suite.
  */
 public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTestCase {
-
     protected XContentParser parser;
 
     @Override
@@ -38,9 +39,16 @@ public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTes
         super.tearDown();
         // test may be skipped so we did not create a parser instance
         if (parser != null) {
-            //this is the way to make sure that we consumed the whole yaml
+            //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
             assertThat(parser.currentToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
+            assertThat(parser.nextToken(), nullValue());
             parser.close();
         }
     }
+
+    @Override
+    protected NamedXContentRegistry xContentRegistry() {
+        return ExecutableSection.XCONTENT_REGISTRY;
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
similarity index 75%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
index c908f079149..fcef7467835 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion;
@@ -33,14 +33,13 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class AssertionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseIsTrue() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "get.fields._timestamp"
         );
 
-        IsTrueParser isTrueParser = new IsTrueParser();
-        IsTrueAssertion trueAssertion = isTrueParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        IsTrueAssertion trueAssertion = IsTrueAssertion.parse(parser);
 
         assertThat(trueAssertion, notNullValue());
         assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp"));
@@ -51,8 +50,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "docs.1._source"
         );
 
-        IsFalseParser isFalseParser = new IsFalseParser();
-        IsFalseAssertion falseAssertion = isFalseParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        IsFalseAssertion falseAssertion = IsFalseAssertion.parse(parser);
 
         assertThat(falseAssertion, notNullValue());
         assertThat(falseAssertion.getField(), equalTo("docs.1._source"));
@@ -63,8 +61,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 3}"
         );
 
-        GreaterThanParser greaterThanParser = new GreaterThanParser();
-        GreaterThanAssertion greaterThanAssertion = greaterThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        GreaterThanAssertion greaterThanAssertion = GreaterThanAssertion.parse(parser);
         assertThat(greaterThanAssertion, notNullValue());
         assertThat(greaterThanAssertion.getField(), equalTo("field"));
         assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -76,8 +73,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 3}"
         );
 
-        LessThanParser lessThanParser = new LessThanParser();
-        LessThanAssertion lessThanAssertion = lessThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        LessThanAssertion lessThanAssertion = LessThanAssertion.parse(parser);
         assertThat(lessThanAssertion, notNullValue());
         assertThat(lessThanAssertion.getField(), equalTo("field"));
         assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -89,8 +85,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ _id: 22}"
         );
 
-        LengthParser lengthParser = new LengthParser();
-        LengthAssertion lengthAssertion = lengthParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        LengthAssertion lengthAssertion = LengthAssertion.parse(parser);
         assertThat(lengthAssertion, notNullValue());
         assertThat(lengthAssertion.getField(), equalTo("_id"));
         assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
@@ -102,8 +97,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ field: 10 }"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("field"));
@@ -116,8 +110,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ foo: bar }"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("foo"));
@@ -130,13 +123,12 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{'matches': ['test_percolator_1', 'test_percolator_2']}"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("matches"));
         assertThat(matchAssertion.getExpectedValue(), instanceOf(List.class));
-        List strings = (List) matchAssertion.getExpectedValue();
+        List<?> strings = (List<?>) matchAssertion.getExpectedValue();
         assertThat(strings.size(), equalTo(2));
         assertThat(strings.get(0).toString(), equalTo("test_percolator_1"));
         assertThat(strings.get(1).toString(), equalTo("test_percolator_2"));
@@ -148,8 +140,7 @@ public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserT
                 "{ _source: { responses.0.hits.total: 3, foo: bar  }}"
         );
 
-        MatchParser matchParser = new MatchParser();
-        MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        MatchAssertion matchAssertion = MatchAssertion.parse(parser);
 
         assertThat(matchAssertion, notNullValue());
         assertThat(matchAssertion.getField(), equalTo("_source"));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
index db12c3af9b9..0947c834513 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java
@@ -19,15 +19,22 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+
+import java.util.Map;
 
 import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
-public class ClientYamlTestSectionTests extends ESTestCase {
+public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testAddingDoWithoutWarningWithoutSkip() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(SkipSection.EMPTY);
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         section.addExecutableSection(doSection);
@@ -35,7 +42,7 @@ public class ClientYamlTestSectionTests extends ESTestCase {
 
     public void testAddingDoWithWarningWithSkip() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(new SkipSection(null, singletonList("warnings"), null));
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         doSection.setExpectedWarningHeaders(singletonList("foo"));
@@ -44,7 +51,7 @@ public class ClientYamlTestSectionTests extends ESTestCase {
 
     public void testAddingDoWithWarningWithSkipButNotWarnings() {
         int lineNumber = between(1, 10000);
-        ClientYamlTestSection section = new ClientYamlTestSection("test");
+        ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test");
         section.setSkipSection(new SkipSection(null, singletonList("yaml"), null));
         DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0));
         doSection.setExpectedWarningHeaders(singletonList("foo"));
@@ -53,4 +60,207 @@ public class ClientYamlTestSectionTests extends ESTestCase {
                 + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage());
     }
 
+    public void testParseTestSectionWithDoSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                " - do :\n" +
+                "     catch: missing\n" +
+                "     indices.get_warmer:\n" +
+                "         index: test_index\n" +
+                "         name: test_warmer"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(1));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"First test section\": \n" +
+                        "  - skip:\n" +
+                        "      version:  \"2.0.0 - 2.2.0\"\n" +
+                        "      reason:   \"Update doesn't return metadata fields, waiting for #3259\"\n" +
+                        "  - do :\n" +
+                        "      catch: missing\n" +
+                        "      indices.get_warmer:\n" +
+                        "          index: test_index\n" +
+                        "          name: test_warmer\n" +
+                        "  - set: {_scroll_id: scroll_id}");
+
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("First test section"));
+        assertThat(testSection.getSkipSection(), notNullValue());
+        assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0));
+        assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        SetSection setSection = (SetSection) testSection.getExecutableSections().get(1);
+        assertThat(setSection.getStash().size(), equalTo(1));
+        assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id"));
+    }
+
+    public void testParseTestSectionWithMultipleDoSections() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(2));
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"Basic\":\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      index:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "        body:  { \"foo\": \"Hello: 中文\" }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        type:  test\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - match: { _index:   test_1 }\n" +
+                        "  - is_true: _source\n" +
+                        "  - match: { _source:  { foo: \"Hello: 中文\" } }\n" +
+                        "\n" +
+                        "  - do:\n" +
+                        "      get:\n" +
+                        "        index: test_1\n" +
+                        "        id:    中文\n" +
+                        "\n" +
+                        "  - length: { _index:   6 }\n" +
+                        "  - is_false: whatever\n" +
+                        "  - gt: { size: 5      }\n" +
+                        "  - lt: { size: 10      }"
+        );
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("Basic"));
+        assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY));
+        assertThat(testSection.getExecutableSections().size(), equalTo(10));
+
+        DoSection doSection = (DoSection)testSection.getExecutableSections().get(0);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("index"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(true));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(1);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2);
+        assertThat(matchAssertion.getField(), equalTo("_index"));
+        assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1"));
+
+        IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3);
+        assertThat(trueAssertion.getField(), equalTo("_source"));
+
+        matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4);
+        assertThat(matchAssertion.getField(), equalTo("_source"));
+        assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class));
+        Map map = (Map) matchAssertion.getExpectedValue();
+        assertThat(map.size(), equalTo(1));
+        assertThat(map.get("foo").toString(), equalTo("Hello: 中文"));
+
+        doSection = (DoSection)testSection.getExecutableSections().get(5);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("get"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+
+        LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6);
+        assertThat(lengthAssertion.getField(), equalTo("_index"));
+        assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6));
+
+        IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7);
+        assertThat(falseAssertion.getField(), equalTo("whatever"));
+
+        GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8);
+        assertThat(greaterThanAssertion.getField(), equalTo("size"));
+        assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5));
+
+        LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9);
+        assertThat(lessThanAssertion.getField(), equalTo("size"));
+        assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class));
+        assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10));
+    }
+
+    public void testSmallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "\"node_info test\":\n" +
+                "  - do:\n" +
+                "      cluster.node_info: {}\n" +
+                "  \n" +
+                "  - is_true: nodes\n" +
+                "  - is_true: cluster_name\n");
+
+        ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser);
+        assertThat(testSection, notNullValue());
+        assertThat(testSection.getName(), equalTo("node_info test"));
+        assertThat(testSection.getExecutableSections().size(), equalTo(3));
+    }
+
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
similarity index 90%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
index 801bc4e06e4..4c96986146b 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java
@@ -16,20 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser;
-import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite;
-import org.elasticsearch.test.rest.yaml.section.DoSection;
-import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion;
-import org.elasticsearch.test.rest.yaml.section.MatchAssertion;
-import org.junit.After;
 
 import java.util.Map;
 
@@ -39,21 +31,7 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class ClientYamlSuiteTestParserTests extends ESTestCase {
-    private XContentParser parser;
-
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        //makes sure that we consumed the whole stream, XContentParser doesn't expose isClosed method
-        //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times
-        assertThat(parser.currentToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        assertThat(parser.nextToken(), nullValue());
-        parser.close();
-    }
-
+public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseTestSetupTeardownAndSections() throws Exception {
         final boolean includeSetup = randomBoolean();
         final boolean includeTeardown = randomBoolean();
@@ -103,11 +81,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                         "  - match: {test_type.properties.text.analyzer: whitespace}\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
         assertThat(restTestSuite.getSetupSection(), notNullValue());
         if (includeSetup) {
             assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
@@ -207,11 +184,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                 " - match:   { _source: { foo: bar }}"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -320,11 +296,10 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                 "            params:       { bar: 'xxx' }\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser);
 
         assertThat(restTestSuite, notNullValue());
-        assertThat(restTestSuite.getName(), equalTo("suite"));
+        assertThat(restTestSuite.getName(), equalTo(getTestName()));
 
         assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
 
@@ -394,12 +369,8 @@ public class ClientYamlSuiteTestParserTests extends ESTestCase {
                         "\n"
         );
 
-        ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser();
-        try {
-            testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), containsString("duplicate test section"));
-        }
+        Exception e = expectThrows(ParsingException.class, () ->
+            ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser));
+        assertThat(e.getMessage(), containsString("duplicate test section"));
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
index e981b2d999c..463e71341bb 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java
@@ -19,16 +19,24 @@
 
 package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentLocation;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 
-public class DoSectionTests extends ESTestCase {
+public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testWarningHeaders() throws IOException {
         DoSection section = new DoSection(new XContentLocation(1, 1));
 
@@ -63,4 +71,397 @@ public class DoSectionTests extends ESTestCase {
         assertEquals("got unexpected warning headers [\ncat\n] didn't get expected warning headers [\nanother\nsome more\n]",
                 e.getMessage());
     }
+
+    public void testParseDoSectionNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "get:\n" +
+                "    index:    test_index\n" +
+                "    type:    test_type\n" +
+                "    id:        1"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("get"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_index"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test_type"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionNoParamsNoBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "cluster.node_info: {}"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("cluster.node_info"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithJsonBody() throws Exception {
+        String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }";
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   " + body
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception {
+        String bodies[] = new String[]{
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n",
+                "{ \"f1\":\"v1\", \"f2\":42 }\n",
+                "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n",
+                "{ \"f1\":\"v2\", \"f2\":47 }\n"
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: |\n" +
+                "        " + bodies[0] +
+                "        " + bodies[1] +
+                "        " + bodies[2] +
+                "        " + bodies[3]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(4));
+    }
+
+    public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception {
+        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
+            XContent.isStrictDuplicateDetectionEnabled());
+
+        String[] bodies = new String[] {
+                "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }",
+                "{ \"f1\":\"v1\", \"f2\":42 }",
+        };
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body: \n" +
+                "        " + bodies[0] + "\n" +
+                "    body: \n" +
+                "        " + bodies[1]
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBody() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "search:\n" +
+                "    body:\n" +
+                "        \"_source\": [ include.field1, include.field2 ]\n" +
+                "        \"query\": { \"match_all\": {} }"
+        );
+        String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("search"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodies() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        - index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "        - f1: v1\n" +
+                "          f2: 42\n" +
+                "        - index:\n" +
+                "            _index: test_index2\n" +
+                "            _type:  test_type2\n" +
+                "            _id:    test_id2\n" +
+                "        - f1: v2\n" +
+                "          f2: 47"
+        );
+        String[] bodies = new String[4];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+        bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\":  \"test_type2\", \"_id\": \"test_id2\"}}";
+        bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception {
+        assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled",
+            XContent.isStrictDuplicateDetectionEnabled());
+
+        parser = createParser(YamlXContent.yamlXContent,
+                "bulk:\n" +
+                "    refresh: true\n" +
+                "    body:\n" +
+                "        index:\n" +
+                "            _index: test_index\n" +
+                "            _type:  test_type\n" +
+                "            _id:    test_id\n" +
+                "    body:\n" +
+                "        f1: v1\n" +
+                "        f2: 42\n"
+        );
+        String[] bodies = new String[2];
+        bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\":  \"test_type\", \"_id\": \"test_id\"}}";
+        bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("bulk"));
+        assertThat(apiCallSection.getParams().size(), equalTo(1));
+        assertThat(apiCallSection.getParams().get("refresh"), equalTo("true"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length));
+
+        for (int i = 0; i < bodies.length; i++) {
+            assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]);
+        }
+    }
+
+    public void testParseDoSectionWithYamlBodyMultiGet() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "mget:\n" +
+                "    body:\n" +
+                "        docs:\n" +
+                "            - { _index: test_2, _type: test, _id: 1}\n" +
+                "            - { _index: test_1, _type: none, _id: 1}"
+        );
+        String body = "{ \"docs\": [ " +
+                "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " +
+                "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " +
+                "]}";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("mget"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        assertJsonEquals(apiCallSection.getBodies().get(0), body);
+    }
+
+    public void testParseDoSectionWithBodyStringified() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    index:  test_1\n" +
+                "    type:   test\n" +
+                "    id:     1\n" +
+                "    body:   \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\""
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection, notNullValue());
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(3));
+        assertThat(apiCallSection.getParams().get("index"), equalTo("test_1"));
+        assertThat(apiCallSection.getParams().get("type"), equalTo("test"));
+        assertThat(apiCallSection.getParams().get("id"), equalTo("1"));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(1));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+    }
+
+    public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "index:\n" +
+                "    body:\n" +
+                "        - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" +
+                "        - { size: 100, query: { match_all: {} } }"
+        );
+
+        String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }";
+
+        DoSection doSection = DoSection.parse(parser);
+        ApiCallSection apiCallSection = doSection.getApiCallSection();
+
+        assertThat(apiCallSection.getApi(), equalTo("index"));
+        assertThat(apiCallSection.getParams().size(), equalTo(0));
+        assertThat(apiCallSection.hasBody(), equalTo(true));
+        assertThat(apiCallSection.getBodies().size(), equalTo(2));
+        //stringified body is taken as is
+        assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }");
+        assertJsonEquals(apiCallSection.getBodies().get(1), body);
+    }
+
+    public void testParseDoSectionWithCatch() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), equalTo("missing"));
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+    }
+
+    public void testParseDoSectionWithHeaders() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "headers:\n" +
+                "    Authorization: \"thing one\"\n" +
+                "    Content-Type: \"application/json\"\n" +
+                "indices.get_warmer:\n" +
+                "    index: test_index\n" +
+                "    name: test_warmer"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getHeaders(), notNullValue());
+        assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one"));
+        assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json"));
+    }
+
+    public void testParseDoSectionWithoutClientCallSection() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "catch: missing\n"
+        );
+
+        Exception e = expectThrows(IllegalArgumentException.class, () -> DoSection.parse(parser));
+        assertThat(e.getMessage(), is("client call section is mandatory within a do section"));
+    }
+
+    public void testParseDoSectionMultivaluedField() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "        field: [ text , text1 ]"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+    }
+
+    public void testParseDoSectionExpectedWarnings() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "        type: test_type\n" +
+                "warnings:\n" +
+                "    - some test warning they are typically pretty long\n" +
+                "    - some other test warning somtimes they have [in] them"
+        );
+
+        DoSection doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping"));
+        assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2));
+        assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type"));
+        assertThat(doSection.getApiCallSection().hasBody(), equalTo(false));
+        assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0));
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList(
+                "some test warning they are typically pretty long",
+                "some other test warning somtimes they have [in] them")));
+
+        parser = createParser(YamlXContent.yamlXContent,
+                "indices.get_field_mapping:\n" +
+                "        index: test_index\n" +
+                "warnings:\n" +
+                "    - just one entry this time"
+        );
+
+        doSection = DoSection.parse(parser);
+        assertThat(doSection.getCatch(), nullValue());
+        assertThat(doSection.getApiCallSection(), notNullValue());
+        assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList(
+                "just one entry this time")));
+
+    }
+
+    private void assertJsonEquals(Map<String, Object> actual, String expected) throws IOException {
+        Map<String,Object> expectedMap;
+        try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) {
+            expectedMap = parser.mapOrdered();
+        }
+        MatcherAssert.assertThat(actual, equalTo(expectedMap));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
similarity index 69%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
index 91d35758785..74b3d774bd9 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java
@@ -16,25 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.section.SetSection;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class SetSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseSetSectionSingleValue() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                         "{ _id: id }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        SetSection setSection = SetSection.parse(parser);
         assertThat(setSection, notNullValue());
         assertThat(setSection.getStash(), notNullValue());
         assertThat(setSection.getStash().size(), equalTo(1));
@@ -46,10 +43,7 @@ public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserT
                 "{ _id: id, _type: type, _index: index }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-
-        SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        SetSection setSection = SetSection.parse(parser);
         assertThat(setSection, notNullValue());
         assertThat(setSection.getStash(), notNullValue());
         assertThat(setSection.getStash().size(), equalTo(3));
@@ -63,12 +57,7 @@ public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserT
                 "{ }"
         );
 
-        SetSectionParser setSectionParser = new SetSectionParser();
-        try {
-            setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-            fail("Expected RestTestParseException");
-        } catch (ClientYamlTestParseException e) {
-            assertThat(e.getMessage(), is("set section must set at least a value"));
-        }
+        Exception e = expectThrows(ParsingException.class, () -> SetSection.parse(parser));
+        assertThat(e.getMessage(), is("set section must set at least a value"));
     }
 }
\ No newline at end of file
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
similarity index 88%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
index e18e4a3e78e..f6174cf0be2 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
@@ -25,7 +25,7 @@ import org.elasticsearch.test.rest.yaml.section.SetupSection;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
+public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseSetupSection() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "  - do:\n" +
@@ -42,8 +42,7 @@ public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParse
                 "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
         );
 
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        SetupSection setupSection = SetupSection.parse(parser);
 
         assertThat(setupSection, notNullValue());
         assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true));
@@ -71,8 +70,7 @@ public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParse
                         "        body:   { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n"
         );
 
-        SetupSectionParser setupSectionParser = new SetupSectionParser();
-        SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
+        SetupSection setupSection = SetupSection.parse(parser);
 
         assertThat(setupSection, notNullValue());
         assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
index c8f7b351282..984cbf1715c 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java
@@ -20,12 +20,19 @@
 package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.VersionUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
 
-public class SkipSectionTests extends ESTestCase {
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
 
     public void testSkip() {
         SkipSection section = new SkipSection("2.0.0 - 2.1.0", randomBoolean() ? Collections.emptyList() :
@@ -44,4 +51,92 @@ public class SkipSectionTests extends ESTestCase {
         section = new SkipSection(null, Arrays.asList("warnings"), null);
         assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR"));
     }
+
+    public void testParseSkipSectionVersionNoFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 2.1.0\"\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionAllVersions() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+            "version:     \" all \"\n" +
+            "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion()));
+        assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT));
+        assertThat(skipSection.getFeatures().size(), equalTo(0));
+        assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param"));
+    }
+
+    public void testParseSkipSectionFeatureNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     regex"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(1));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionFeaturesNoVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "features:     [regex1,regex2,regex3]"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertThat(skipSection, notNullValue());
+        assertThat(skipSection.isVersionCheck(), equalTo(false));
+        assertThat(skipSection.getFeatures().size(), equalTo(3));
+        assertThat(skipSection.getFeatures().get(0), equalTo("regex1"));
+        assertThat(skipSection.getFeatures().get(1), equalTo("regex2"));
+        assertThat(skipSection.getFeatures().get(2), equalTo("regex3"));
+        assertThat(skipSection.getReason(), nullValue());
+    }
+
+    public void testParseSkipSectionBothFeatureAndVersion() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n" +
+                "features:     regex\n" +
+                "reason:      Delete ignores the parent param"
+        );
+
+        SkipSection skipSection = SkipSection.parse(parser);
+        assertEquals(VersionUtils.getFirstVersion(), skipSection.getLowerVersion());
+        assertEquals(Version.fromString("0.90.2"), skipSection.getUpperVersion());
+        assertEquals(Arrays.asList("regex"), skipSection.getFeatures());
+        assertEquals("Delete ignores the parent param", skipSection.getReason());
+    }
+
+    public void testParseSkipSectionNoReason() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "version:     \" - 0.90.2\"\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("reason is mandatory within skip version section"));
+    }
+
+    public void testParseSkipSectionNoVersionNorFeature() throws Exception {
+        parser = createParser(YamlXContent.yamlXContent,
+                "reason:      Delete ignores the parent param\n"
+        );
+
+        Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser));
+        assertThat(e.getMessage(), is("version or features is mandatory within skip section"));
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
similarity index 80%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
index e84eb6b6818..f057d0d370d 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java
@@ -17,22 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.test.rest.yaml.parser;
+package org.elasticsearch.test.rest.yaml.section;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext;
-import org.elasticsearch.test.rest.yaml.parser.TeardownSectionParser;
-import org.elasticsearch.test.rest.yaml.section.TeardownSection;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 /**
- * Unit tests for the teardown section parser
+ * Unit tests for the teardown section.
  */
-public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase {
-
+public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTestCase {
     public void testParseTeardownSection() throws Exception {
         parser = createParser(YamlXContent.yamlXContent,
                 "  - do:\n" +
@@ -49,9 +45,7 @@ public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentPa
                 "        ignore: 404"
         );
 
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        TeardownSection section = TeardownSection.parse(parser);
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(true));
         assertThat(section.getDoSections().size(), equalTo(2));
@@ -78,9 +72,7 @@ public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentPa
                 "        ignore: 404"
         );
 
-        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
-        TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser));
-
+        TeardownSection section = TeardownSection.parse(parser);
         assertThat(section, notNullValue());
         assertThat(section.getSkipSection().isEmpty(), equalTo(false));
         assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
index f5f1cb77a73..0845fc2546f 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
@@ -82,27 +82,42 @@ public class LoggingListenerTests extends ESTestCase {
 
         Logger abcLogger = Loggers.getLogger("abc");
         Logger xyzLogger = Loggers.getLogger("xyz");
+        // we include foo and foo.bar to maintain that logging levels are applied from the top of the hierarchy down; this ensures that
+        // setting the logging level for a parent logger and a child logger applies the parent level first and then the child as otherwise
+        // setting the parent level would overwrite the child level
+        Logger fooLogger = Loggers.getLogger("foo");
+        Logger fooBarLogger = Loggers.getLogger("foo.bar");
 
         final Level level = ESLoggerFactory.getRootLogger().getLevel();
 
         assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(level));
+        assertThat(fooLogger.getLevel(), equalTo(level));
+        assertThat(fooBarLogger.getLevel(), equalTo(level));
         loggingListener.testRunStarted(suiteDescription);
         assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         Description testDescription = Description.createTestDescription(LoggingListenerTests.class, "test");
         loggingListener.testStarted(testDescription);
         assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         loggingListener.testFinished(testDescription);
         assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooLogger.getLevel(), equalTo(Level.WARN));
+        assertThat(fooBarLogger.getLevel(), equalTo(Level.ERROR));
 
         loggingListener.testRunFinished(new Result());
         assertThat(xyzLogger.getLevel(), equalTo(level));
         assertThat(abcLogger.getLevel(), equalTo(level));
+        assertThat(fooLogger.getLevel(), equalTo(level));
+        assertThat(fooBarLogger.getLevel(), equalTo(level));
     }
 
     public void testCustomLevelPerClassAndPerMethod() throws Exception {
@@ -148,26 +163,77 @@ public class LoggingListenerTests extends ESTestCase {
         assertThat(abcLogger.getLevel(), equalTo(level));
     }
 
+    public void testInvalidClassTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidClass.class);
+
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testRunStarted(suiteDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc]"));
+    }
+
+    public void testInvalidMethodTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidMethod.class);
+
+        loggingListener.testRunStarted(suiteDescription);
+
+        final Method method = InvalidMethod.class.getMethod("invalidMethod");
+        final TestLogging annotation = method.getAnnotation(TestLogging.class);
+        Description testDescription = Description.createTestDescription(InvalidMethod.class, "invalidMethod", annotation);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testStarted(testDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc:INFO:WARN]"));
+    }
+
     /**
-     * dummy class used to create a junit suite description that has the @TestLogging annotation
+     * Dummy class used to create a JUnit suite description that has the {@link TestLogging} annotation.
      */
-    @TestLogging("abc:WARN")
+    @TestLogging("abc:WARN,foo:WARN,foo.bar:ERROR")
     public static class AnnotatedTestClass {
 
     }
 
     /**
-     * dummy class used to create a junit suite description that doesn't have the @TestLogging annotation, but its test methods have it
+     * Dummy class used to create a JUnit suite description that doesn't have the {@link TestLogging} annotation, but its test methods have
+     * it.
      */
     public static class TestClass {
 
         @SuppressWarnings("unused")
-        @TestLogging("xyz:TRACE")
-        public void annotatedTestMethod() {}
+        @TestLogging("xyz:TRACE,foo:WARN,foo.bar:ERROR")
+        public void annotatedTestMethod() {
+
+        }
 
         @SuppressWarnings("unused")
         @TestLogging("abc:TRACE,xyz:DEBUG")
-        public void annotatedTestMethod2() {}
+        public void annotatedTestMethod2() {
+
+        }
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation.
+     */
+    @TestLogging("abc")
+    public static class InvalidClass {
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation on a method.
+     */
+    public static class InvalidMethod {
+
+        @SuppressWarnings("unused")
+        @TestLogging("abc:INFO:WARN")
+        public void invalidMethod() {
+
+        }
 
     }
 
diff --git a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
index 928cc0a1fb9..4ee2e473890 100644
--- a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
+++ b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
@@ -176,7 +176,7 @@ public class ESLoggerUsageChecker {
         private final Consumer<WrongLoggerUsage> wrongUsageCallback;
         private final Predicate<String> methodsToCheck;
 
-        public ClassChecker(Consumer<WrongLoggerUsage> wrongUsageCallback, Predicate<String> methodsToCheck) {
+        ClassChecker(Consumer<WrongLoggerUsage> wrongUsageCallback, Predicate<String> methodsToCheck) {
             super(Opcodes.ASM5);
             this.wrongUsageCallback = wrongUsageCallback;
             this.methodsToCheck = methodsToCheck;
@@ -210,7 +210,7 @@ public class ESLoggerUsageChecker {
         private final Consumer<WrongLoggerUsage> wrongUsageCallback;
         private boolean ignoreChecks;
 
-        public MethodChecker(String className, int access, String name, String desc, Consumer<WrongLoggerUsage> wrongUsageCallback) {
+        MethodChecker(String className, int access, String name, String desc, Consumer<WrongLoggerUsage> wrongUsageCallback) {
             super(Opcodes.ASM5, new MethodNode(access, name, desc, null, null));
             this.className = className;
             this.wrongUsageCallback = wrongUsageCallback;
@@ -409,13 +409,13 @@ public class ESLoggerUsageChecker {
         protected final int minValue;
         protected final int maxValue;
 
-        public IntMinMaxTrackingBasicValue(Type type, int value) {
+        IntMinMaxTrackingBasicValue(Type type, int value) {
             super(type);
             this.minValue = value;
             this.maxValue = value;
         }
 
-        public IntMinMaxTrackingBasicValue(Type type, int minValue, int maxValue) {
+        IntMinMaxTrackingBasicValue(Type type, int minValue, int maxValue) {
             super(type);
             this.minValue = minValue;
             this.maxValue = maxValue;
@@ -454,27 +454,27 @@ public class ESLoggerUsageChecker {
     private static final class PlaceHolderStringBasicValue extends IntMinMaxTrackingBasicValue {
         public static final Type STRING_OBJECT_TYPE = Type.getObjectType("java/lang/String");
 
-        public PlaceHolderStringBasicValue(int placeHolders) {
+        PlaceHolderStringBasicValue(int placeHolders) {
             super(STRING_OBJECT_TYPE, placeHolders);
         }
 
-        public PlaceHolderStringBasicValue(int minPlaceHolders, int maxPlaceHolders) {
+        PlaceHolderStringBasicValue(int minPlaceHolders, int maxPlaceHolders) {
             super(STRING_OBJECT_TYPE, minPlaceHolders, maxPlaceHolders);
         }
     }
 
     private static final class ArraySizeBasicValue extends IntMinMaxTrackingBasicValue {
-        public ArraySizeBasicValue(Type type, int minArraySize, int maxArraySize) {
+        ArraySizeBasicValue(Type type, int minArraySize, int maxArraySize) {
             super(type, minArraySize, maxArraySize);
         }
     }
 
     private static final class IntegerConstantBasicValue extends IntMinMaxTrackingBasicValue {
-        public IntegerConstantBasicValue(Type type, int constant) {
+        IntegerConstantBasicValue(Type type, int constant) {
             super(type, constant);
         }
 
-        public IntegerConstantBasicValue(Type type, int minConstant, int maxConstant) {
+        IntegerConstantBasicValue(Type type, int minConstant, int maxConstant) {
             super(type, minConstant, maxConstant);
         }
     }